rattail/rattail-fabric2
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add logic for installing luigi service as part of backup app

Browse source
This commit is contained in:
Lance Edgar 2019-09-26 23:40:55 -05:00
parent 477af6c6a0
commit 067c02e965
6 changed files with 212 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

52
rattail_fabric2/backup.py
View file

@ -27,7 +27,7 @@ Fabric library for Backup app
import os
import datetime
from rattail_fabric2 import borg, python, exists, make_deploy, mkdir, UNSPECIFIED
from rattail_fabric2 import apt, borg, python, exists, make_deploy, mkdir, UNSPECIFIED
deploy_generic = make_deploy(__file__)
@ -55,6 +55,9 @@ def deploy_backup_everything(c, **context):
def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
install_borg=False,
link_borg_to_bin=True,
install_luigi=False,
luigi_history_db=None,
luigi_listen_address='0.0.0.0',
install_rattail=True,
config=None,
rattail_backup_script=None,
@ -79,6 +82,9 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
if install_borg:
borg.install_dependencies(c)
if install_luigi:
c.sudo('supervisorctl stop backup:')
# virtualenv
if mkvirtualenv:
python.mkvirtualenv(c, envname, python='/usr/bin/python3', runas_user=user)
@ -128,6 +134,50 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
if link_borg_to_bin:
c.sudo("ln -sf {}/bin/borg /usr/local/bin/borg".format(envpath))
# luigi
if install_luigi:
packages = ['luigi']
if luigi_history_db:
packages.append('SQLAlchemy')
if luigi_history_db.startswith('postgresql://'):
packages.append('psycopg2')
c.sudo("bash -l -c 'workon {}; pip install {}'".format(envname, ' '.join(packages)), user=user)
# basic config
mkdir(c, ['{}/app/luigitasks'.format(envpath),
'{}/app/luigi'.format(envpath),
'{}/app/luigi/log'.format(envpath),
'{}/app/work'.format(envpath),
], owner=user, use_sudo=True)
c.sudo('touch {}/app/luigitasks/__init__.py'.format(envpath), user=user)
deploy_generic(c, 'backup/luigi.cfg.mako', '{}/app/luigi/luigi.cfg'.format(envpath),
owner=user, mode='0600', use_sudo=True,
context={'envpath': envpath, 'history_db': luigi_history_db})
deploy_generic(c, 'backup/luigi-logging.conf.mako', '{}/app/luigi/luigi-logging.conf'.format(envpath),
owner=user, use_sudo=True, context={'envpath': envpath})
# app/luigitasks/overnight.py - should define the OvernightBackups wrapper task
path = '{}/luigi-overnight.py'.format(envname)
if deploy.local_exists(path):
deploy(c, path, '{}/app/luigitasks/overnight.py'.format(envpath),
owner=user, use_sudo=True)
else:
deploy_generic(c, 'backup/luigi-overnight.py', '{}/app/luigitasks/overnight.py'.format(envpath),
owner=user, use_sudo=True)
# app/overnight-backups.sh - generic script to invoke OvernightBackups task
deploy_generic(c, 'backup/overnight-backups.sh.mako', '{}/app/overnight-backups.sh'.format(envpath),
owner=user, mode='0755', use_sudo=True, context={'envpath': envpath})
# supervisor / luigid
apt.install(c, 'supervisor')
deploy_generic(c, 'backup/supervisor.conf.mako', '/etc/supervisor/conf.d/backup.conf',
use_sudo=True, context={
'envpath': envpath, 'user': user,
'listen_address': luigi_listen_address})
c.sudo('supervisorctl update')
c.sudo('supervisorctl start backup:')
# backup-everything script
if install_rattail or everything:
everything_context = dict(context)