Add logic for installing luigi service as part of backup app

This commit is contained in:
Lance Edgar 2019-09-26 23:40:55 -05:00
parent 477af6c6a0
commit 067c02e965
6 changed files with 212 additions and 1 deletions

View file

@ -27,7 +27,7 @@ Fabric library for Backup app
import os import os
import datetime import datetime
from rattail_fabric2 import borg, python, exists, make_deploy, mkdir, UNSPECIFIED from rattail_fabric2 import apt, borg, python, exists, make_deploy, mkdir, UNSPECIFIED
deploy_generic = make_deploy(__file__) deploy_generic = make_deploy(__file__)
@ -55,6 +55,9 @@ def deploy_backup_everything(c, **context):
def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail', def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
install_borg=False, install_borg=False,
link_borg_to_bin=True, link_borg_to_bin=True,
install_luigi=False,
luigi_history_db=None,
luigi_listen_address='0.0.0.0',
install_rattail=True, install_rattail=True,
config=None, config=None,
rattail_backup_script=None, rattail_backup_script=None,
@ -79,6 +82,9 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
if install_borg: if install_borg:
borg.install_dependencies(c) borg.install_dependencies(c)
if install_luigi:
c.sudo('supervisorctl stop backup:')
# virtualenv # virtualenv
if mkvirtualenv: if mkvirtualenv:
python.mkvirtualenv(c, envname, python='/usr/bin/python3', runas_user=user) python.mkvirtualenv(c, envname, python='/usr/bin/python3', runas_user=user)
@ -128,6 +134,50 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
if link_borg_to_bin: if link_borg_to_bin:
c.sudo("ln -sf {}/bin/borg /usr/local/bin/borg".format(envpath)) c.sudo("ln -sf {}/bin/borg /usr/local/bin/borg".format(envpath))
# luigi
if install_luigi:
packages = ['luigi']
if luigi_history_db:
packages.append('SQLAlchemy')
if luigi_history_db.startswith('postgresql://'):
packages.append('psycopg2')
c.sudo("bash -l -c 'workon {}; pip install {}'".format(envname, ' '.join(packages)), user=user)
# basic config
mkdir(c, ['{}/app/luigitasks'.format(envpath),
'{}/app/luigi'.format(envpath),
'{}/app/luigi/log'.format(envpath),
'{}/app/work'.format(envpath),
], owner=user, use_sudo=True)
c.sudo('touch {}/app/luigitasks/__init__.py'.format(envpath), user=user)
deploy_generic(c, 'backup/luigi.cfg.mako', '{}/app/luigi/luigi.cfg'.format(envpath),
owner=user, mode='0600', use_sudo=True,
context={'envpath': envpath, 'history_db': luigi_history_db})
deploy_generic(c, 'backup/luigi-logging.conf.mako', '{}/app/luigi/luigi-logging.conf'.format(envpath),
owner=user, use_sudo=True, context={'envpath': envpath})
# app/luigitasks/overnight.py - should define the OvernightBackups wrapper task
path = '{}/luigi-overnight.py'.format(envname)
if deploy.local_exists(path):
deploy(c, path, '{}/app/luigitasks/overnight.py'.format(envpath),
owner=user, use_sudo=True)
else:
deploy_generic(c, 'backup/luigi-overnight.py', '{}/app/luigitasks/overnight.py'.format(envpath),
owner=user, use_sudo=True)
# app/overnight-backups.sh - generic script to invoke OvernightBackups task
deploy_generic(c, 'backup/overnight-backups.sh.mako', '{}/app/overnight-backups.sh'.format(envpath),
owner=user, mode='0755', use_sudo=True, context={'envpath': envpath})
# supervisor / luigid
apt.install(c, 'supervisor')
deploy_generic(c, 'backup/supervisor.conf.mako', '/etc/supervisor/conf.d/backup.conf',
use_sudo=True, context={
'envpath': envpath, 'user': user,
'listen_address': luigi_listen_address})
c.sudo('supervisorctl update')
c.sudo('supervisorctl start backup:')
# backup-everything script # backup-everything script
if install_rattail or everything: if install_rattail or everything:
everything_context = dict(context) everything_context = dict(context)

View file

@ -0,0 +1,39 @@
## -*- mode: conf; -*-
${'#'}###########################################################
#
# Luigi logging config
#
${'#'}###########################################################
[loggers]
keys = root
[handlers]
keys = file, console
[formatters]
keys = generic, console
[logger_root]
handlers = file, console
level = DEBUG
[handler_file]
class = handlers.RotatingFileHandler
args = ('${envpath}/app/luigi/log/luigi.log', 'a', 1000000, 20, 'utf_8')
formatter = generic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
formatter = console
level = WARNING
[formatter_generic]
format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
datefmt = %Y-%m-%d %H:%M:%S
[formatter_console]
format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

View file

@ -0,0 +1,67 @@
# -*- coding: utf-8; -*-
"""
Luigi tasks for "overnight backups"
"""
import subprocess
import logging
import luigi
from rattail.luigi import OvernightTask
from rattail.logging import LuigiSummaryFilter
# log WARNING for Luigi execution summary
logging.getLogger('luigi-interface').addFilter(LuigiSummaryFilter())
class BackupSomething(OvernightTask):
"""
Backup the 'something' machine.
"""
filename = 'backup-something'
def run_command(self):
# note, this command would assume a "local" machine backup
# subprocess.check_call([
# 'sudo', '/usr/local/bin/backup-everything',
# ])
print('backed something up!')
class BackupAnother(OvernightTask):
"""
Backup the 'another' machine.
"""
filename = 'backup-another'
# note, you must daisy-chain the tasks together, so each task "requires"
# the previous task. (there probably should be a better way though?)
# our goal with that is just to make sure they run sequentially.
def requires(self):
return BackupSomething(self.date)
def run_command(self):
# note, this command would assume a "remote" machine backup
# (also assumes ssh keys/config have already been established)
# subprocess.check_call([
# 'ssh', '-o', 'ServerAliveInterval=120', 'another.example.com',
# 'sudo', '/usr/local/bin/backup-everything',
# ])
print('backed another up!')
class OvernightBackups(luigi.WrapperTask):
"""
Wrapper task for "overnight-backups" automation
"""
date = luigi.DateParameter()
# this is our "wrapper" task which is invoked from `overnight-backups.sh`
# we list each sequential task here for clarity, even though that may be
# redundant due to how we daisy-chain them via requires() above (i.e. we
# might be able to just "require" the last task here? needs testing)
def requires(self):
yield BackupSomething(self.date)
yield BackupAnother(self.date)

View file

@ -0,0 +1,17 @@
## -*- mode: conf; -*-
${'#'}###########################################################
#
# Luigi config
#
${'#'}###########################################################
[scheduler]
state_path = ${envpath}/app/luigi/state.pickle
% if history_db:
record_task_history = true
[task_history]
db_connection = ${history_db}
% endif

View file

@ -0,0 +1,28 @@
#!/bin/bash
${'#'}###############################################################################
#
# overnight "backups" automation
#
${'#'}###############################################################################
set -e
DATE=$1
if [ "$1" = "--verbose" ]; then
DATE=$2
VERBOSE='--verbose'
else
VERBOSE=
fi
if [ "$DATE" = "" ]; then
DATE=`date --date='yesterday' +%Y-%m-%d`
fi
LUIGI='${envpath}/bin/luigi --logging-conf-file luigi-logging.conf'
export PYTHONPATH=${envpath}/app/
cd ${envpath}/app/luigi
$LUIGI --module luigitasks.overnight OvernightBackups --date $DATE

View file

@ -0,0 +1,10 @@
## -*- mode: conf; -*-
[group:backup]
programs=luigid
[program:luigid]
command=${envpath}/bin/luigid --logdir ${envpath}/app/luigi/log --state-path ${envpath}/app/luigi/state.pickle --address ${listen_address}
directory=${envpath}/app/work
environment=LUIGI_CONFIG_PATH="${envpath}/app/luigi/luigi.cfg"
user=${user}