Add logic for installing luigi service as part of backup app

rattail_fabric2/backup.py

@@ -27,7 +27,7 @@ Fabric library for Backup app
 import os
 import datetime
 
-from rattail_fabric2 import borg, python, exists, make_deploy, mkdir, UNSPECIFIED
+from rattail_fabric2 import apt, borg, python, exists, make_deploy, mkdir, UNSPECIFIED
 
 
 deploy_generic = make_deploy(__file__)
@@ -55,6 +55,9 @@ def deploy_backup_everything(c, **context):
 def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
                       install_borg=False,
                       link_borg_to_bin=True,
+                      install_luigi=False,
+                      luigi_history_db=None,
+                      luigi_listen_address='0.0.0.0',
                       install_rattail=True,
                       config=None,
                       rattail_backup_script=None,
@@ -79,6 +82,9 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
     if install_borg:
         borg.install_dependencies(c)
 
+    if install_luigi:
+        c.sudo('supervisorctl stop backup:')
+
     # virtualenv
     if mkvirtualenv:
         python.mkvirtualenv(c, envname, python='/usr/bin/python3', runas_user=user)
@@ -128,6 +134,50 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
         if link_borg_to_bin:
             c.sudo("ln -sf {}/bin/borg /usr/local/bin/borg".format(envpath))
 
+    # luigi
+    if install_luigi:
+        packages = ['luigi']
+        if luigi_history_db:
+            packages.append('SQLAlchemy')
+            if luigi_history_db.startswith('postgresql://'):
+                packages.append('psycopg2')
+        c.sudo("bash -l -c 'workon {}; pip install {}'".format(envname, ' '.join(packages)), user=user)
+
+        # basic config
+        mkdir(c, ['{}/app/luigitasks'.format(envpath),
+                  '{}/app/luigi'.format(envpath),
+                  '{}/app/luigi/log'.format(envpath),
+                  '{}/app/work'.format(envpath),
+        ], owner=user, use_sudo=True)
+        c.sudo('touch {}/app/luigitasks/__init__.py'.format(envpath), user=user)
+        deploy_generic(c, 'backup/luigi.cfg.mako', '{}/app/luigi/luigi.cfg'.format(envpath),
+                       owner=user, mode='0600', use_sudo=True,
+                       context={'envpath': envpath, 'history_db': luigi_history_db})
+        deploy_generic(c, 'backup/luigi-logging.conf.mako', '{}/app/luigi/luigi-logging.conf'.format(envpath),
+                       owner=user, use_sudo=True, context={'envpath': envpath})
+
+        # app/luigitasks/overnight.py - should define the OvernightBackups wrapper task
+        path = '{}/luigi-overnight.py'.format(envname)
+        if deploy.local_exists(path):
+            deploy(c, path, '{}/app/luigitasks/overnight.py'.format(envpath),
+                   owner=user, use_sudo=True)
+        else:
+            deploy_generic(c, 'backup/luigi-overnight.py', '{}/app/luigitasks/overnight.py'.format(envpath),
+                           owner=user, use_sudo=True)
+
+        # app/overnight-backups.sh - generic script to invoke OvernightBackups task
+        deploy_generic(c, 'backup/overnight-backups.sh.mako', '{}/app/overnight-backups.sh'.format(envpath),
+                       owner=user, mode='0755', use_sudo=True, context={'envpath': envpath})
+
+        # supervisor / luigid
+        apt.install(c, 'supervisor')
+        deploy_generic(c, 'backup/supervisor.conf.mako', '/etc/supervisor/conf.d/backup.conf',
+                       use_sudo=True, context={
+                           'envpath': envpath, 'user': user,
+                           'listen_address': luigi_listen_address})
+        c.sudo('supervisorctl update')
+        c.sudo('supervisorctl start backup:')
+
     # backup-everything script
     if install_rattail or everything:
         everything_context = dict(context)

rattail_fabric2/deploy/backup/luigi-logging.conf.mako

@@ -0,0 +1,39 @@
+## -*- mode: conf; -*-
+
+${'#'}###########################################################
+#
+# Luigi logging config
+#
+${'#'}###########################################################
+
+
+[loggers]
+keys = root
+
+[handlers]
+keys = file, console
+
+[formatters]
+keys = generic, console
+
+[logger_root]
+handlers = file, console
+level = DEBUG
+
+[handler_file]
+class = handlers.RotatingFileHandler
+args = ('${envpath}/app/luigi/log/luigi.log', 'a', 1000000, 20, 'utf_8')
+formatter = generic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = console
+level = WARNING
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_console]
+format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

rattail_fabric2/deploy/backup/luigi-overnight.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8; -*-
+"""
+Luigi tasks for "overnight backups"
+"""
+
+import subprocess
+import logging
+
+import luigi
+
+from rattail.luigi import OvernightTask
+from rattail.logging import LuigiSummaryFilter
+
+
+# log WARNING for Luigi execution summary
+logging.getLogger('luigi-interface').addFilter(LuigiSummaryFilter())
+
+
+class BackupSomething(OvernightTask):
+    """
+    Backup the 'something' machine.
+    """
+    filename = 'backup-something'
+
+    def run_command(self):
+        # note, this command would assume a "local" machine backup
+        # subprocess.check_call([
+        #     'sudo', '/usr/local/bin/backup-everything',
+        # ])
+        print('backed something up!')
+
+
+class BackupAnother(OvernightTask):
+    """
+    Backup the 'another' machine.
+    """
+    filename = 'backup-another'
+
+    # note, you must daisy-chain the tasks together, so each task "requires"
+    # the previous task.  (there probably should be a better way though?)
+    # our goal with that is just to make sure they run sequentially.
+    def requires(self):
+        return BackupSomething(self.date)
+
+    def run_command(self):
+        # note, this command would assume a "remote" machine backup
+        # (also assumes ssh keys/config have already been established)
+        # subprocess.check_call([
+        #     'ssh', '-o', 'ServerAliveInterval=120', 'another.example.com',
+        #     'sudo', '/usr/local/bin/backup-everything',
+        # ])
+        print('backed another up!')
+
+
+class OvernightBackups(luigi.WrapperTask):
+    """
+    Wrapper task for "overnight-backups" automation
+    """
+    date = luigi.DateParameter()
+
+    # this is our "wrapper" task which is invoked from `overnight-backups.sh`
+    # we list each sequential task here for clarity, even though that may be
+    # redundant due to how we daisy-chain them via requires() above (i.e. we
+    # might be able to just "require" the last task here? needs testing)
+    def requires(self):
+        yield BackupSomething(self.date)
+        yield BackupAnother(self.date)

rattail_fabric2/deploy/backup/luigi.cfg.mako

@@ -0,0 +1,17 @@
+## -*- mode: conf; -*-
+
+${'#'}###########################################################
+#
+# Luigi config
+#
+${'#'}###########################################################
+
+
+[scheduler]
+state_path = ${envpath}/app/luigi/state.pickle
+% if history_db:
+record_task_history = true
+
+[task_history]
+db_connection = ${history_db}
+% endif

rattail_fabric2/deploy/backup/overnight-backups.sh.mako

@@ -0,0 +1,28 @@
+#!/bin/bash
+${'#'}###############################################################################
+#
+# overnight "backups" automation
+#
+${'#'}###############################################################################
+
+set -e
+
+DATE=$1
+
+if [ "$1" = "--verbose" ]; then
+    DATE=$2
+    VERBOSE='--verbose'
+else
+    VERBOSE=
+fi
+
+if [ "$DATE" = "" ]; then
+    DATE=`date --date='yesterday' +%Y-%m-%d`
+fi
+
+LUIGI='${envpath}/bin/luigi --logging-conf-file luigi-logging.conf'
+export PYTHONPATH=${envpath}/app/
+
+cd ${envpath}/app/luigi
+
+$LUIGI --module luigitasks.overnight OvernightBackups --date $DATE

rattail_fabric2/deploy/backup/supervisor.conf.mako

@@ -0,0 +1,10 @@
+## -*- mode: conf; -*-
+
+[group:backup]
+programs=luigid
+
+[program:luigid]
+command=${envpath}/bin/luigid --logdir ${envpath}/app/luigi/log --state-path ${envpath}/app/luigi/state.pickle --address ${listen_address}
+directory=${envpath}/app/work
+environment=LUIGI_CONFIG_PATH="${envpath}/app/luigi/luigi.cfg"
+user=${user}