Add logic for installing luigi service as part of backup app

rattail_fabric2/deploy/backup/luigi-logging.conf.mako

@@ -0,0 +1,39 @@
+## -*- mode: conf; -*-
+
+${'#'}###########################################################
+#
+# Luigi logging config
+#
+${'#'}###########################################################
+
+
+[loggers]
+keys = root
+
+[handlers]
+keys = file, console
+
+[formatters]
+keys = generic, console
+
+[logger_root]
+handlers = file, console
+level = DEBUG
+
+[handler_file]
+class = handlers.RotatingFileHandler
+args = ('${envpath}/app/luigi/log/luigi.log', 'a', 1000000, 20, 'utf_8')
+formatter = generic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = console
+level = WARNING
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_console]
+format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

rattail_fabric2/deploy/backup/luigi-overnight.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8; -*-
+"""
+Luigi tasks for "overnight backups"
+"""
+
+import subprocess
+import logging
+
+import luigi
+
+from rattail.luigi import OvernightTask
+from rattail.logging import LuigiSummaryFilter
+
+
+# log WARNING for Luigi execution summary
+logging.getLogger('luigi-interface').addFilter(LuigiSummaryFilter())
+
+
+class BackupSomething(OvernightTask):
+    """
+    Backup the 'something' machine.
+    """
+    filename = 'backup-something'
+
+    def run_command(self):
+        # note, this command would assume a "local" machine backup
+        # subprocess.check_call([
+        #     'sudo', '/usr/local/bin/backup-everything',
+        # ])
+        print('backed something up!')
+
+
+class BackupAnother(OvernightTask):
+    """
+    Backup the 'another' machine.
+    """
+    filename = 'backup-another'
+
+    # note, you must daisy-chain the tasks together, so each task "requires"
+    # the previous task.  (there probably should be a better way though?)
+    # our goal with that is just to make sure they run sequentially.
+    def requires(self):
+        return BackupSomething(self.date)
+
+    def run_command(self):
+        # note, this command would assume a "remote" machine backup
+        # (also assumes ssh keys/config have already been established)
+        # subprocess.check_call([
+        #     'ssh', '-o', 'ServerAliveInterval=120', 'another.example.com',
+        #     'sudo', '/usr/local/bin/backup-everything',
+        # ])
+        print('backed another up!')
+
+
+class OvernightBackups(luigi.WrapperTask):
+    """
+    Wrapper task for "overnight-backups" automation
+    """
+    date = luigi.DateParameter()
+
+    # this is our "wrapper" task which is invoked from `overnight-backups.sh`
+    # we list each sequential task here for clarity, even though that may be
+    # redundant due to how we daisy-chain them via requires() above (i.e. we
+    # might be able to just "require" the last task here? needs testing)
+    def requires(self):
+        yield BackupSomething(self.date)
+        yield BackupAnother(self.date)

rattail_fabric2/deploy/backup/luigi.cfg.mako

@@ -0,0 +1,17 @@
+## -*- mode: conf; -*-
+
+${'#'}###########################################################
+#
+# Luigi config
+#
+${'#'}###########################################################
+
+
+[scheduler]
+state_path = ${envpath}/app/luigi/state.pickle
+% if history_db:
+record_task_history = true
+
+[task_history]
+db_connection = ${history_db}
+% endif

rattail_fabric2/deploy/backup/overnight-backups.sh.mako

@@ -0,0 +1,28 @@
+#!/bin/bash
+${'#'}###############################################################################
+#
+# overnight "backups" automation
+#
+${'#'}###############################################################################
+
+set -e
+
+DATE=$1
+
+if [ "$1" = "--verbose" ]; then
+    DATE=$2
+    VERBOSE='--verbose'
+else
+    VERBOSE=
+fi
+
+if [ "$DATE" = "" ]; then
+    DATE=`date --date='yesterday' +%Y-%m-%d`
+fi
+
+LUIGI='${envpath}/bin/luigi --logging-conf-file luigi-logging.conf'
+export PYTHONPATH=${envpath}/app/
+
+cd ${envpath}/app/luigi
+
+$LUIGI --module luigitasks.overnight OvernightBackups --date $DATE

rattail_fabric2/deploy/backup/supervisor.conf.mako

@@ -0,0 +1,10 @@
+## -*- mode: conf; -*-
+
+[group:backup]
+programs=luigid
+
+[program:luigid]
+command=${envpath}/bin/luigid --logdir ${envpath}/app/luigi/log --state-path ${envpath}/app/luigi/state.pickle --address ${listen_address}
+directory=${envpath}/app/work
+environment=LUIGI_CONFIG_PATH="${envpath}/app/luigi/luigi.cfg"
+user=${user}