Add generic Luigi install logic

at least try to do what we can to reduce boilerplate

rattail_fabric2/deploy/luigi/cron-overnight.sh.mako

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+<%text>############################################################</%text>
+#
+# overnight automation (${automation}) via cron
+#
+<%text>############################################################</%text>
+
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    PROGRESS='--progress'
+else
+    VERBOSE=
+    PROGRESS=
+fi
+
+cd ${envroot}
+
+RATTAIL="bin/rattail --config=app/cron.conf $PROGRESS"
+
+$RATTAIL run-n-mail --no-versioning --skip-if-empty --subject 'Overnight automation: ${automation}' ${envroot}/app/overnight_${automation.lower()}.sh

rattail_fabric2/deploy/luigi/logging.conf.mako

@@ -0,0 +1,39 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# Luigi logging config
+#
+<%text>############################################################</%text>
+
+
+[loggers]
+keys = root
+
+[handlers]
+keys = file, console
+
+[formatters]
+keys = generic, console
+
+[logger_root]
+handlers = file, console
+level = DEBUG
+
+[handler_file]
+class = handlers.WatchedFileHandler
+args = ('${appdir}/luigi/log/luigi.log', 'a', 'utf_8')
+formatter = generic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = console
+level = WARNING
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_console]
+format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

rattail_fabric2/deploy/luigi/luigi-logrotate.conf.mako

@@ -0,0 +1,24 @@
+## -*- mode: conf; -*-
+
+${appdir}/luigi/log/luigi.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 600 rattail rattail
+}
+
+${appdir}/luigi/log/luigi-server.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 600 rattail rattail
+    postrotate
+        supervisorctl restart luigi:luigid > /dev/null
+    endscript
+}

rattail_fabric2/deploy/luigi/luigi.cfg.mako

@@ -0,0 +1,36 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# Luigi config
+#
+# cf. https://luigi.readthedocs.io/en/stable/configuration.html
+#
+<%text>############################################################</%text>
+
+
+# [core]
+
+# # Number of seconds to wait before timing out when making an API call. Defaults
+# # to 10.0
+# # (sometimes things can lag for us and we simply need to give it more time)
+# rpc_connect_timeout = 60
+
+# # The maximum number of retries to connect the central scheduler before giving
+# # up. Defaults to 3
+# # (occasional network issues seem to cause us to need more/longer retries)
+# rpc_retry_attempts = 10
+
+# # Number of seconds to wait before the next attempt will be started to connect
+# # to the central scheduler between two retry attempts. Defaults to 30
+# # (occasional network issues seem to cause us to need more/longer retries)
+# rpc_retry_wait = 60
+
+[scheduler]
+state_path = ${appdir}/luigi/state.pickle
+% if db_connection:
+record_task_history = true
+
+[task_history]
+db_connection = ${db_connection}
+% endif

rattail_fabric2/deploy/luigi/overnight.sh.mako

@@ -0,0 +1,28 @@
+#!/bin/bash
+<%text>############################################################</%text>
+#
+# overnight automation for '${automation}'
+#
+<%text>############################################################</%text>
+
+set -e
+
+DATE=$1
+
+if [ "$1" = "--verbose" ]; then
+    DATE=$2
+    VERBOSE='--verbose'
+else
+    VERBOSE=
+fi
+
+if [ "$DATE" = "" ]; then
+    DATE=`date --date='yesterday' +%Y-%m-%d`
+fi
+
+LUIGI='${envroot}/bin/luigi --logging-conf-file logging.conf'
+export PYTHONPATH=${appdir}
+
+cd ${appdir}/luigi
+
+$LUIGI --module luigitasks.overnight_${automation.lower()} Overnight${automation} --date $DATE

rattail_fabric2/deploy/luigi/restart-overnight.sh.mako

@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+DATE=`date --date='yesterday' +%Y-%m-%d`
+
+echo "${envroot}/bin/rattail -c ${appdir}/cron.conf --no-versioning run-n-mail -S 'Overnight catch-up: ${automation}' '${appdir}/overnight-${automation.lower()}.sh $DATE'" | at 'now + 1 minute'

rattail_fabric2/deploy/luigi/rotate-logs.sh.mako

@@ -0,0 +1,14 @@
+#!/bin/sh -e
+<%text>######################################################################</%text>
+#
+# rotate Luigi server log file
+#
+<%text>######################################################################</%text>
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+else
+    VERBOSE=
+fi
+
+/usr/sbin/logrotate $VERBOSE ${appdir}/luigi/logrotate.conf

rattail_fabric2/deploy/luigi/supervisor.conf.mako

@@ -0,0 +1,11 @@
+## -*- mode: conf; -*-
+
+[group:luigi]
+programs=luigid
+
+[program:luigid]
+command=${envroot}/bin/luigid --logdir ${appdir}/luigi/log --state-path ${appdir}/luigi/state.pickle --address 127.0.0.1
+directory=${appdir}/work
+environment=LUIGI_CONFIG_PATH="${appdir}/luigi/luigi.cfg"
+user=${user}
+autostart=${'true' if autostart else 'false'}