Add generic Luigi install logic

at least try to do what we can to reduce boilerplate

rattail_fabric2/deploy/luigi/cron-overnight.sh.mako

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+<%text>############################################################</%text>
+#
+# overnight automation (${automation}) via cron
+#
+<%text>############################################################</%text>
+
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    PROGRESS='--progress'
+else
+    VERBOSE=
+    PROGRESS=
+fi
+
+cd ${envroot}
+
+RATTAIL="bin/rattail --config=app/cron.conf $PROGRESS"
+
+$RATTAIL run-n-mail --no-versioning --skip-if-empty --subject 'Overnight automation: ${automation}' ${envroot}/app/overnight_${automation.lower()}.sh

rattail_fabric2/deploy/luigi/logging.conf.mako

@@ -0,0 +1,39 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# Luigi logging config
+#
+<%text>############################################################</%text>
+
+
+[loggers]
+keys = root
+
+[handlers]
+keys = file, console
+
+[formatters]
+keys = generic, console
+
+[logger_root]
+handlers = file, console
+level = DEBUG
+
+[handler_file]
+class = handlers.WatchedFileHandler
+args = ('${appdir}/luigi/log/luigi.log', 'a', 'utf_8')
+formatter = generic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = console
+level = WARNING
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_console]
+format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

rattail_fabric2/deploy/luigi/luigi-logrotate.conf.mako

@@ -0,0 +1,24 @@
+## -*- mode: conf; -*-
+
+${appdir}/luigi/log/luigi.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 600 rattail rattail
+}
+
+${appdir}/luigi/log/luigi-server.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 600 rattail rattail
+    postrotate
+        supervisorctl restart luigi:luigid > /dev/null
+    endscript
+}

rattail_fabric2/deploy/luigi/luigi.cfg.mako

@@ -0,0 +1,36 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# Luigi config
+#
+# cf. https://luigi.readthedocs.io/en/stable/configuration.html
+#
+<%text>############################################################</%text>
+
+
+# [core]
+
+# # Number of seconds to wait before timing out when making an API call. Defaults
+# # to 10.0
+# # (sometimes things can lag for us and we simply need to give it more time)
+# rpc_connect_timeout = 60
+
+# # The maximum number of retries to connect the central scheduler before giving
+# # up. Defaults to 3
+# # (occasional network issues seem to cause us to need more/longer retries)
+# rpc_retry_attempts = 10
+
+# # Number of seconds to wait before the next attempt will be started to connect
+# # to the central scheduler between two retry attempts. Defaults to 30
+# # (occasional network issues seem to cause us to need more/longer retries)
+# rpc_retry_wait = 60
+
+[scheduler]
+state_path = ${appdir}/luigi/state.pickle
+% if db_connection:
+record_task_history = true
+
+[task_history]
+db_connection = ${db_connection}
+% endif

rattail_fabric2/deploy/luigi/overnight.sh.mako

@@ -0,0 +1,28 @@
+#!/bin/bash
+<%text>############################################################</%text>
+#
+# overnight automation for '${automation}'
+#
+<%text>############################################################</%text>
+
+set -e
+
+DATE=$1
+
+if [ "$1" = "--verbose" ]; then
+    DATE=$2
+    VERBOSE='--verbose'
+else
+    VERBOSE=
+fi
+
+if [ "$DATE" = "" ]; then
+    DATE=`date --date='yesterday' +%Y-%m-%d`
+fi
+
+LUIGI='${envroot}/bin/luigi --logging-conf-file logging.conf'
+export PYTHONPATH=${appdir}
+
+cd ${appdir}/luigi
+
+$LUIGI --module luigitasks.overnight_${automation.lower()} Overnight${automation} --date $DATE

rattail_fabric2/deploy/luigi/restart-overnight.sh.mako

@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+DATE=`date --date='yesterday' +%Y-%m-%d`
+
+echo "${envroot}/bin/rattail -c ${appdir}/cron.conf --no-versioning run-n-mail -S 'Overnight catch-up: ${automation}' '${appdir}/overnight-${automation.lower()}.sh $DATE'" | at 'now + 1 minute'

rattail_fabric2/deploy/luigi/rotate-logs.sh.mako

@@ -0,0 +1,14 @@
+#!/bin/sh -e
+<%text>######################################################################</%text>
+#
+# rotate Luigi server log file
+#
+<%text>######################################################################</%text>
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+else
+    VERBOSE=
+fi
+
+/usr/sbin/logrotate $VERBOSE ${appdir}/luigi/logrotate.conf

rattail_fabric2/deploy/luigi/supervisor.conf.mako

@@ -0,0 +1,11 @@
+## -*- mode: conf; -*-
+
+[group:luigi]
+programs=luigid
+
+[program:luigid]
+command=${envroot}/bin/luigid --logdir ${appdir}/luigi/log --state-path ${appdir}/luigi/state.pickle --address 127.0.0.1
+directory=${appdir}/work
+environment=LUIGI_CONFIG_PATH="${appdir}/luigi/luigi.cfg"
+user=${user}
+autostart=${'true' if autostart else 'false'}

rattail_fabric2/luigi.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2022 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Fabric library for Luigi apps
+"""
+
+import os
+
+from rattail_fabric2 import make_deploy, mkdir
+
+
+deploy_common = make_deploy(__file__)
+
+
+def install_luigi(c, envroot, user='rattail', db_connection=None):
+    """
+    Install and configure Luigi to the given virtualenv.
+    """
+    envroot = envroot.rstrip('/')
+    envname = os.path.basename(envroot)
+    appdir = '{}/app'.format(envroot)
+
+    # package
+    c.sudo("bash -lc 'workon {} && pip install luigi'".format(envname),
+           user=user)
+
+    # dirs
+    mkdir(c, ['{}/luigi'.format(appdir),
+              '{}/luigi/log'.format(appdir),
+              '{}/luigitasks'.format(appdir),
+    ], use_sudo=True, owner=user)
+
+    # tasks
+    c.sudo('touch {}/luigitasks/__init__.py'.format(appdir),
+           user=user)
+
+    # config
+    deploy_common(c, 'luigi/luigi.cfg.mako', '{}/luigi/luigi.cfg'.format(appdir),
+                  use_sudo=True, owner=user, mode='0640',
+                  context={'appdir': appdir,
+                           'db_connection': db_connection})
+    deploy_common(c, 'luigi/logging.conf.mako', '{}/luigi/logging.conf'.format(appdir),
+                  use_sudo=True, owner=user,
+                  context={'appdir': appdir})
+
+    # logrotate
+    deploy_common(c, 'luigi/luigi-logrotate.conf.mako', '{}/luigi/logrotate.conf'.format(appdir),
+                  use_sudo=True, owner='root:', # must be owned by root (TODO: why is that again?)
+                  context={'appdir': appdir})
+    deploy_common(c, 'luigi/rotate-logs.sh.mako', '{}/rotate-luigi-logs.sh'.format(appdir),
+                  use_sudo=True, owner=user,
+                  context={'appdir': appdir})
+
+
+def install_overnight_script(c, envroot, user='rattail', automation='All'):
+    """
+    Install an overnight automation script
+    """
+    envroot = envroot.rstrip('/')
+    appdir = '{}/app'.format(envroot)
+
+    # overnight-*.sh
+    filename = 'overnight-{}.sh'.format(automation.lower())
+    deploy_common(c, 'luigi/overnight.sh.mako',
+                  '{}/{}'.format(appdir, filename),
+                  use_sudo=True, owner=user, mode='0755',
+                  context={'envroot': envroot, 'appdir': appdir,
+                           'automation': automation})
+
+    # cron-overnight-*.sh
+    filename = 'cron-overnight-{}.sh'.format(automation.lower())
+    deploy_common(c, 'luigi/cron-overnight.sh.mako',
+                  '{}/{}'.format(appdir, filename),
+                  use_sudo=True, owner=user, mode='0755',
+                  context={'envroot': envroot, 
+                           'automation': automation})
+
+    # restart-overnight-*.sh
+    filename = 'restart-overnight-{}.sh'.format(automation.lower())
+    deploy_common(c, 'luigi/restart-overnight.sh.mako',
+                  '{}/{}'.format(appdir, filename),
+                  use_sudo=True, owner=user, mode='0755',
+                  context={'envroot': envroot, 
+                           'appdir': appdir,
+                           'automation': automation})
+
+
+def register_with_supervisor(c, envroot, user='rattail', autostart=False):
+    """
+    Register the Luigi scheduler daemon with supervisor
+    """
+    envroot = envroot.rstrip('/')
+    appdir = '{}/app'.format(envroot)
+
+    deploy_common(c, 'luigi/supervisor.conf.mako',
+                  '/etc/supervisor/conf.d/luigi.conf',
+                  use_sudo=True,
+                  context={'envroot': envroot,
+                           'appdir': appdir,
+                           'user': user,
+                           'autostart': autostart})
+    c.sudo('supervisorctl update')
+    if autostart:
+        c.sudo('supervisorctl start luigi:')