From 6bf697da1dd5b79a547af8deb30d268409e735a8 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 28 Jan 2022 15:29:32 -0600 Subject: [PATCH] Add generic Luigi install logic at least try to do what we can to reduce boilerplate --- .../deploy/luigi/cron-overnight.sh.mako | 21 +++ .../deploy/luigi/logging.conf.mako | 39 ++++++ .../deploy/luigi/luigi-logrotate.conf.mako | 24 ++++ rattail_fabric2/deploy/luigi/luigi.cfg.mako | 36 +++++ .../deploy/luigi/overnight.sh.mako | 28 ++++ .../deploy/luigi/restart-overnight.sh.mako | 5 + .../deploy/luigi/rotate-logs.sh.mako | 14 ++ .../deploy/luigi/supervisor.conf.mako | 11 ++ rattail_fabric2/luigi.py | 124 ++++++++++++++++++ 9 files changed, 302 insertions(+) create mode 100755 rattail_fabric2/deploy/luigi/cron-overnight.sh.mako create mode 100644 rattail_fabric2/deploy/luigi/logging.conf.mako create mode 100644 rattail_fabric2/deploy/luigi/luigi-logrotate.conf.mako create mode 100644 rattail_fabric2/deploy/luigi/luigi.cfg.mako create mode 100755 rattail_fabric2/deploy/luigi/overnight.sh.mako create mode 100644 rattail_fabric2/deploy/luigi/restart-overnight.sh.mako create mode 100755 rattail_fabric2/deploy/luigi/rotate-logs.sh.mako create mode 100644 rattail_fabric2/deploy/luigi/supervisor.conf.mako create mode 100644 rattail_fabric2/luigi.py diff --git a/rattail_fabric2/deploy/luigi/cron-overnight.sh.mako b/rattail_fabric2/deploy/luigi/cron-overnight.sh.mako new file mode 100755 index 0000000..21df735 --- /dev/null +++ b/rattail_fabric2/deploy/luigi/cron-overnight.sh.mako @@ -0,0 +1,21 @@ +#!/bin/sh -e +<%text>############################################################ +# +# overnight automation (${automation}) via cron +# +<%text>############################################################ + + +if [ "$1" = "--verbose" ]; then + VERBOSE='--verbose' + PROGRESS='--progress' +else + VERBOSE= + PROGRESS= +fi + +cd ${envroot} + +RATTAIL="bin/rattail --config=app/cron.conf $PROGRESS" + +$RATTAIL run-n-mail --no-versioning --skip-if-empty --subject 'Overnight automation: ${automation}' ${envroot}/app/overnight_${automation.lower()}.sh diff --git a/rattail_fabric2/deploy/luigi/logging.conf.mako b/rattail_fabric2/deploy/luigi/logging.conf.mako new file mode 100644 index 0000000..c8cead9 --- /dev/null +++ b/rattail_fabric2/deploy/luigi/logging.conf.mako @@ -0,0 +1,39 @@ +## -*- mode: conf; -*- + +<%text>############################################################ +# +# Luigi logging config +# +<%text>############################################################ + + +[loggers] +keys = root + +[handlers] +keys = file, console + +[formatters] +keys = generic, console + +[logger_root] +handlers = file, console +level = DEBUG + +[handler_file] +class = handlers.WatchedFileHandler +args = ('${appdir}/luigi/log/luigi.log', 'a', 'utf_8') +formatter = generic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +formatter = console +level = WARNING + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S + +[formatter_console] +format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s diff --git a/rattail_fabric2/deploy/luigi/luigi-logrotate.conf.mako b/rattail_fabric2/deploy/luigi/luigi-logrotate.conf.mako new file mode 100644 index 0000000..79aa18b --- /dev/null +++ b/rattail_fabric2/deploy/luigi/luigi-logrotate.conf.mako @@ -0,0 +1,24 @@ +## -*- mode: conf; -*- + +${appdir}/luigi/log/luigi.log { + daily + missingok + rotate 30 + compress + delaycompress + notifempty + create 600 rattail rattail +} + +${appdir}/luigi/log/luigi-server.log { + daily + missingok + rotate 30 + compress + delaycompress + notifempty + create 600 rattail rattail + postrotate + supervisorctl restart luigi:luigid > /dev/null + endscript +} diff --git a/rattail_fabric2/deploy/luigi/luigi.cfg.mako b/rattail_fabric2/deploy/luigi/luigi.cfg.mako new file mode 100644 index 0000000..da7c25b --- /dev/null +++ b/rattail_fabric2/deploy/luigi/luigi.cfg.mako @@ -0,0 +1,36 @@ +## -*- mode: conf; -*- + +<%text>############################################################ +# +# Luigi config +# +# cf. https://luigi.readthedocs.io/en/stable/configuration.html +# +<%text>############################################################ + + +# [core] + +# # Number of seconds to wait before timing out when making an API call. Defaults +# # to 10.0 +# # (sometimes things can lag for us and we simply need to give it more time) +# rpc_connect_timeout = 60 + +# # The maximum number of retries to connect the central scheduler before giving +# # up. Defaults to 3 +# # (occasional network issues seem to cause us to need more/longer retries) +# rpc_retry_attempts = 10 + +# # Number of seconds to wait before the next attempt will be started to connect +# # to the central scheduler between two retry attempts. Defaults to 30 +# # (occasional network issues seem to cause us to need more/longer retries) +# rpc_retry_wait = 60 + +[scheduler] +state_path = ${appdir}/luigi/state.pickle +% if db_connection: +record_task_history = true + +[task_history] +db_connection = ${db_connection} +% endif diff --git a/rattail_fabric2/deploy/luigi/overnight.sh.mako b/rattail_fabric2/deploy/luigi/overnight.sh.mako new file mode 100755 index 0000000..a5ba8eb --- /dev/null +++ b/rattail_fabric2/deploy/luigi/overnight.sh.mako @@ -0,0 +1,28 @@ +#!/bin/bash +<%text>############################################################ +# +# overnight automation for '${automation}' +# +<%text>############################################################ + +set -e + +DATE=$1 + +if [ "$1" = "--verbose" ]; then + DATE=$2 + VERBOSE='--verbose' +else + VERBOSE= +fi + +if [ "$DATE" = "" ]; then + DATE=`date --date='yesterday' +%Y-%m-%d` +fi + +LUIGI='${envroot}/bin/luigi --logging-conf-file logging.conf' +export PYTHONPATH=${appdir} + +cd ${appdir}/luigi + +$LUIGI --module luigitasks.overnight_${automation.lower()} Overnight${automation} --date $DATE diff --git a/rattail_fabric2/deploy/luigi/restart-overnight.sh.mako b/rattail_fabric2/deploy/luigi/restart-overnight.sh.mako new file mode 100644 index 0000000..39ce809 --- /dev/null +++ b/rattail_fabric2/deploy/luigi/restart-overnight.sh.mako @@ -0,0 +1,5 @@ +#!/bin/sh -e + +DATE=`date --date='yesterday' +%Y-%m-%d` + +echo "${envroot}/bin/rattail -c ${appdir}/cron.conf --no-versioning run-n-mail -S 'Overnight catch-up: ${automation}' '${appdir}/overnight-${automation.lower()}.sh $DATE'" | at 'now + 1 minute' diff --git a/rattail_fabric2/deploy/luigi/rotate-logs.sh.mako b/rattail_fabric2/deploy/luigi/rotate-logs.sh.mako new file mode 100755 index 0000000..74443a8 --- /dev/null +++ b/rattail_fabric2/deploy/luigi/rotate-logs.sh.mako @@ -0,0 +1,14 @@ +#!/bin/sh -e +<%text>###################################################################### +# +# rotate Luigi server log file +# +<%text>###################################################################### + +if [ "$1" = "--verbose" ]; then + VERBOSE='--verbose' +else + VERBOSE= +fi + +/usr/sbin/logrotate $VERBOSE ${appdir}/luigi/logrotate.conf diff --git a/rattail_fabric2/deploy/luigi/supervisor.conf.mako b/rattail_fabric2/deploy/luigi/supervisor.conf.mako new file mode 100644 index 0000000..b7d2d68 --- /dev/null +++ b/rattail_fabric2/deploy/luigi/supervisor.conf.mako @@ -0,0 +1,11 @@ +## -*- mode: conf; -*- + +[group:luigi] +programs=luigid + +[program:luigid] +command=${envroot}/bin/luigid --logdir ${appdir}/luigi/log --state-path ${appdir}/luigi/state.pickle --address 127.0.0.1 +directory=${appdir}/work +environment=LUIGI_CONFIG_PATH="${appdir}/luigi/luigi.cfg" +user=${user} +autostart=${'true' if autostart else 'false'} diff --git a/rattail_fabric2/luigi.py b/rattail_fabric2/luigi.py new file mode 100644 index 0000000..240bdd1 --- /dev/null +++ b/rattail_fabric2/luigi.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2022 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Fabric library for Luigi apps +""" + +import os + +from rattail_fabric2 import make_deploy, mkdir + + +deploy_common = make_deploy(__file__) + + +def install_luigi(c, envroot, user='rattail', db_connection=None): + """ + Install and configure Luigi to the given virtualenv. + """ + envroot = envroot.rstrip('/') + envname = os.path.basename(envroot) + appdir = '{}/app'.format(envroot) + + # package + c.sudo("bash -lc 'workon {} && pip install luigi'".format(envname), + user=user) + + # dirs + mkdir(c, ['{}/luigi'.format(appdir), + '{}/luigi/log'.format(appdir), + '{}/luigitasks'.format(appdir), + ], use_sudo=True, owner=user) + + # tasks + c.sudo('touch {}/luigitasks/__init__.py'.format(appdir), + user=user) + + # config + deploy_common(c, 'luigi/luigi.cfg.mako', '{}/luigi/luigi.cfg'.format(appdir), + use_sudo=True, owner=user, mode='0640', + context={'appdir': appdir, + 'db_connection': db_connection}) + deploy_common(c, 'luigi/logging.conf.mako', '{}/luigi/logging.conf'.format(appdir), + use_sudo=True, owner=user, + context={'appdir': appdir}) + + # logrotate + deploy_common(c, 'luigi/luigi-logrotate.conf.mako', '{}/luigi/logrotate.conf'.format(appdir), + use_sudo=True, owner='root:', # must be owned by root (TODO: why is that again?) + context={'appdir': appdir}) + deploy_common(c, 'luigi/rotate-logs.sh.mako', '{}/rotate-luigi-logs.sh'.format(appdir), + use_sudo=True, owner=user, + context={'appdir': appdir}) + + +def install_overnight_script(c, envroot, user='rattail', automation='All'): + """ + Install an overnight automation script + """ + envroot = envroot.rstrip('/') + appdir = '{}/app'.format(envroot) + + # overnight-*.sh + filename = 'overnight-{}.sh'.format(automation.lower()) + deploy_common(c, 'luigi/overnight.sh.mako', + '{}/{}'.format(appdir, filename), + use_sudo=True, owner=user, mode='0755', + context={'envroot': envroot, 'appdir': appdir, + 'automation': automation}) + + # cron-overnight-*.sh + filename = 'cron-overnight-{}.sh'.format(automation.lower()) + deploy_common(c, 'luigi/cron-overnight.sh.mako', + '{}/{}'.format(appdir, filename), + use_sudo=True, owner=user, mode='0755', + context={'envroot': envroot, + 'automation': automation}) + + # restart-overnight-*.sh + filename = 'restart-overnight-{}.sh'.format(automation.lower()) + deploy_common(c, 'luigi/restart-overnight.sh.mako', + '{}/{}'.format(appdir, filename), + use_sudo=True, owner=user, mode='0755', + context={'envroot': envroot, + 'appdir': appdir, + 'automation': automation}) + + +def register_with_supervisor(c, envroot, user='rattail', autostart=False): + """ + Register the Luigi scheduler daemon with supervisor + """ + envroot = envroot.rstrip('/') + appdir = '{}/app'.format(envroot) + + deploy_common(c, 'luigi/supervisor.conf.mako', + '/etc/supervisor/conf.d/luigi.conf', + use_sudo=True, + context={'envroot': envroot, + 'appdir': appdir, + 'user': user, + 'autostart': autostart}) + c.sudo('supervisorctl update') + if autostart: + c.sudo('supervisorctl start luigi:')