Initial commit, as generated by Rattail Demo

machines/server/README.md

@@ -0,0 +1,41 @@
+
+# server
+
+This is for a generic "server" which runs Corporal.
+
+
+## Updating Live Server
+
+Note that this assumes you have defined `server` within your SSH config,
+e.g. at `~/.ssh/config`.  Also you should configure the root password within
+`./fabric.yaml`.
+
+Install everything with:
+
+    fab2 -e -H server bootstrap-all
+
+
+## Testing with Vagrant
+
+You should be able to get a VM going with a simple:
+
+    vagrant up
+
+You can then SSH directly into the VM with:
+
+    vagrant ssh
+
+You can confirm SSH credentials needed for connecting to it, with:
+
+    vagrant ssh-config
+
+Now you can "bootstrap" the machine with Fabric.  Please double-check your
+`fabenv.py` file and make sure it contains:
+
+    env.machine_is_live = False
+
+After all this machine is *not* live, it's just for testing.  Finally, here is
+the bootstrap command.  Note that it's possible you may need to modify some
+parameters based on the output of `vagrant ssh-config` above.
+
+    fab2 -e -H vagrant@localhost:2222 -i .vagrant/machines/default/virtualbox/private_key bootstrap-all

machines/server/Vagrantfile

@@ -0,0 +1,24 @@
+# -*- mode: ruby; -*-
+
+Vagrant.configure("2") do |config|
+
+  # live machine runs Debian 10 Buster
+  config.vm.box = "debian/buster64"
+
+  # # live machine runs Ubuntu 20.04 Focal Fossa
+  # config.vm.box = "ubuntu/focal64"
+
+  # this may be necessary for big data import tasks.  you can raise or lower
+  # it, or comment out if you find that you don't need it
+  config.vm.provider "virtualbox" do |v|
+    v.memory = 4096
+  end
+
+  # Corporal web app
+  config.vm.network "forwarded_port", guest: 9761, host: 9761
+
+  # apache
+  config.vm.network "forwarded_port", guest: 80, host: 8080
+  config.vm.network "forwarded_port", guest: 443, host: 8443
+
+end

machines/server/deploy/corporal/cron.conf

@@ -0,0 +1,34 @@
+
+############################################################
+#
+# cron config for Corporal
+#
+############################################################
+
+
+##############################
+# rattail
+##############################
+
+[rattail.config]
+include = %(here)s/rattail.conf
+
+
+##############################
+# alembic
+##############################
+
+[alembic]
+script_location = rattail.db:alembic
+version_locations = rattail.db:alembic/versions
+
+
+##############################
+# logging
+##############################
+
+[handler_console]
+level = WARNING
+
+[handler_file]
+args = ('/srv/envs/corporal/app/log/cron.log', 'a', 'utf_8')

machines/server/deploy/corporal/crontab.mako

@@ -0,0 +1,11 @@
+
+${'#'}###########################################################
+#
+# crontab for Corporal
+#
+${'#'}###########################################################
+
+MAILTO="root@localhost,fred@mailinator.com"
+
+# overnight automation starts at 1:00am
+${'' if env.machine_is_live else '#'}00 01 * * *  rattail  /srv/envs/corporal/app/overnight-wrapper.sh

machines/server/deploy/corporal/logrotate.conf

@@ -0,0 +1,20 @@
+
+/srv/envs/corporal/pip.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 640 rattail rattail
+}
+
+/srv/envs/corporal/app/log/*.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 640 rattail rattail
+}

machines/server/deploy/corporal/overnight-wrapper.sh

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+############################################################
+#
+# wrapper script for Corporal overnight automation
+#
+############################################################
+
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    PROGRESS='--progress'
+else
+    VERBOSE=
+    PROGRESS=
+fi
+
+cd /srv/envs/corporal
+
+RATTAIL="bin/rattail --config=app/cron.conf $PROGRESS"
+
+$RATTAIL run-n-mail --no-versioning --skip-if-empty --subject 'Overnight automation' /srv/envs/corporal/app/overnight.sh

machines/server/deploy/corporal/overnight.sh

@@ -0,0 +1,36 @@
+#!/bin/sh -e
+############################################################
+#
+# overnight automation for Corporal
+#
+############################################################
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    PROGRESS='--progress'
+else
+    VERBOSE=
+    PROGRESS=
+fi
+
+cd /srv/envs/corporal
+
+RATTAIL="bin/rattail --config=app/cron.conf $PROGRESS"
+
+
+##############################
+# data sync
+##############################
+
+# import latest data from Catapult
+$RATTAIL --runas catapult import-catapult --delete --warnings
+
+# make sure version data is correct
+$RATTAIL import-versions --delete --dry-run --warnings
+
+
+##############################
+# problem reports
+##############################
+
+$RATTAIL problems

machines/server/deploy/corporal/rattail.conf.mako

@@ -0,0 +1,148 @@
+## -*- mode: conf; -*-
+
+${'#'}###########################################################
+#
+# base config for Corporal
+#
+${'#'}###########################################################
+
+
+
+
+${'#'}#############################
+# rattail
+${'#'}#############################
+
+[rattail]
+production = ${'true' if env.machine_is_live else 'false'}
+
+# TODO: this will of course depend on your location
+timezone.default = America/Chicago
+
+# TODO: set this to a valid user within your DB
+#runas.default = corporal
+
+appdir = /srv/envs/corporal/app
+datadir = /srv/envs/corporal/app/data
+workdir = /srv/envs/corporal/app/work
+batch.files = /srv/envs/corporal/app/batch
+export.files = /srv/envs/corporal/app/data/exports
+
+[rattail.config]
+#include = /etc/rattail/rattail.conf
+configure_logging = true
+usedb = true
+preferdb = true
+
+[rattail.db]
+default.url = postgresql://rattail:${env.password_postgresql_rattail}@localhost/corporal
+versioning.enabled = true
+
+[rattail.mail]
+# this is the master switch, *no* emails are sent if false
+send_emails = true
+
+smtp.server = localhost
+
+templates =
+    corporal:templates/mail
+    rattail:templates/mail
+
+default.prefix = [Corporal]
+default.from = ${env.email_default_sender}
+default.to = ${', '.join(env.email_default_recipients)}
+#default.enabled = false
+
+[rattail.upgrades]
+command = sudo /srv/envs/corporal/app/upgrade-wrapper.sh --verbose
+files = /srv/envs/corporal/app/data/upgrades
+
+
+${'#'}#############################
+# alembic
+${'#'}#############################
+
+[alembic]
+script_location = rattail.db:alembic
+version_locations = rattail.db:alembic/versions
+
+
+${'#'}#############################
+# logging
+${'#'}#############################
+
+[loggers]
+keys = root, exc_logger, beaker, txn, sqlalchemy, django_db, flufl_bounce, requests
+
+[handlers]
+keys = file, console, email
+
+[formatters]
+keys = generic, console
+
+[logger_root]
+handlers = file, console, email
+level = DEBUG
+
+[logger_exc_logger]
+qualname = exc_logger
+handlers = email
+level = ERROR
+
+[logger_beaker]
+qualname = beaker
+handlers =
+level = INFO
+
+[logger_txn]
+qualname = txn
+handlers =
+level = INFO
+
+[logger_sqlalchemy]
+qualname = sqlalchemy.engine
+handlers =
+# handlers = file
+# level = INFO
+
+[logger_django_db]
+qualname = django.db.backends
+handlers =
+level = INFO
+# level = DEBUG
+
+[logger_flufl_bounce]
+qualname = flufl.bounce
+handlers =
+level = WARNING
+
+[logger_requests]
+qualname = requests
+handlers =
+# level = WARNING
+
+[handler_file]
+class = handlers.WatchedFileHandler
+args = ('/srv/envs/corporal/app/log/rattail.log', 'a', 'utf_8')
+formatter = generic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = console
+# formatter = generic
+# level = INFO
+# level = WARNING
+
+[handler_email]
+class = handlers.SMTPHandler
+args = ('localhost', '${env.email_default_sender}', ${env.email_default_recipients}, "[Corporal] Logging")
+formatter = generic
+level = ERROR
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(funcName)s: %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_console]
+format = %(levelname)-5.5s [%(name)s] %(funcName)s: %(message)s

machines/server/deploy/corporal/sudoers

@@ -0,0 +1,5 @@
+# -*- mode: conf; -*-
+
+# let rattail upgrade the app
+rattail ALL = NOPASSWD: /srv/envs/corporal/app/upgrade-wrapper.sh
+rattail ALL = NOPASSWD: /srv/envs/corporal/app/upgrade-wrapper.sh --verbose

machines/server/deploy/corporal/supervisor.conf

@@ -0,0 +1,8 @@
+
+[group:corporal]
+programs=corporal_webmain
+
+[program:corporal_webmain]
+command=/srv/envs/corporal/bin/pserve pastedeploy+ini:/srv/envs/corporal/app/web.conf
+directory=/srv/envs/corporal/app/work
+user=rattail

machines/server/deploy/corporal/tasks.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8; -*-
+
+from invoke import task
+
+
+@task
+def upgrade(ctx):
+    ctx.run('/srv/envs/corporal/app/upgrade.sh --verbose')

machines/server/deploy/corporal/upgrade-wrapper.sh

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    INVOKE_ARGS='--echo'
+else
+    VERBOSE=
+    INVOKE_ARGS=
+fi
+
+cd /srv/envs/corporal
+
+INVOKE="sudo -H -u rattail bin/invoke --collection=app/tasks $INVOKE_ARGS"
+
+# run upgrade task, as rattail user
+$INVOKE upgrade
+
+# restart web app
+sh -c 'sleep 10; supervisorctl restart corporal:corporal_webmain' &

machines/server/deploy/corporal/upgrade.sh

@@ -0,0 +1,38 @@
+#!/bin/sh -e
+
+# NOTE: this script is meant be ran by the 'rattail' user!
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    QUIET=
+else
+    VERBOSE=
+    QUIET='--quiet'
+fi
+
+SRC=/srv/envs/corporal/src
+PIP=/srv/envs/corporal/bin/pip
+export PIP_CONFIG_FILE=/srv/envs/corporal/pip.conf
+
+# upgrade pip
+$PIP install $QUIET --disable-pip-version-check --upgrade pip
+# $PIP install $QUIET --upgrade --upgrade-strategy eager setuptools wheel ndg-httpsclient
+
+# upgrade app software...
+
+# how you need to upgrade your app will depend on whether you are running any
+# packages "from source" as opposed to only using built/released packages
+
+# if running Corporal from source, you should first fetch/install latest code:
+#cd $SRC/corporal
+#git pull $QUIET
+#find . -name '*.pyc' -delete
+#$PIP install $QUIET --editable .
+
+# in any case the last step is always the same.  note that this will ensure the
+# "latest" Corporal is used, but also will upgrade any dependencies
+$PIP install $QUIET --upgrade --upgrade-strategy eager 'Corporal'
+
+# migrate database schema
+cd /srv/envs/corporal
+bin/alembic --config app/rattail.conf upgrade heads

machines/server/deploy/corporal/web.conf.mako

@@ -0,0 +1,72 @@
+## -*- mode: conf; -*-
+
+${'#'}###########################################################
+#
+# config for Corporal web app
+#
+${'#'}###########################################################
+
+
+${'#'}#############################
+# rattail
+${'#'}#############################
+
+[rattail.config]
+include = %(here)s/rattail.conf
+
+
+${'#'}#############################
+# pyramid
+${'#'}#############################
+
+[app:main]
+use = egg:Corporal
+
+pyramid.reload_templates = false
+pyramid.debug_all = false
+pyramid.default_locale_name = en
+pyramid.includes = pyramid_exclog
+
+beaker.session.type = file
+beaker.session.data_dir = %(here)s/sessions/data
+beaker.session.lock_dir = %(here)s/sessions/lock
+beaker.session.secret = ${env.tailbone_beaker_secret}
+beaker.session.key = corporal
+
+pyramid_deform.tempdir = %(here)s/data/uploads
+
+exclog.extra_info = true
+
+# required for tailbone
+rattail.config = %(__file__)s
+
+[server:main]
+use = egg:waitress#main
+# TODO: should ideally use 127.0.0.1 as host address, e.g. with reverse proxy
+host = 0.0.0.0
+port = 9761
+
+# NOTE: this is needed for local reverse proxy stuff to work with HTTPS
+# https://docs.pylonsproject.org/projects/waitress/en/latest/reverse-proxy.html
+# https://docs.pylonsproject.org/projects/waitress/en/latest/arguments.html
+# trusted_proxy = 127.0.0.1
+
+# TODO: leave this empty if proxy serves as root site, e.g. http://rattail.example.com/
+# url_prefix =
+
+# TODO: or, if proxy serves as subpath of root site, e.g. http://rattail.example.com/backend/
+# url_prefix = /backend
+
+
+${'#'}#############################
+# logging
+${'#'}#############################
+
+[logger_root]
+handlers = file, console
+
+[handler_console]
+level = INFO
+
+[handler_file]
+args = ('/srv/envs/corporal/app/log/web.log', 'a', 'utf_8')

machines/server/fabenv.py.dist

@@ -0,0 +1,44 @@
+# -*- coding: utf-8; mode: python; -*-
+"""
+Fabric environment tweaks
+"""
+
+from fabfile import env
+
+
+##############################
+# volatile
+##############################
+
+# this should be True only when targeting the truly *live* machine, but False
+# otherwise, e.g. when building a new "live" machine, or using Vagrant
+env.machine_is_live = False
+
+
+##############################
+# stable
+##############################
+
+# default sender and recipients for all emails
+env.email_default_sender = 'rattail@example.com'
+env.email_default_recipients = ['root@example.com']
+
+# this is for the 'rattail' user within PostgreSQL, running on the server.
+# this rattail user owns the 'corporal' database and is used by the
+# Corporal app to access the db
+env.password_postgresql_rattail = 'password'
+
+# this is the hostname for your Catapult WebOffice
+env.catapult_host = 'INSTANCE.catapultweboffice.com'
+
+# these credentials are used to access the ODBC DSN for ECRS Catapult
+env.catapult_odbc_username = 'username'
+env.catapult_odbc_password = 'password'
+
+# this is used for protecting user session data for the web app, which lives in
+# the server's file system.  should probably be at least 20 random characters
+env.tailbone_beaker_secret = 'password'
+
+# these credentials are used to access the "restricted" Rattail Project PyPI
+env.restricted_pypi_username = 'username'
+env.restricted_pypi_password = 'password'

machines/server/fabfile.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8; -*-
+"""
+Fabric script for 'server' machine
+
+Please see the accompanying README for full instructions.
+"""
+
+from fabric2 import task
+
+from rattail.core import Object
+from rattail_fabric2 import apt, postfix, postgresql, python, exists, make_system_user, mkdir
+
+from corporal.fablib import make_deploy
+from corporal.fablib.python import bootstrap_python
+
+
+env = Object()
+deploy = make_deploy(__file__)
+
+
+##############################
+# bootstrap
+##############################
+
+@task
+def bootstrap_all(c):
+    """
+    Bootstrap all aspects of the machine
+    """
+    bootstrap_base(c)
+    bootstrap_corporal(c)
+
+
+@task
+def bootstrap_base(c):
+    """
+    Bootstrap the base system
+    """
+    apt.dist_upgrade(c)
+
+    # postfix
+    postfix.install(c)
+
+    # rattail user
+    make_system_user(c, 'rattail', home='/var/lib/rattail', shell='/bin/bash')
+    postfix.alias(c, 'rattail', 'root')
+
+    # python
+    bootstrap_python(c, user='rattail', env=env,
+                     virtualenvwrapper_from_apt=True,
+                     python3=True)
+
+    # postgres
+    apt.install(c, 'postgresql')
+    postgresql.create_user(c, 'rattail', password=env.password_postgresql_rattail)
+
+
+    # misc.
+    apt.install(
+        c,
+        'git',
+        'libpq-dev',
+        'supervisor',
+    )
+
+    # uncomment this if you prefer to use emacs
+    #apt.install_emacs(c)
+
+
+@task
+def bootstrap_corporal(c):
+    """
+    Bootstrap the Corporal app
+    """
+    user = 'rattail'
+
+    c.sudo('supervisorctl stop corporal:', warn=True)
+
+    # virtualenv
+    if not exists(c, '/srv/envs/corporal'):
+        python.mkvirtualenv(c, 'corporal', python='/usr/bin/python3', runas_user=user)
+    c.sudo('chmod 0600 /srv/envs/corporal/pip.conf')
+    mkdir(c, '/srv/envs/corporal/src', owner=user, use_sudo=True)
+
+    # Corporal
+    if env.machine_is_live:
+        c.sudo("bash -lc 'workon corporal && pip install Corporal'",
+               user=user)
+    else:
+        # TODO: this really only works for vagrant
+        c.sudo("bash -lc 'workon corporal && pip install /vagrant/Corporal-*.tar.gz'",
+               user=user)
+
+    # app dir
+    if not exists(c, '/srv/envs/corporal/app'):
+        c.sudo("bash -lc 'workon corporal && cdvirtualenv && rattail make-appdir'", user=user)
+    c.sudo('chmod 0750 /srv/envs/corporal/app/log')
+    mkdir(c, '/srv/envs/corporal/app/data', use_sudo=True, owner=user)
+
+    # config / scripts
+    deploy(c, 'corporal/rattail.conf.mako', '/srv/envs/corporal/app/rattail.conf',
+           owner=user, mode='0600', use_sudo=True, context={'env': env})
+    if not exists(c, '/srv/envs/corporal/app/quiet.conf'):
+        c.sudo("bash -lc 'workon corporal && cdvirtualenv app && rattail make-config -T quiet'",
+               user=user)
+    deploy(c, 'corporal/cron.conf', '/srv/envs/corporal/app/cron.conf',
+           owner=user, use_sudo=True)
+    deploy(c, 'corporal/web.conf.mako', '/srv/envs/corporal/app/web.conf',
+           owner=user, mode='0600', use_sudo=True, context={'env': env})
+    deploy(c, 'corporal/upgrade.sh', '/srv/envs/corporal/app/upgrade.sh',
+           owner=user, mode='0755', use_sudo=True)
+    deploy(c, 'corporal/tasks.py', '/srv/envs/corporal/app/tasks.py',
+           owner=user, use_sudo=True)
+    deploy(c, 'corporal/upgrade-wrapper.sh', '/srv/envs/corporal/app/upgrade-wrapper.sh',
+           owner=user, mode='0755', use_sudo=True)
+    deploy(c, 'corporal/overnight.sh', '/srv/envs/corporal/app/overnight.sh',
+           owner=user, mode='0755', use_sudo=True)
+    deploy(c, 'corporal/overnight-wrapper.sh', '/srv/envs/corporal/app/overnight-wrapper.sh',
+           owner=user, mode='0755', use_sudo=True)
+
+    # database
+    if not postgresql.db_exists(c, 'corporal'):
+        postgresql.create_db(c, 'corporal', owner='rattail', checkfirst=False)
+        c.sudo("bash -lc 'workon corporal && cdvirtualenv && bin/alembic --config app/rattail.conf upgrade heads'",
+               user=user)
+        postgresql.sql(c, "insert into setting values ('rattail.app_title', 'Corporal')",
+                       database='corporal')
+        postgresql.sql(c, "insert into setting values ('tailbone.theme', 'falafel')",
+                       database='corporal')
+        postgresql.sql(c, "insert into setting values ('tailbone.themes.expose_picker', 'false')",
+                       database='corporal')
+
+    # supervisor
+    deploy(c, 'corporal/supervisor.conf', '/etc/supervisor/conf.d/corporal.conf',
+           use_sudo=True)
+    c.sudo('supervisorctl update')
+    c.sudo('supervisorctl start corporal:')
+
+    # cron etc.
+    deploy.sudoers(c, 'corporal/sudoers', '/etc/sudoers.d/corporal')
+    deploy(c, 'corporal/crontab.mako', '/etc/cron.d/corporal',
+           use_sudo=True, context={'env': env})
+    deploy(c, 'corporal/logrotate.conf', '/etc/logrotate.d/corporal',
+           use_sudo=True)
+
+
+##############################
+# fabenv
+##############################
+
+try:
+    import fabenv
+except ImportError as error:
+    print("\ncouldn't import fabenv: {}".format(error))
+
+env.setdefault('machine_is_live', False)

machines/server/fabric.yaml.dist

@@ -0,0 +1,3 @@
+# -*- mode: yaml; -*-
+sudo:
+  password: "XXXXXXXX"