Add some default fabric install logic for Corporal apps

.gitignore

@@ -1 +1,2 @@
 Corporal.egg-info/
+machines/*/.vagrant/

corporal/fablib/corporal.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8; -*-
+"""
+Fabric library for Corporal systems
+"""
+
+from rattail_fabric2 import postgresql, python, exists, mkdir
+
+from corporal.fablib import deploy_common
+
+
+def bootstrap_corporal_app(c, env, envname='corporal', user='rattail',
+                           port=7900, sitename=None, stage=False):
+    """
+    Create a virtual environment for use with a Corporal app.
+    """
+    safename = envname.replace('-', '_')
+    dbname = envname
+    if not sitename:
+        sitename = envname
+    envdir = '/srv/envs/{}'.format(envname)
+    appdir = '{}/app'.format(envdir)
+    srcdir = '{}/src'.format(envdir)
+    production = False if stage else env.machine_is_live
+
+    c.sudo('supervisorctl stop {}:'.format(safename), warn=True)
+
+    # virtualenv
+    if not exists(c, envdir):
+        python.mkvirtualenv(c, envname, python='/usr/bin/python3', runas_user=user)
+    c.sudo('chmod 0600 {}/pip.conf'.format(envdir))
+    mkdir(c, srcdir, owner=user, use_sudo=True)
+
+    if stage:
+
+        # pycorepos
+        if not exists(c, '{}/pycorepos'.format(srcdir)):
+            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/pycorepos {}/pycorepos'.format(srcdir),
+                   user=user)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/pycorepos'".format(envname, srcdir),
+                   user=user)
+
+        # rattail
+        if not exists(c, '{}/rattail'.format(srcdir)):
+            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/rattail {}/rattail'.format(srcdir),
+                   user=user)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/rattail'".format(envname, srcdir),
+                   user=user)
+
+        # rattail-corepos
+        if not exists(c, '{}/rattail-corepos'.format(srcdir)):
+            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/rattail-corepos {}/rattail-corepos'.format(srcdir),
+                   user=user, echo=False)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/rattail-corepos'".format(envname, srcdir),
+                   user=user)
+
+        # tailbone
+        if not exists(c, '{}/tailbone'.format(srcdir)):
+            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/tailbone {}/tailbone'.format(srcdir),
+                   user=user)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/tailbone'".format(envname, srcdir),
+                   user=user)
+
+        # tailbone-corepos
+        if not exists(c, '{}/tailbone-corepos'.format(srcdir)):
+            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/tailbone-corepos {}/tailbone-corepos'.format(srcdir),
+                   user=user, echo=False)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/tailbone-corepos'".format(envname, srcdir),
+                   user=user)
+
+        # rattail-fabric2
+        if not exists(c, '{}/rattail-fabric2'.format(srcdir)):
+            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/rattail-fabric2 {}/rattail-fabric2'.format(srcdir),
+                   user=user)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/rattail-fabric2'".format(envname, srcdir),
+                   user=user)
+
+        # corporal
+        if not exists(c, '{}/corporal'.format(srcdir)):
+            c.sudo('git clone https://gitlab.com/ttsc/kardish/corporal.git {}/corporal'.format(srcdir),
+                   user=user)
+            c.sudo("bash -lc 'workon {} && pip install -e {}/corporal'".format(envname, srcdir),
+                   user=user)
+
+    else:
+        c.sudo("bash -lc 'workon {} && pip install Corporal'".format(envname),
+               user=user)
+
+    # app dir
+    if not exists(c, appdir):
+        c.sudo("bash -lc 'workon {} && cdvirtualenv && rattail make-appdir'".format(envname), 
+               user=user)
+    c.sudo('chmod 0750 {}/log'.format(appdir))
+    mkdir(c, '{}/data'.format(appdir), use_sudo=True, owner=user)
+
+    # config / scripts
+    deploy_common(c, 'corporal/rattail.conf.mako', '{}/rattail.conf'.format(appdir),
+                  use_sudo=True, owner=user, mode='0600', 
+                  context={'env': env, 'envdir': envdir, 'dbname': dbname,
+                           'production': production})
+    if not exists(c, '{}/quiet.conf'.format(appdir)):
+        c.sudo("bash -lc 'workon {} && cdvirtualenv app && rattail make-config -T quiet'".format(envname),
+               user=user)
+    deploy_common(c, 'corporal/cron.conf.mako', '{}/cron.conf'.format(appdir),
+                  use_sudo=True, owner=user, context={'envdir': envdir})
+    deploy_common(c, 'corporal/web.conf.mako', '{}/web.conf'.format(appdir),
+                  use_sudo=True, owner=user, mode='0600', 
+                  context={'env': env, 'envname': envname, 'envdir': envdir,
+                           'port': port})
+    deploy_common(c, 'corporal/upgrade.sh.mako', '{}/upgrade.sh'.format(appdir),
+                  use_sudo=True, owner=user, 
+                  context={'envdir': envdir, 'production': production})
+    # if host:
+    deploy_common(c, 'corporal/tasks.py.mako', '{}/tasks.py'.format(appdir),
+                  use_sudo=True, owner=user,
+                  context={'envdir': envdir, 'stage': stage})
+    deploy_common(c, 'corporal/upgrade-wrapper.sh.mako', '{}/upgrade-wrapper.sh'.format(appdir),
+                  use_sudo=True, owner=user, 
+                  context={'envdir': envdir, 'safename': safename})
+
+    # database
+    if not postgresql.db_exists(c, dbname):
+        postgresql.create_db(c, dbname, owner='rattail', checkfirst=False)
+        c.sudo("bash -lc 'workon {} && cdvirtualenv && bin/alembic -c app/rattail.conf upgrade heads'".format(envname),
+               user=user)
+        postgresql.sql(c, "insert into setting values ('tailbone.theme', 'falafel')",
+                       database=dbname)
+        postgresql.sql(c, "insert into setting values ('tailbone.themes.expose_picker', 'false')",
+                       database=dbname)
+
+    # supervisor
+    deploy_common(c, 'corporal/supervisor.conf.mako', '/etc/supervisor/conf.d/{}.conf'.format(safename),
+                  use_sudo=True, context={'envdir': envdir, 'safename': safename})
+    c.sudo('supervisorctl update')
+    c.sudo('supervisorctl start {}:'.format(safename))
+
+    # cron etc.
+    deploy_common.sudoers(c, 'corporal/sudoers.mako', '/etc/sudoers.d/{}'.format(safename),
+                          context={'envdir': envdir, 'safename': safename,
+                                   'host': host, 'stage': stage})
+    deploy_common(c, 'corporal/logrotate.conf.mako', '/etc/logrotate.d/{}'.format(safename),
+                  use_sudo=True, context={'envdir': envdir})

corporal/fablib/deploy/corporal/cron.conf.mako

@@ -0,0 +1,35 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# cron config for Corporal
+#
+<%text>############################################################</%text>
+
+
+<%text>##############################</%text>
+# rattail
+<%text>##############################</%text>
+
+[rattail.config]
+include = %(here)s/rattail.conf
+
+
+<%text>##############################</%text>
+# alembic
+<%text>##############################</%text>
+
+[alembic]
+script_location = corporal.db:alembic
+version_locations = corporal.db:alembic/versions rattail.db:alembic/versions
+
+
+<%text>##############################</%text>
+# logging
+<%text>##############################</%text>
+
+[handler_console]
+level = WARNING
+
+[handler_file]
+args = ('${envdir}/app/log/cron.log', 'a', 'utf_8')

corporal/fablib/deploy/corporal/logrotate.conf.mako

@@ -0,0 +1,21 @@
+## -*- mode: conf; -*-
+
+${envdir}/pip.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 640 rattail rattail
+}
+
+${envdir}/app/log/*.log {
+    daily
+    missingok
+    rotate 30
+    compress
+    delaycompress
+    notifempty
+    create 640 rattail rattail
+}

corporal/fablib/deploy/corporal/rattail.conf.mako

@@ -0,0 +1,72 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# base config for Corporal
+#
+<%text>############################################################</%text>
+
+
+[corepos.db.office_op]
+default.url = mysql+mysqlconnector://${env.username_mysql_coreserver}:${env.password_mysql_coreserver}@localhost/core_op
+default.pool_recycle = 3600
+
+
+<%text>##############################</%text>
+# rattail
+<%text>##############################</%text>
+
+[rattail]
+production = ${'true' if production else 'false'}
+app_title = Corporal
+node_type = ${node_type}
+appdir = ${envdir}/app
+datadir = ${envdir}/app/data
+workdir = ${envdir}/app/work
+batch.files = ${envdir}/app/batch
+export.files = ${envdir}/app/data/exports
+runas.default = corporal
+
+[rattail.config]
+include = /etc/rattail/rattail.conf
+usedb = true
+preferdb = true
+
+[rattail.db]
+default.url = postgresql://rattail:${env.password_postgresql_rattail}@localhost/${dbname}
+versioning.enabled = true
+
+[rattail.mail]
+# this is the master switch, *no* emails are sent if false
+send_emails = true
+
+templates =
+    corporal:templates/mail
+    rattail:templates/mail
+
+default.prefix = [Corporal]
+#default.enabled = false
+
+[rattail.upgrades]
+command = sudo ${envdir}/app/upgrade-wrapper.sh --verbose
+files = ${envdir}/app/data/upgrades
+
+
+<%text>##############################</%text>
+# alembic
+<%text>##############################</%text>
+
+[alembic]
+script_location = corporal.db:alembic
+version_locations = corporal.db:alembic/versions rattail.db:alembic/versions
+
+
+<%text>##############################</%text>
+# logging
+<%text>##############################</%text>
+
+[handler_file]
+args = ('${envdir}/app/log/rattail.log', 'a', 'utf_8')
+
+[handler_email]
+args = ('localhost', '${env.email_default_sender}', ${env.email_default_recipients}, "[Corporal] Logging")

corporal/fablib/deploy/corporal/sudoers.mako

@@ -0,0 +1,9 @@
+# -*- mode: conf; -*-
+
+# let rattail upgrade the app
+rattail ALL = NOPASSWD: ${envdir}/app/upgrade-wrapper.sh
+rattail ALL = NOPASSWD: ${envdir}/app/upgrade-wrapper.sh --verbose
+
+# # let rattail manage supervisor daemons
+# rattail ALL = NOPASSWD: /usr/bin/supervisorctl stop ${safename}\:
+# rattail ALL = NOPASSWD: /usr/bin/supervisorctl start ${safename}\:

corporal/fablib/deploy/corporal/supervisor.conf.mako

@@ -0,0 +1,9 @@
+## -*- mode: conf; -*-
+
+[group:${safename}]
+programs=${safename}_webmain
+
+[program:${safename}_webmain]
+command=${envdir}/bin/pserve pastedeploy+ini:${envdir}/app/web.conf
+directory=${envdir}/app/work
+user=rattail

corporal/fablib/deploy/corporal/tasks.py.mako

@@ -0,0 +1,9 @@
+## -*- mode: python; -*-
+# -*- coding: utf-8; -*-
+
+from invoke import task
+
+
+@task
+def upgrade(c):
+    c.run('${envdir}/app/upgrade.sh --verbose')

corporal/fablib/deploy/corporal/upgrade-wrapper.sh.mako

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    INVOKE_ARGS='--echo'
+else
+    VERBOSE=
+    INVOKE_ARGS=
+fi
+
+cd ${envdir}
+
+INVOKE="sudo -H -u rattail bin/invoke --collection=app/tasks $INVOKE_ARGS"
+
+# run upgrade task, as rattail user
+$INVOKE upgrade
+
+# restart web app
+sh -c 'sleep 10; supervisorctl restart ${safename}:${safename}_webmain' &

corporal/fablib/deploy/corporal/upgrade.sh.mako

@@ -0,0 +1,66 @@
+#!/bin/sh -e
+
+# NOTE: this script is meant be ran by the 'rattail' user!
+
+if [ "$1" = "--verbose" ]; then
+    VERBOSE='--verbose'
+    QUIET=
+else
+    VERBOSE=
+    QUIET='--quiet'
+fi
+
+SRC=${envdir}/src
+PIP=${envdir}/bin/pip
+export PIP_CONFIG_FILE=${envdir}/pip.conf
+
+# upgrade pip and friends
+$PIP install $QUIET --disable-pip-version-check --upgrade pip
+$PIP install $QUIET --upgrade setuptools wheel
+
+% if not production:
+# update all source packages...
+
+cd $SRC/pycorepos
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+cd $SRC/rattail
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+cd $SRC/rattail-corepos
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+cd $SRC/tailbone
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+cd $SRC/tailbone-corepos
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+cd $SRC/rattail-fabric2
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+cd $SRC/corporal
+git pull $QUIET
+find . -name '*.pyc' -delete
+$PIP install $QUIET --editable .
+
+% endif
+
+# upgrade all dependencies for Corporal
+$PIP install $QUIET --upgrade --upgrade-strategy eager 'Corporal'
+
+# migrate database schema
+cd ${envdir}
+bin/alembic --config app/rattail.conf upgrade heads

corporal/fablib/deploy/corporal/web.conf.mako

@@ -0,0 +1,71 @@
+## -*- mode: conf; -*-
+
+<%text>############################################################</%text>
+#
+# config for Corporal web app
+#
+<%text>############################################################</%text>
+
+
+<%text>##############################</%text>
+# rattail
+<%text>##############################</%text>
+
+[rattail.config]
+include = %(here)s/rattail.conf
+
+
+<%text>##############################</%text>
+# pyramid
+<%text>##############################</%text>
+
+[app:main]
+use = egg:Corporal
+
+pyramid.reload_templates = false
+pyramid.debug_all = false
+pyramid.default_locale_name = en
+pyramid.includes = pyramid_exclog
+
+beaker.session.type = file
+beaker.session.data_dir = %(here)s/sessions/data
+beaker.session.lock_dir = %(here)s/sessions/lock
+beaker.session.secret = ${env.tailbone_beaker_secret}
+beaker.session.key = ${envname}
+
+pyramid_deform.tempdir = %(here)s/data/uploads
+
+exclog.extra_info = true
+
+# required for tailbone
+rattail.config = %(__file__)s
+
+[server:main]
+use = egg:waitress#main
+host = 127.0.0.1
+port = ${port}
+
+# NOTE: this is needed for local reverse proxy stuff to work with HTTPS
+# https://docs.pylonsproject.org/projects/waitress/en/latest/reverse-proxy.html
+# https://docs.pylonsproject.org/projects/waitress/en/latest/arguments.html
+# trusted_proxy = 127.0.0.1
+
+# TODO: leave this empty if proxy serves as root site, e.g. http://rattail.example.com/
+# url_prefix =
+
+# TODO: or, if proxy serves as subpath of root site, e.g. http://rattail.example.com/backend/
+# url_prefix = /backend
+
+
+<%text>##############################</%text>
+# logging
+<%text>##############################</%text>
+
+[logger_root]
+handlers = file, console
+
+[handler_console]
+level = INFO
+
+[handler_file]
+args = ('${envdir}/app/log/web.log', 'a', 'utf_8')