282 lines
11 KiB
Python
282 lines
11 KiB
Python
|
# -*- coding: utf-8; -*-
|
||
|
|
"""
|
||
|
|
Fabric script for a server running Theo, the order system
|
||
|
|
|
||
|
|
Please see the accompanying README for full instructions.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import datetime
|
||
|
|
|
||
|
|
from fabric2 import task
|
||
|
|
|
||
|
|
from rattail.core import Object
|
||
|
|
from rattail_fabric2 import make_deploy, apt, postfix, postgresql, exists, make_system_user, mkdir
|
||
|
|
|
||
|
|
|
||
|
|
env = Object()
|
||
|
|
deploy = make_deploy(__file__)
|
||
|
|
|
||
|
|
|
||
|
|
##############################
|
||
|
|
# bootstrap
|
||
|
|
##############################
|
||
|
|
|
||
|
|
@task
|
||
|
|
def bootstrap_all(c):
|
||
|
|
"""
|
||
|
|
Bootstrap all aspects of the machine
|
||
|
|
"""
|
||
|
|
bootstrap_base(c)
|
||
|
|
bootstrap_theo(c)
|
||
|
|
bootstrap_theo_stage(c)
|
||
|
|
|
||
|
|
|
||
|
|
@task
|
||
|
|
def bootstrap_base(c):
|
||
|
|
"""
|
||
|
|
Bootstrap the base system
|
||
|
|
"""
|
||
|
|
# we do `apt update && apt dist-upgrade` every time this task runs
|
||
|
|
apt.dist_upgrade(c)
|
||
|
|
|
||
|
|
# we assume postfix is installed, but the configuration thereof is not
|
||
|
|
# dealt with here; you may need to configure further on your own
|
||
|
|
postfix.install(c)
|
||
|
|
|
||
|
|
# rattail user + common config
|
||
|
|
make_system_user(c, 'rattail', home='/var/lib/rattail', shell='/bin/bash')
|
||
|
|
postfix.alias(c, 'rattail', 'root')
|
||
|
|
mkdir(c, '/etc/rattail', use_sudo=True)
|
||
|
|
deploy(c, 'rattail/rattail.conf.mako', '/etc/rattail/rattail.conf',
|
||
|
|
use_sudo=True, context={'env': env})
|
||
|
|
|
||
|
|
# postgresql
|
||
|
|
postgresql.install(c)
|
||
|
|
postgresql.create_user(c, 'rattail', password=env.password_postgresql_rattail)
|
||
|
|
|
||
|
|
# python
|
||
|
|
mkdir(c, '/srv/envs', use_sudo=True, owner='rattail:')
|
||
|
|
apt.install(
|
||
|
|
c,
|
||
|
|
'build-essential',
|
||
|
|
'git',
|
||
|
|
'libpq-dev',
|
||
|
|
'python3-dev',
|
||
|
|
'python3-venv',
|
||
|
|
'supervisor',
|
||
|
|
)
|
||
|
|
|
||
|
|
# catapult extras
|
||
|
|
if env.theo_integrates_with == 'catapult':
|
||
|
|
|
||
|
|
# freetds / odbc
|
||
|
|
apt.install(c, 'unixodbc', 'unixodbc-dev')
|
||
|
|
# we must install FreeTDS from source, b/c the version APT makes available
|
||
|
|
# is too old. however the *latest* source seems to have some issue(s)
|
||
|
|
# which cause it to use too much memory, so we use a more stable branch
|
||
|
|
from rattail_fabric2 import freetds
|
||
|
|
freetds.install_from_source(c, user='rattail', branch='Branch-1_2')
|
||
|
|
deploy(c, 'rattail/freetds.conf.mako', '/usr/local/etc/freetds.conf',
|
||
|
|
use_sudo=True, context={'env': env})
|
||
|
|
deploy(c, 'rattail/odbc.ini', '/etc/odbc.ini', use_sudo=True)
|
||
|
|
|
||
|
|
|
||
|
|
@task
|
||
|
|
def bootstrap_theo(c):
|
||
|
|
"""
|
||
|
|
Bootstrap Theo, the order system
|
||
|
|
"""
|
||
|
|
install_theo_app(c, 'theo', True, 7000,
|
||
|
|
# overnight runs at 1:00am
|
||
|
|
run_overnight_at=datetime.time(1))
|
||
|
|
|
||
|
|
|
||
|
|
@task
|
||
|
|
def bootstrap_theo_stage(c):
|
||
|
|
"""
|
||
|
|
Bootstrap "stage" for Theo, the order system
|
||
|
|
"""
|
||
|
|
install_theo_app(c, 'theo-stage', False, 7010, from_source=True,
|
||
|
|
# overnight runs at 12:30am
|
||
|
|
run_overnight_at=datetime.time(0, 30))
|
||
|
|
|
||
|
|
|
||
|
|
##############################
|
||
|
|
# support functions
|
||
|
|
##############################
|
||
|
|
|
||
|
|
def install_theo_app(c, envname, production, port, from_source=False,
|
||
|
|
run_overnight_at=None):
|
||
|
|
"""
|
||
|
|
Install a Theo app, per the given parameters
|
||
|
|
"""
|
||
|
|
dbname = envname
|
||
|
|
safename = envname.replace('-', '_')
|
||
|
|
envroot = '/srv/envs/{}'.format(envname)
|
||
|
|
|
||
|
|
c.sudo('supervisorctl stop {}:'.format(safename), warn=True)
|
||
|
|
|
||
|
|
# virtualenv
|
||
|
|
if not exists(c, envroot):
|
||
|
|
c.sudo('python3 -m venv {}'.format(envroot), user='rattail')
|
||
|
|
c.sudo('{}/bin/pip install --upgrade pip wheel'.format(envroot), user='rattail')
|
||
|
|
deploy(c, 'theo-common/pip.conf.mako', '{}/pip.conf'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0600',
|
||
|
|
context={'env': env, 'envroot': envroot})
|
||
|
|
c.sudo('touch {}/pip.log'.format(envroot), user='rattail')
|
||
|
|
c.sudo('chmod 0600 {}/pip.log'.format(envroot))
|
||
|
|
|
||
|
|
# theo
|
||
|
|
if from_source:
|
||
|
|
install_theo_source(c, envroot)
|
||
|
|
# always install theo package by name, even if running from source
|
||
|
|
if env.theo_integrates_with == 'corepos':
|
||
|
|
pkgname = 'tailbone-theo[app,corepos]'
|
||
|
|
elif env.theo_integrates_with == 'catapult':
|
||
|
|
pkgname = 'tailbone-theo[app,catapult]'
|
||
|
|
else:
|
||
|
|
pkgname = 'tailbone-theo[app]'
|
||
|
|
c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install {1}'".format(envroot, pkgname),
|
||
|
|
user='rattail')
|
||
|
|
|
||
|
|
# app dir
|
||
|
|
if not exists(c, '{}/app'.format(envroot)):
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail make-appdir'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
c.sudo('chmod 0750 {}/app/log'.format(envroot))
|
||
|
|
|
||
|
|
# config
|
||
|
|
deploy(c, 'theo-common/rattail.conf.mako', '{}/app/rattail.conf'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0600',
|
||
|
|
context={'env': env, 'envroot': envroot, 'dbname': dbname, 'production': production})
|
||
|
|
if not exists(c, '{}/app/quiet.conf'.format(envroot)):
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail make-config -T quiet -O app/'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
deploy(c, 'theo-common/web.conf.mako', '{}/app/web.conf'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0600',
|
||
|
|
context={'env': env, 'envroot': envroot, 'dbname': dbname, 'port': port})
|
||
|
|
deploy(c, 'theo-common/cron.conf.mako', '{}/app/cron.conf'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:',
|
||
|
|
context={'envroot': envroot})
|
||
|
|
|
||
|
|
# scripts
|
||
|
|
deploy(c, 'theo-common/upgrade.sh.mako', '{}/app/upgrade.sh'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0755',
|
||
|
|
context={'env': env, 'envroot': envroot, 'production': production})
|
||
|
|
deploy(c, 'theo-common/tasks.py.mako', '{}/app/tasks.py'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:',
|
||
|
|
context={'envroot': envroot})
|
||
|
|
deploy(c, 'theo-common/upgrade-wrapper.sh.mako', '{}/app/upgrade-wrapper.sh'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0755',
|
||
|
|
context={'envroot': envroot, 'safename': safename})
|
||
|
|
deploy(c, 'theo-common/overnight.sh.mako', '{}/app/overnight.sh'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0755',
|
||
|
|
context={'env': env, 'envroot': envroot})
|
||
|
|
if env.theo_integrates_with == 'corepos':
|
||
|
|
deploy(c, 'theo-common/import-corepos-first.sh.mako', '{}/app/import-corepos-first.sh'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0755',
|
||
|
|
context={'envroot': envroot})
|
||
|
|
elif env.theo_integrates_with == 'catapult':
|
||
|
|
deploy(c, 'theo-common/import-catapult-first.sh.mako', '{}/app/import-catapult-first.sh'.format(envroot),
|
||
|
|
use_sudo=True, owner='rattail:', mode='0755',
|
||
|
|
context={'envroot': envroot})
|
||
|
|
|
||
|
|
# theo db
|
||
|
|
if not postgresql.db_exists(c, dbname):
|
||
|
|
postgresql.create_db(c, dbname, owner='rattail', checkfirst=False)
|
||
|
|
c.sudo("bash -c 'cd {} && bin/alembic -c app/rattail.conf upgrade heads'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
|
||
|
|
# misc. default settings
|
||
|
|
postgresql.sql(c, "insert into setting values ('tailbone.theme', 'falafel')",
|
||
|
|
database=dbname)
|
||
|
|
postgresql.sql(c, "insert into setting values ('tailbone.themes.expose_picker', 'false')",
|
||
|
|
database=dbname)
|
||
|
|
postgresql.sql(c, "insert into setting values ('tailbone.global_help_url', 'https://rattailproject.org/moin/Documentation')",
|
||
|
|
database=dbname)
|
||
|
|
|
||
|
|
# theo user
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --no-versioning make-user theo --no-password'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo import-versions User'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
|
||
|
|
# admin user
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --admin {} --password {}'".format(
|
||
|
|
envroot, env.theo_admin_username, env.theo_admin_password),
|
||
|
|
user='rattail', echo=False)
|
||
|
|
|
||
|
|
# integration users
|
||
|
|
if env.theo_integrates_with == 'corepos':
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password corepos'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
elif env.theo_integrates_with == 'catapult':
|
||
|
|
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password catapult'".format(envroot),
|
||
|
|
user='rattail')
|
||
|
|
|
||
|
|
# supervisor
|
||
|
|
deploy(c, 'theo-common/supervisor.conf.mako', '/etc/supervisor/conf.d/{}.conf'.format(safename),
|
||
|
|
use_sudo=True, context={'envroot': envroot, 'safename': safename})
|
||
|
|
c.sudo('supervisorctl update')
|
||
|
|
c.sudo('supervisorctl start {}:'.format(safename))
|
||
|
|
|
||
|
|
# sudoers, crontab, logrotate
|
||
|
|
deploy.sudoers(c, 'theo-common/sudoers.mako', '/etc/sudoers.d/{}'.format(safename),
|
||
|
|
context={'envname': envname, 'envroot': envroot})
|
||
|
|
deploy(c, 'theo-common/crontab.mako', '/etc/cron.d/{}'.format(safename),
|
||
|
|
use_sudo=True,
|
||
|
|
context={'env': env, 'envname': envname, 'envroot': envroot,
|
||
|
|
'pretty_time': run_overnight_at.strftime('%I:%M %p'),
|
||
|
|
'cron_time': run_overnight_at.strftime('%M %H')})
|
||
|
|
deploy(c, 'theo-common/logrotate.conf.mako', '/etc/logrotate.d/{}'.format(safename),
|
||
|
|
use_sudo=True, context={'envroot': envroot})
|
||
|
|
|
||
|
|
|
||
|
|
def install_theo_source(c, envroot):
|
||
|
|
"""
|
||
|
|
Install source code for Theo
|
||
|
|
"""
|
||
|
|
# rattail
|
||
|
|
install_source_package(c, envroot, 'rattail')
|
||
|
|
install_source_package(c, envroot, 'tailbone')
|
||
|
|
|
||
|
|
# corepos
|
||
|
|
if env.theo_integrates_with == 'corepos':
|
||
|
|
install_source_package(c, envroot, 'pycorepos')
|
||
|
|
install_source_package(c, envroot, 'rattail-corepos')
|
||
|
|
install_source_package(c, envroot, 'tailbone-corepos')
|
||
|
|
|
||
|
|
# catapult
|
||
|
|
if env.theo_integrates_with == 'catapult':
|
||
|
|
install_source_package(c, envroot, 'onager', restricted=True)
|
||
|
|
install_source_package(c, envroot, 'rattail-onager', restricted=True)
|
||
|
|
install_source_package(c, envroot, 'tailbone-onager', restricted=True)
|
||
|
|
|
||
|
|
# theo
|
||
|
|
install_source_package(c, envroot, 'theo')
|
||
|
|
|
||
|
|
|
||
|
|
def install_source_package(c, envroot, name, restricted=False):
|
||
|
|
if not exists(c, '{}/src/{}'.format(envroot, name)):
|
||
|
|
if restricted:
|
||
|
|
c.sudo('git clone https://{0}:{1}@kallithea.rattailproject.org/rattail-restricted/{2} {3}/src/{2}'.format(
|
||
|
|
env.kallithea_username, env.kallithea_password, name, envroot),
|
||
|
|
user='rattail', echo=False)
|
||
|
|
else:
|
||
|
|
c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/{0} {1}/src/{0}'.format(name, envroot),
|
||
|
|
user='rattail')
|
||
|
|
c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install -e src/{1}'".format(envroot, name),
|
||
|
|
user='rattail')
|
||
|
|
|
||
|
|
|
||
|
|
##############################
|
||
|
|
# fabenv
|
||
|
|
##############################
|
||
|
|
|
||
|
|
try:
|
||
|
|
import fabenv
|
||
|
|
except ImportError as error:
|
||
|
|
print("\ncouldn't import fabenv: {}".format(error))
|
||
|
|
|
||
|
|
env.setdefault('machine_is_live', False)
|