# -*- coding: utf-8; -*-
"""
Fabric script for a server running Theo, the order system
Please see the accompanying README for full instructions.
"""
import datetime
from fabric2 import task
from rattail.core import Object
from rattail_fabric2 import make_deploy, apt, mysql, postfix, postgresql, exists, make_system_user, mkdir
env = Object()
deploy = make_deploy(__file__)
##############################
# bootstrap
##############################
@task
def bootstrap_all(c):
"""
Bootstrap all aspects of the machine
"""
bootstrap_base(c)
if env.theo_install_production:
bootstrap_theo(c)
bootstrap_theo_stage(c)
@task
def bootstrap_base(c):
"""
Bootstrap the base system
"""
# we do `apt update && apt dist-upgrade` every time this task runs
apt.dist_upgrade(c)
# we assume postfix is installed, but the configuration thereof is not
# dealt with here; you may need to configure further on your own
postfix.install(c)
# rattail user + common config
make_system_user(c, 'rattail', home='/var/lib/rattail', shell='/bin/bash')
postfix.alias(c, 'rattail', 'root')
mkdir(c, '/etc/rattail', use_sudo=True)
deploy(c, 'rattail/rattail.conf.mako', '/etc/rattail/rattail.conf',
use_sudo=True, context={'env': env})
# postgresql
postgresql.install(c)
postgresql.create_user(c, 'rattail', password=env.password_postgresql_rattail)
# mysql (maybe)
if env.theo_mirror_posdb == 'mysql':
mysql.install(c)
mysql.create_user(c, 'rattail', password=env.password_mysql_rattail)
# python
mkdir(c, '/srv/envs', use_sudo=True, owner='rattail:')
apt.install(
c,
'build-essential',
'git',
'libpq-dev',
'python3-dev',
'python3-venv',
'supervisor',
)
# catapult extras
if env.theo_integrates_with == 'catapult':
# freetds / odbc
apt.install(c, 'unixodbc', 'unixodbc-dev')
# we must install FreeTDS from source, b/c the version APT makes available
# is too old. however the *latest* source seems to have some issue(s)
# which cause it to use too much memory, so we use a more stable branch
from rattail_fabric2 import freetds
freetds.install_from_source(c, user='rattail', branch='Branch-1_2')
deploy(c, 'rattail/catapult/freetds.conf.mako', '/usr/local/etc/freetds.conf',
use_sudo=True, context={'env': env})
deploy(c, 'rattail/catapult/odbc.ini', '/etc/odbc.ini', use_sudo=True)
# locsms extras
elif env.theo_integrates_with == 'locsms':
apt.install(c, 'unixodbc', 'unixodbc-dev')
# mssql odbc
if env.locsms_odbc_driver == 'mssql':
from rattail_fabric2 import mssql
mssql.install_mssql_odbc(c, accept_eula=env.mssql_driver_accept_eula)
deploy(c, 'rattail/locsms/mssql/odbc.ini.mako', '/etc/odbc.ini',
use_sudo=True, context={'env': env})
else: # freetds odbc
apt.install(c, 'tdsodbc')
deploy(c, 'rattail/locsms/freetds/freetds.conf.mako', '/etc/freetds/freetds.conf',
use_sudo=True, context={'env': env})
deploy(c, 'rattail/locsms/freetds/odbc.ini.mako', '/etc/odbc.ini',
use_sudo=True, context={'env': env})
@task
def bootstrap_theo(c):
"""
Bootstrap Theo, the order system
"""
install_theo_app(c, 'theo', True, 7000,
# overnight runs at 1:00am
run_overnight_at=datetime.time(1))
@task
def bootstrap_theo_stage(c):
"""
Bootstrap "stage" for Theo, the order system
"""
install_theo_app(c, 'theo-stage', False, 7010, from_source=True,
# overnight runs at 12:30am
run_overnight_at=datetime.time(0, 30))
##############################
# support functions
##############################
def install_theo_app(c, envname, production, port, from_source=False,
run_overnight_at=None):
"""
Install a Theo app, per the given parameters
"""
dbname = envname
safename = envname.replace('-', '_')
envroot = '/srv/envs/{}'.format(envname)
mirrordb = None
if env.theo_integrates_with == 'catapult':
mirrordb = 'catapult-mirror' if production else 'catapult-mirror-stage'
c.sudo('supervisorctl stop {}:'.format(safename), warn=True)
# virtualenv
if not exists(c, envroot):
c.sudo('python3 -m venv {}'.format(envroot), user='rattail')
c.sudo('{}/bin/pip install --upgrade pip wheel'.format(envroot), user='rattail')
deploy(c, 'theo-common/pip.conf.mako', '{}/pip.conf'.format(envroot),
use_sudo=True, owner='rattail:', mode='0600',
context={'env': env, 'envroot': envroot})
c.sudo('touch {}/pip.log'.format(envroot), user='rattail')
c.sudo('chmod 0600 {}/pip.log'.format(envroot))
c.sudo('PIP_CONFIG_FILE={0}/pip.conf {0}/bin/pip install ipdb'.format(envroot),
user='rattail')
# theo
if from_source:
install_theo_source(c, envroot)
# always install theo package by name, even if running from source
if env.theo_integrates_with == 'corepos':
pkgname = 'tailbone-theo[app,corepos]'
elif env.theo_integrates_with == 'catapult':
pkgname = 'tailbone-theo[app,catapult]'
elif env.theo_integrates_with == 'locsms':
pkgname = 'tailbone-theo[app,locsms]'
else:
pkgname = 'tailbone-theo[app]'
c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install {1}'".format(envroot, pkgname),
user='rattail')
# app dir
if not exists(c, '{}/app'.format(envroot)):
c.sudo("bash -c 'cd {} && bin/rattail make-appdir'".format(envroot),
user='rattail')
c.sudo('chmod 0750 {}/app/log'.format(envroot))
# config
deploy(c, 'theo-common/rattail.conf.mako', '{}/app/rattail.conf'.format(envroot),
use_sudo=True, owner='rattail:', mode='0600',
context={'env': env, 'envroot': envroot, 'dbname': dbname,
'production': production, 'mirrordb': mirrordb})
if not exists(c, '{}/app/quiet.conf'.format(envroot)):
c.sudo("bash -c 'cd {} && bin/rattail make-config -T quiet -O app/'".format(envroot),
user='rattail')
deploy(c, 'theo-common/web.conf.mako', '{}/app/web.conf'.format(envroot),
use_sudo=True, owner='rattail:', mode='0600',
context={'env': env, 'envroot': envroot, 'dbname': dbname, 'port': port})
deploy(c, 'theo-common/cron.conf.mako', '{}/app/cron.conf'.format(envroot),
use_sudo=True, owner='rattail:',
context={'envroot': envroot})
if env.theo_mirror_posdb:
if env.theo_integrates_with == 'catapult':
deploy(c, 'theo-common/catapult-mirror.conf', '{}/app/catapult-mirror.conf'.format(envroot),
use_sudo=True, owner='rattail:')
# scripts
deploy(c, 'theo-common/upgrade.sh.mako', '{}/app/upgrade.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'env': env, 'envroot': envroot, 'production': production})
deploy(c, 'theo-common/tasks.py.mako', '{}/app/tasks.py'.format(envroot),
use_sudo=True, owner='rattail:',
context={'envroot': envroot})
deploy(c, 'theo-common/upgrade-wrapper.sh.mako', '{}/app/upgrade-wrapper.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'envroot': envroot, 'safename': safename})
deploy(c, 'theo-common/overnight.sh.mako', '{}/app/overnight.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'env': env, 'envroot': envroot})
if env.theo_integrates_with == 'corepos':
deploy(c, 'theo-common/import-corepos-first.sh.mako', '{}/app/import-corepos-first.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'envroot': envroot})
elif env.theo_integrates_with == 'catapult':
deploy(c, 'theo-common/import-catapult-first.sh.mako', '{}/app/import-catapult-first.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'envroot': envroot})
if env.theo_mirror_posdb:
deploy(c, 'theo-common/mirror-catapult-first.sh.mako', '{}/app/mirror-catapult-first.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'envroot': envroot})
elif env.theo_integrates_with == 'locsms':
deploy(c, 'theo-common/import-locsms-first.sh.mako', '{}/app/import-locsms-first.sh'.format(envroot),
use_sudo=True, owner='rattail:', mode='0755',
context={'envroot': envroot})
# theo db
if not postgresql.db_exists(c, dbname):
postgresql.create_db(c, dbname, owner='rattail', checkfirst=False)
c.sudo("bash -c 'cd {} && bin/alembic -c app/rattail.conf upgrade heads'".format(envroot),
user='rattail')
# misc. default settings
postgresql.sql(c, "insert into setting values ('tailbone.theme', 'falafel')",
database=dbname)
postgresql.sql(c, "insert into setting values ('tailbone.themes.expose_picker', 'false')",
database=dbname)
postgresql.sql(c, "insert into setting values ('tailbone.global_help_url', 'https://rattailproject.org/moin/Documentation')",
database=dbname)
# theo user
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --no-versioning make-user theo --no-password'".format(envroot),
user='rattail')
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo import-versions User'".format(envroot),
user='rattail')
# admin user
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --admin {} --password {}'".format(
envroot, env.theo_admin_username, env.theo_admin_password),
user='rattail', echo=False)
# integration users
if env.theo_integrates_with == 'corepos':
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password corepos'".format(envroot),
user='rattail')
elif env.theo_integrates_with == 'catapult':
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password catapult'".format(envroot),
user='rattail')
elif env.theo_integrates_with == 'locsms':
c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password locsms'".format(envroot),
user='rattail')
# tweaks for mirror DB
if env.theo_mirror_posdb and env.theo_integrates_with == 'catapult':
postgresql.sql(c, "insert into setting values ('tailbone.engines.catapult.pretend_default', 'mirror')",
database=dbname)
postgresql.sql(c, "insert into setting values ('rattail_onager.catapult.importing.db.default_models', 'VersionInfo Store Department Vendor Brand FamilyLine ComboGroupProfile WeightProfile PricingLevelInfo DiscountProfile StockInventory PricingLevelDetail Prefix Suffix DiscountGroup Customer Employee PhoneDescription Phone EmailDescription Email AddressDescription Address')",
database=dbname)
postgresql.sql(c, "insert into setting values ('rattail.mail.rattail_onager_import_catapult_updates.subject', 'Changes for Catapult (production) -> Catapult (mirror)')",
database=dbname)
# maybe establish the mirror DB
if env.theo_mirror_posdb == 'mysql':
if env.theo_integrates_with == 'catapult':
if not mysql.db_exists(c, mirrordb):
mysql.create_db(c, mirrordb, checkfirst=False, user='rattail@localhost')
c.sudo("bash -c 'cd {} && bin/alembic -c app/catapult-mirror.conf upgrade heads'".format(envroot),
user='rattail')
# supervisor
deploy(c, 'theo-common/supervisor.conf.mako', '/etc/supervisor/conf.d/{}.conf'.format(safename),
use_sudo=True, context={'envroot': envroot, 'safename': safename})
c.sudo('supervisorctl update')
c.sudo('supervisorctl start {}:'.format(safename))
# sudoers, crontab, logrotate
deploy.sudoers(c, 'theo-common/sudoers.mako', '/etc/sudoers.d/{}'.format(safename),
context={'envname': envname, 'envroot': envroot})
deploy(c, 'theo-common/crontab.mako', '/etc/cron.d/{}'.format(safename),
use_sudo=True,
context={'env': env, 'envname': envname, 'envroot': envroot,
'pretty_time': run_overnight_at.strftime('%I:%M %p'),
'cron_time': run_overnight_at.strftime('%M %H')})
deploy(c, 'theo-common/logrotate.conf.mako', '/etc/logrotate.d/{}'.format(safename),
use_sudo=True, context={'envroot': envroot})
def install_theo_source(c, envroot):
"""
Install source code for Theo
"""
# rattail
install_source_package(c, envroot, 'rattail')
install_source_package(c, envroot, 'tailbone')
# corepos
if env.theo_integrates_with == 'corepos':
install_source_package(c, envroot, 'pycorepos')
install_source_package(c, envroot, 'rattail-corepos')
install_source_package(c, envroot, 'tailbone-corepos')
# catapult
elif env.theo_integrates_with == 'catapult':
install_source_package(c, envroot, 'onager', restricted=True)
install_source_package(c, envroot, 'rattail-onager', restricted=True)
install_source_package(c, envroot, 'tailbone-onager', restricted=True)
# locsms
elif env.theo_integrates_with == 'locsms':
install_source_package(c, envroot, 'luckysmores', restricted=True)
install_source_package(c, envroot, 'rattail-luckysmores', restricted=True)
install_source_package(c, envroot, 'tailbone-locsms', restricted=True)
# theo
install_source_package(c, envroot, 'theo')
def install_source_package(c, envroot, name, restricted=False):
if not exists(c, '{}/src/{}'.format(envroot, name)):
if restricted:
c.sudo('git clone https://{0}:{1}@kallithea.rattailproject.org/rattail-restricted/{2} {3}/src/{2}'.format(
env.kallithea_username, env.kallithea_password, name, envroot),
user='rattail', echo=False)
else:
c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/{0} {1}/src/{0}'.format(name, envroot),
user='rattail')
c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install -e src/{1}'".format(envroot, name),
user='rattail')
##############################
# fabenv
##############################
try:
import fabenv
except ImportError as error:
print("\ncouldn't import fabenv: {}".format(error))
env.setdefault('machine_is_live', False)