# -*- coding: utf-8; -*- """ Fabric script for a server running Theo, the order system Please see the accompanying README for full instructions. """ import datetime from fabric2 import task from rattail.core import Object from rattail_fabric2 import make_deploy, apt, mysql, postfix, postgresql, exists, make_system_user, mkdir env = Object() deploy = make_deploy(__file__) ############################## # bootstrap ############################## @task def bootstrap_all(c): """ Bootstrap all aspects of the machine """ bootstrap_base(c) bootstrap_theo(c) bootstrap_theo_stage(c) @task def bootstrap_base(c): """ Bootstrap the base system """ # we do `apt update && apt dist-upgrade` every time this task runs apt.dist_upgrade(c) # we assume postfix is installed, but the configuration thereof is not # dealt with here; you may need to configure further on your own postfix.install(c) # rattail user + common config make_system_user(c, 'rattail', home='/var/lib/rattail', shell='/bin/bash') postfix.alias(c, 'rattail', 'root') mkdir(c, '/etc/rattail', use_sudo=True) deploy(c, 'rattail/rattail.conf.mako', '/etc/rattail/rattail.conf', use_sudo=True, context={'env': env}) # postgresql postgresql.install(c) postgresql.create_user(c, 'rattail', password=env.password_postgresql_rattail) # mysql (maybe) if env.theo_mirror_posdb == 'mysql': mysql.install(c) mysql.create_user(c, 'rattail', password=env.password_mysql_rattail) # python mkdir(c, '/srv/envs', use_sudo=True, owner='rattail:') apt.install( c, 'build-essential', 'git', 'libpq-dev', 'python3-dev', 'python3-venv', 'supervisor', ) # catapult extras if env.theo_integrates_with == 'catapult': # freetds / odbc apt.install(c, 'unixodbc', 'unixodbc-dev') # we must install FreeTDS from source, b/c the version APT makes available # is too old. however the *latest* source seems to have some issue(s) # which cause it to use too much memory, so we use a more stable branch from rattail_fabric2 import freetds freetds.install_from_source(c, user='rattail', branch='Branch-1_2') deploy(c, 'rattail/catapult/freetds.conf.mako', '/usr/local/etc/freetds.conf', use_sudo=True, context={'env': env}) deploy(c, 'rattail/catapult/odbc.ini', '/etc/odbc.ini', use_sudo=True) # locsms extras elif env.theo_integrates_with == 'locsms': apt.install(c, 'unixodbc', 'unixodbc-dev') # mssql odbc if env.locsms_odbc_driver == 'mssql': from rattail_fabric2 import mssql mssql.install_mssql_odbc(c, accept_eula=env.mssql_driver_accept_eula) deploy(c, 'rattail/locsms/mssql/odbc.ini.mako', '/etc/odbc.ini', use_sudo=True, context={'env': env}) else: # freetds odbc apt.install(c, 'tdsodbc') deploy(c, 'rattail/locsms/freetds/freetds.conf.mako', '/etc/freetds/freetds.conf', use_sudo=True, context={'env': env}) deploy(c, 'rattail/locsms/freetds/odbc.ini.mako', '/etc/odbc.ini', use_sudo=True, context={'env': env}) @task def bootstrap_theo(c): """ Bootstrap Theo, the order system """ install_theo_app(c, 'theo', True, 7000, # overnight runs at 1:00am run_overnight_at=datetime.time(1)) @task def bootstrap_theo_stage(c): """ Bootstrap "stage" for Theo, the order system """ install_theo_app(c, 'theo-stage', False, 7010, from_source=True, # overnight runs at 12:30am run_overnight_at=datetime.time(0, 30)) ############################## # support functions ############################## def install_theo_app(c, envname, production, port, from_source=False, run_overnight_at=None): """ Install a Theo app, per the given parameters """ dbname = envname safename = envname.replace('-', '_') envroot = '/srv/envs/{}'.format(envname) mirrordb = None if env.theo_integrates_with == 'catapult': mirrordb = 'catapult-mirror' if production else 'catapult-mirror-stage' c.sudo('supervisorctl stop {}:'.format(safename), warn=True) # virtualenv if not exists(c, envroot): c.sudo('python3 -m venv {}'.format(envroot), user='rattail') c.sudo('{}/bin/pip install --upgrade pip wheel'.format(envroot), user='rattail') deploy(c, 'theo-common/pip.conf.mako', '{}/pip.conf'.format(envroot), use_sudo=True, owner='rattail:', mode='0600', context={'env': env, 'envroot': envroot}) c.sudo('touch {}/pip.log'.format(envroot), user='rattail') c.sudo('chmod 0600 {}/pip.log'.format(envroot)) c.sudo('PIP_CONFIG_FILE={0}/pip.conf {0}/bin/pip install ipdb'.format(envroot), user='rattail') # theo if from_source: install_theo_source(c, envroot) # always install theo package by name, even if running from source if env.theo_integrates_with == 'corepos': pkgname = 'tailbone-theo[app,corepos]' elif env.theo_integrates_with == 'catapult': pkgname = 'tailbone-theo[app,catapult]' elif env.theo_integrates_with == 'locsms': pkgname = 'tailbone-theo[app,locsms]' else: pkgname = 'tailbone-theo[app]' c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install {1}'".format(envroot, pkgname), user='rattail') # app dir if not exists(c, '{}/app'.format(envroot)): c.sudo("bash -c 'cd {} && bin/rattail make-appdir'".format(envroot), user='rattail') c.sudo('chmod 0750 {}/app/log'.format(envroot)) # config deploy(c, 'theo-common/rattail.conf.mako', '{}/app/rattail.conf'.format(envroot), use_sudo=True, owner='rattail:', mode='0600', context={'env': env, 'envroot': envroot, 'dbname': dbname, 'production': production, 'mirrordb': mirrordb}) if not exists(c, '{}/app/quiet.conf'.format(envroot)): c.sudo("bash -c 'cd {} && bin/rattail make-config -T quiet -O app/'".format(envroot), user='rattail') deploy(c, 'theo-common/web.conf.mako', '{}/app/web.conf'.format(envroot), use_sudo=True, owner='rattail:', mode='0600', context={'env': env, 'envroot': envroot, 'dbname': dbname, 'port': port}) deploy(c, 'theo-common/cron.conf.mako', '{}/app/cron.conf'.format(envroot), use_sudo=True, owner='rattail:', context={'envroot': envroot}) if env.theo_mirror_posdb: if env.theo_integrates_with == 'catapult': deploy(c, 'theo-common/catapult-mirror.conf', '{}/app/catapult-mirror.conf'.format(envroot), use_sudo=True, owner='rattail:') # scripts deploy(c, 'theo-common/upgrade.sh.mako', '{}/app/upgrade.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'env': env, 'envroot': envroot, 'production': production}) deploy(c, 'theo-common/tasks.py.mako', '{}/app/tasks.py'.format(envroot), use_sudo=True, owner='rattail:', context={'envroot': envroot}) deploy(c, 'theo-common/upgrade-wrapper.sh.mako', '{}/app/upgrade-wrapper.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'envroot': envroot, 'safename': safename}) deploy(c, 'theo-common/overnight.sh.mako', '{}/app/overnight.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'env': env, 'envroot': envroot}) if env.theo_integrates_with == 'corepos': deploy(c, 'theo-common/import-corepos-first.sh.mako', '{}/app/import-corepos-first.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'envroot': envroot}) elif env.theo_integrates_with == 'catapult': deploy(c, 'theo-common/import-catapult-first.sh.mako', '{}/app/import-catapult-first.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'envroot': envroot}) if env.theo_mirror_posdb: deploy(c, 'theo-common/mirror-catapult-first.sh.mako', '{}/app/mirror-catapult-first.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'envroot': envroot}) elif env.theo_integrates_with == 'locsms': deploy(c, 'theo-common/import-locsms-first.sh.mako', '{}/app/import-locsms-first.sh'.format(envroot), use_sudo=True, owner='rattail:', mode='0755', context={'envroot': envroot}) # theo db if not postgresql.db_exists(c, dbname): postgresql.create_db(c, dbname, owner='rattail', checkfirst=False) c.sudo("bash -c 'cd {} && bin/alembic -c app/rattail.conf upgrade heads'".format(envroot), user='rattail') # misc. default settings postgresql.sql(c, "insert into setting values ('tailbone.theme', 'falafel')", database=dbname) postgresql.sql(c, "insert into setting values ('tailbone.themes.expose_picker', 'false')", database=dbname) postgresql.sql(c, "insert into setting values ('tailbone.global_help_url', 'https://rattailproject.org/moin/Documentation')", database=dbname) # theo user c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --no-versioning make-user theo --no-password'".format(envroot), user='rattail') c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo import-versions User'".format(envroot), user='rattail') # admin user c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --admin {} --password {}'".format( envroot, env.theo_admin_username, env.theo_admin_password), user='rattail', echo=False) # integration users if env.theo_integrates_with == 'corepos': c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password corepos'".format(envroot), user='rattail') elif env.theo_integrates_with == 'catapult': c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password catapult'".format(envroot), user='rattail') elif env.theo_integrates_with == 'locsms': c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password locsms'".format(envroot), user='rattail') # tweaks for mirror DB if env.theo_mirror_posdb and env.theo_integrates_with == 'catapult': postgresql.sql(c, "insert into setting values ('tailbone.engines.catapult.pretend_default', 'mirror')", database=dbname) # maybe establish the mirror DB if env.theo_mirror_posdb == 'mysql': if env.theo_integrates_with == 'catapult': if not mysql.db_exists(c, mirrordb): mysql.create_db(c, mirrordb, checkfirst=False, user='rattail@localhost') c.sudo("bash -c 'cd {} && bin/alembic -c app/catapult-mirror.conf upgrade heads'".format(envroot), user='rattail') # supervisor deploy(c, 'theo-common/supervisor.conf.mako', '/etc/supervisor/conf.d/{}.conf'.format(safename), use_sudo=True, context={'envroot': envroot, 'safename': safename}) c.sudo('supervisorctl update') c.sudo('supervisorctl start {}:'.format(safename)) # sudoers, crontab, logrotate deploy.sudoers(c, 'theo-common/sudoers.mako', '/etc/sudoers.d/{}'.format(safename), context={'envname': envname, 'envroot': envroot}) deploy(c, 'theo-common/crontab.mako', '/etc/cron.d/{}'.format(safename), use_sudo=True, context={'env': env, 'envname': envname, 'envroot': envroot, 'pretty_time': run_overnight_at.strftime('%I:%M %p'), 'cron_time': run_overnight_at.strftime('%M %H')}) deploy(c, 'theo-common/logrotate.conf.mako', '/etc/logrotate.d/{}'.format(safename), use_sudo=True, context={'envroot': envroot}) def install_theo_source(c, envroot): """ Install source code for Theo """ # rattail install_source_package(c, envroot, 'rattail') install_source_package(c, envroot, 'tailbone') # corepos if env.theo_integrates_with == 'corepos': install_source_package(c, envroot, 'pycorepos') install_source_package(c, envroot, 'rattail-corepos') install_source_package(c, envroot, 'tailbone-corepos') # catapult elif env.theo_integrates_with == 'catapult': install_source_package(c, envroot, 'onager', restricted=True) install_source_package(c, envroot, 'rattail-onager', restricted=True) install_source_package(c, envroot, 'tailbone-onager', restricted=True) # locsms elif env.theo_integrates_with == 'locsms': install_source_package(c, envroot, 'luckysmores', restricted=True) install_source_package(c, envroot, 'rattail-luckysmores', restricted=True) install_source_package(c, envroot, 'tailbone-locsms', restricted=True) # theo install_source_package(c, envroot, 'theo') def install_source_package(c, envroot, name, restricted=False): if not exists(c, '{}/src/{}'.format(envroot, name)): if restricted: c.sudo('git clone https://{0}:{1}@kallithea.rattailproject.org/rattail-restricted/{2} {3}/src/{2}'.format( env.kallithea_username, env.kallithea_password, name, envroot), user='rattail', echo=False) else: c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/{0} {1}/src/{0}'.format(name, envroot), user='rattail') c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install -e src/{1}'".format(envroot, name), user='rattail') ############################## # fabenv ############################## try: import fabenv except ImportError as error: print("\ncouldn't import fabenv: {}".format(error)) env.setdefault('machine_is_live', False)