theo/machines/theo-server/fabfile.py

# -*- coding: utf-8; -*-
"""
Fabric script for a server running Theo, the order system

Please see the accompanying README for full instructions.
"""

import datetime

from fabric2 import task

from rattail.core import Object
from rattail_fabric2 import make_deploy, apt, mysql, postfix, postgresql, exists, make_system_user, mkdir


env = Object()
deploy = make_deploy(__file__)


##############################
# bootstrap
##############################

@task
def bootstrap_all(c):
    """
    Bootstrap all aspects of the machine
    """
    bootstrap_base(c)
    if env.theo_install_production:
        bootstrap_theo(c)
    bootstrap_theo_stage(c)


@task
def bootstrap_base(c):
    """
    Bootstrap the base system
    """
    # we do `apt update && apt dist-upgrade` every time this task runs
    apt.dist_upgrade(c)

    # we assume postfix is installed, but the configuration thereof is not
    # dealt with here; you may need to configure further on your own
    postfix.install(c)

    # rattail user + common config
    make_system_user(c, 'rattail', home='/var/lib/rattail', shell='/bin/bash')
    postfix.alias(c, 'rattail', 'root')
    mkdir(c, '/etc/rattail', use_sudo=True)
    deploy(c, 'rattail/rattail.conf.mako', '/etc/rattail/rattail.conf',
           use_sudo=True, context={'env': env})

    # postgresql
    postgresql.install(c)
    postgresql.create_user(c, 'rattail', password=env.password_postgresql_rattail)

    # mysql (maybe)
    if env.theo_mirror_posdb == 'mysql':
        mysql.install(c)
        mysql.create_user(c, 'rattail', password=env.password_mysql_rattail)

    # python
    mkdir(c, '/srv/envs', use_sudo=True, owner='rattail:')
    apt.install(
        c,
        'build-essential',
        'git',
        'libpq-dev',
        'python3-dev',
        'python3-venv',
        'supervisor',
    )

    # catapult extras
    if env.theo_integrates_with == 'catapult':

        # freetds / odbc
        apt.install(c, 'unixodbc', 'unixodbc-dev')
        # we must install FreeTDS from source, b/c the version APT makes available
        # is too old.  however the *latest* source seems to have some issue(s)
        # which cause it to use too much memory, so we use a more stable branch
        from rattail_fabric2 import freetds
        freetds.install_from_source(c, user='rattail', branch='Branch-1_2')
        deploy(c, 'rattail/catapult/freetds.conf.mako', '/usr/local/etc/freetds.conf',
               use_sudo=True, context={'env': env})
        deploy(c, 'rattail/catapult/odbc.ini', '/etc/odbc.ini', use_sudo=True)

    # locsms extras
    elif env.theo_integrates_with == 'locsms':
        apt.install(c, 'unixodbc', 'unixodbc-dev')

        # mssql odbc
        if env.locsms_odbc_driver == 'mssql':
            from rattail_fabric2 import mssql
            mssql.install_mssql_odbc(c, accept_eula=env.mssql_driver_accept_eula)
            deploy(c, 'rattail/locsms/mssql/odbc.ini.mako', '/etc/odbc.ini',
                   use_sudo=True, context={'env': env})

        else: # freetds odbc
            apt.install(c, 'tdsodbc')
            deploy(c, 'rattail/locsms/freetds/freetds.conf.mako', '/etc/freetds/freetds.conf',
                   use_sudo=True, context={'env': env})
            deploy(c, 'rattail/locsms/freetds/odbc.ini.mako', '/etc/odbc.ini',
                   use_sudo=True, context={'env': env})


@task
def bootstrap_theo(c):
    """
    Bootstrap Theo, the order system
    """
    install_theo_app(c, 'theo', True, 7000,
                     # overnight runs at 1:00am
                     run_overnight_at=datetime.time(1))


@task
def bootstrap_theo_stage(c):
    """
    Bootstrap "stage" for Theo, the order system
    """
    install_theo_app(c, 'theo-stage', False, 7010, from_source=True,
                     # overnight runs at 12:30am
                     run_overnight_at=datetime.time(0, 30))


##############################
# support functions
##############################

def install_theo_app(c, envname, production, port, from_source=False,
                     run_overnight_at=None):
    """
    Install a Theo app, per the given parameters
    """
    dbname = envname
    safename = envname.replace('-', '_')
    envroot = '/srv/envs/{}'.format(envname)

    mirrordb = None
    if env.theo_integrates_with == 'catapult':
        mirrordb = 'catapult-mirror' if production else 'catapult-mirror-stage'

    c.sudo('supervisorctl stop {}:'.format(safename), warn=True)

    # virtualenv
    if not exists(c, envroot):
        c.sudo('python3 -m venv {}'.format(envroot), user='rattail')
        c.sudo('{}/bin/pip install --upgrade pip wheel'.format(envroot), user='rattail')
    deploy(c, 'theo-common/pip.conf.mako', '{}/pip.conf'.format(envroot),
           use_sudo=True, owner='rattail:', mode='0600',
           context={'env': env, 'envroot': envroot})
    c.sudo('touch {}/pip.log'.format(envroot), user='rattail')
    c.sudo('chmod 0600 {}/pip.log'.format(envroot))
    c.sudo('PIP_CONFIG_FILE={0}/pip.conf {0}/bin/pip install ipdb'.format(envroot),
           user='rattail')

    # theo
    if from_source:
        install_theo_source(c, envroot)
    # always install theo package by name, even if running from source
    if env.theo_integrates_with == 'corepos':
        pkgname = 'tailbone-theo[app,corepos]'
    elif env.theo_integrates_with == 'catapult':
        pkgname = 'tailbone-theo[app,catapult]'
    elif env.theo_integrates_with == 'locsms':
        pkgname = 'tailbone-theo[app,locsms]'
    else:
        pkgname = 'tailbone-theo[app]'
    c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install {1}'".format(envroot, pkgname),
           user='rattail')

    # app dir
    if not exists(c, '{}/app'.format(envroot)):
        c.sudo("bash -c 'cd {} && bin/rattail make-appdir'".format(envroot),
               user='rattail')
    c.sudo('chmod 0750 {}/app/log'.format(envroot))

    # config
    deploy(c, 'theo-common/rattail.conf.mako', '{}/app/rattail.conf'.format(envroot),
           use_sudo=True, owner='rattail:', mode='0600',
           context={'env': env, 'envroot': envroot, 'dbname': dbname,
                    'production': production, 'mirrordb': mirrordb})
    if not exists(c, '{}/app/quiet.conf'.format(envroot)):
        c.sudo("bash -c 'cd {} && bin/rattail make-config -T quiet -O app/'".format(envroot),
               user='rattail')
    deploy(c, 'theo-common/web.conf.mako', '{}/app/web.conf'.format(envroot),
           use_sudo=True, owner='rattail:', mode='0600',
           context={'env': env, 'envroot': envroot, 'dbname': dbname, 'port': port})
    deploy(c, 'theo-common/cron.conf.mako', '{}/app/cron.conf'.format(envroot),
           use_sudo=True, owner='rattail:',
           context={'envroot': envroot})
    if env.theo_mirror_posdb:
        if env.theo_integrates_with == 'catapult':
            deploy(c, 'theo-common/catapult-mirror.conf', '{}/app/catapult-mirror.conf'.format(envroot),
                   use_sudo=True, owner='rattail:')

    # scripts
    deploy(c, 'theo-common/upgrade.sh.mako', '{}/app/upgrade.sh'.format(envroot),
           use_sudo=True, owner='rattail:', mode='0755',
           context={'env': env, 'envroot': envroot, 'production': production})
    deploy(c, 'theo-common/tasks.py.mako', '{}/app/tasks.py'.format(envroot),
           use_sudo=True, owner='rattail:',
           context={'envroot': envroot})
    deploy(c, 'theo-common/upgrade-wrapper.sh.mako', '{}/app/upgrade-wrapper.sh'.format(envroot),
           use_sudo=True, owner='rattail:', mode='0755',
           context={'envroot': envroot, 'safename': safename})
    deploy(c, 'theo-common/overnight.sh.mako', '{}/app/overnight.sh'.format(envroot),
           use_sudo=True, owner='rattail:', mode='0755',
           context={'env': env, 'envroot': envroot})
    if env.theo_integrates_with == 'corepos':
        deploy(c, 'theo-common/import-corepos-first.sh.mako', '{}/app/import-corepos-first.sh'.format(envroot),
               use_sudo=True, owner='rattail:', mode='0755',
               context={'envroot': envroot})
    elif env.theo_integrates_with == 'catapult':
        deploy(c, 'theo-common/import-catapult-first.sh.mako', '{}/app/import-catapult-first.sh'.format(envroot),
               use_sudo=True, owner='rattail:', mode='0755',
               context={'envroot': envroot})
        if env.theo_mirror_posdb:
            deploy(c, 'theo-common/mirror-catapult-first.sh.mako', '{}/app/mirror-catapult-first.sh'.format(envroot),
                   use_sudo=True, owner='rattail:', mode='0755',
                   context={'envroot': envroot})
    elif env.theo_integrates_with == 'locsms':
        deploy(c, 'theo-common/import-locsms-first.sh.mako', '{}/app/import-locsms-first.sh'.format(envroot),
               use_sudo=True, owner='rattail:', mode='0755',
               context={'envroot': envroot})

    # theo db
    if not postgresql.db_exists(c, dbname):
        postgresql.create_db(c, dbname, owner='rattail', checkfirst=False)
        c.sudo("bash -c 'cd {} && bin/alembic -c app/rattail.conf upgrade heads'".format(envroot),
               user='rattail')

        # misc. default settings
        postgresql.sql(c, "insert into setting values ('tailbone.theme', 'falafel')",
                       database=dbname)
        postgresql.sql(c, "insert into setting values ('tailbone.themes.expose_picker', 'false')",
                       database=dbname)
        postgresql.sql(c, "insert into setting values ('tailbone.global_help_url', 'https://rattailproject.org/moin/Documentation')",
                       database=dbname)

        # theo user
        c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --no-versioning make-user theo --no-password'".format(envroot),
               user='rattail')
        c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo import-versions User'".format(envroot),
               user='rattail')

        # admin user
        c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --admin {} --password {}'".format(
            envroot, env.theo_admin_username, env.theo_admin_password),
               user='rattail', echo=False)

        # integration users
        if env.theo_integrates_with == 'corepos':
            c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password corepos'".format(envroot),
                   user='rattail')
        elif env.theo_integrates_with == 'catapult':
            c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password catapult'".format(envroot),
                   user='rattail')
        elif env.theo_integrates_with == 'locsms':
            c.sudo("bash -c 'cd {} && bin/rattail -c app/quiet.conf --runas theo make-user --no-password locsms'".format(envroot),
                   user='rattail')

        # tweaks for mirror DB
        if env.theo_mirror_posdb and env.theo_integrates_with == 'catapult':
            postgresql.sql(c, "insert into setting values ('tailbone.engines.catapult.pretend_default', 'mirror')",
                           database=dbname)
            postgresql.sql(c, "insert into setting values ('rattail_onager.catapult.importing.db.default_models', 'VersionInfo Store Department Vendor Brand FamilyLine ComboGroupProfile WeightProfile PricingLevelInfo DiscountProfile StockInventory PricingLevelDetail Prefix Suffix DiscountGroup Customer Employee PhoneDescription Phone EmailDescription Email AddressDescription Address')",
                           database=dbname)
            postgresql.sql(c, "insert into setting values ('rattail.mail.rattail_onager_import_catapult_updates.subject', 'Changes for Catapult (production) -> Catapult (mirror)')",
                           database=dbname)

    # maybe establish the mirror DB
    if env.theo_mirror_posdb == 'mysql':
        if env.theo_integrates_with == 'catapult':
            if not mysql.db_exists(c, mirrordb):
                mysql.create_db(c, mirrordb, checkfirst=False, user='rattail@localhost')
                c.sudo("bash -c 'cd {} && bin/alembic -c app/catapult-mirror.conf upgrade heads'".format(envroot),
                       user='rattail')

    # supervisor
    deploy(c, 'theo-common/supervisor.conf.mako', '/etc/supervisor/conf.d/{}.conf'.format(safename),
           use_sudo=True, context={'envroot': envroot, 'safename': safename})
    c.sudo('supervisorctl update')
    c.sudo('supervisorctl start {}:'.format(safename))

    # sudoers, crontab, logrotate
    deploy.sudoers(c, 'theo-common/sudoers.mako', '/etc/sudoers.d/{}'.format(safename),
                   context={'envname': envname, 'envroot': envroot})
    deploy(c, 'theo-common/crontab.mako', '/etc/cron.d/{}'.format(safename),
           use_sudo=True,
           context={'env': env, 'envname': envname, 'envroot': envroot,
                    'pretty_time': run_overnight_at.strftime('%I:%M %p'),
                    'cron_time': run_overnight_at.strftime('%M %H')})
    deploy(c, 'theo-common/logrotate.conf.mako', '/etc/logrotate.d/{}'.format(safename),
           use_sudo=True, context={'envroot': envroot})


def install_theo_source(c, envroot):
    """
    Install source code for Theo
    """
    # rattail
    install_source_package(c, envroot, 'rattail')
    install_source_package(c, envroot, 'tailbone')

    # corepos
    if env.theo_integrates_with == 'corepos':
        install_source_package(c, envroot, 'pycorepos')
        install_source_package(c, envroot, 'rattail-corepos')
        install_source_package(c, envroot, 'tailbone-corepos')

    # catapult
    elif env.theo_integrates_with == 'catapult':
        install_source_package(c, envroot, 'onager', restricted=True)
        install_source_package(c, envroot, 'rattail-onager', restricted=True)
        install_source_package(c, envroot, 'tailbone-onager', restricted=True)

    # locsms
    elif env.theo_integrates_with == 'locsms':
        install_source_package(c, envroot, 'luckysmores', restricted=True)
        install_source_package(c, envroot, 'rattail-luckysmores', restricted=True)
        install_source_package(c, envroot, 'tailbone-locsms', restricted=True)

    # theo
    install_source_package(c, envroot, 'theo')


def install_source_package(c, envroot, name, restricted=False):
    if not exists(c, '{}/src/{}'.format(envroot, name)):
        if restricted:
            c.sudo('git clone https://{0}:{1}@kallithea.rattailproject.org/rattail-restricted/{2} {3}/src/{2}'.format(
                env.kallithea_username, env.kallithea_password, name, envroot),
                   user='rattail', echo=False)
        else:
            c.sudo('git clone https://kallithea.rattailproject.org/rattail-project/{0} {1}/src/{0}'.format(name, envroot),
                   user='rattail')
        c.sudo("bash -c 'PIP_CONFIG_FILE={0}/pip.conf cd {0} && bin/pip install -e src/{1}'".format(envroot, name),
               user='rattail')


##############################
# fabenv
##############################

try:
    import fabenv
except ImportError as error:
    print("\ncouldn't import fabenv: {}".format(error))

env.setdefault('machine_is_live', False)