diff --git a/.gitignore b/.gitignore index c16e5b6..e482cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +*~ *.pyc rattail_fabric.egg-info/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..5893279 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ + +# rattail-fabric + +Rattail is a retail software framework, released under the GNU General Public +License. + +This package contains various utility functions for use with +[Fabric](http://www.fabfile.org/). + +Please see Rattail's [home page](https://rattailproject.org/) for more +information. diff --git a/README.rst b/README.rst deleted file mode 100644 index 6622b69..0000000 --- a/README.rst +++ /dev/null @@ -1,14 +0,0 @@ - -rattail-fabric -============== - -Rattail is a retail software framework, released under the GNU General Public -License. - -This package contains various utility functions for use with `Fabric`_. - -.. _`Fabric`: http://www.fabfile.org/ - -Please see Rattail's `home page`_ for more information. - -.. _`home page`: https://rattailproject.org/ diff --git a/rattail_fabric/apache.py b/rattail_fabric/apache.py index b9bf70a..8061271 100644 --- a/rattail_fabric/apache.py +++ b/rattail_fabric/apache.py @@ -51,6 +51,17 @@ def get_version(): return float(match.group(1)) +def get_php_version(): + """ + Fetch the version of PHP running on the target system + """ + result = sudo('php --version') + if result.succeeded: + match = re.match(r'^PHP (\d+\.\d+)\.\d+-', result) + if match: + return float(match.group(1)) + + def install_wsgi(python_home=None, python3=False): """ Install the mod_wsgi Apache module, with optional ``WSGIPythonHome`` value. diff --git a/rattail_fabric/backup.py b/rattail_fabric/backup.py index c73bf05..07e6aec 100644 --- a/rattail_fabric/backup.py +++ b/rattail_fabric/backup.py @@ -37,6 +37,15 @@ from rattail_fabric import make_deploy, mkdir, python, UNSPECIFIED deploy_generic = make_deploy(__file__) +def deploy_rattail_backup_script(**context): + """ + Deploy the generic `rattail-backup` script + """ + context.setdefault('envname', 'backup') + deploy_generic('backup/rattail-backup.mako', '/usr/local/bin/rattail-backup', + mode='0700', context=context) + + def deploy_backup_everything(**context): """ Deploy the generic `backup-everything` script @@ -48,28 +57,35 @@ def deploy_backup_everything(**context): def deploy_backup_app(deploy, envname, mkvirtualenv=True, user='rattail', + python_exe='/usr/bin/python3', + rattail_backup_script=None, config=None, everything=None, crontab=None, runat=UNSPECIFIED): """ Make an app which can run backups for the server. """ if not config: - path = '{}/rattail.conf'.format(envname) + path = '{}/rattail.conf.mako'.format(envname) if deploy.local_exists(path): config = path else: - raise ValueError("Must provide config path for backup app") + path = '{}/rattail.conf'.format(envname) + if deploy.local_exists(path): + config = path + else: + raise ValueError("Must provide config path for backup app") if runat is UNSPECIFIED: runat = datetime.time(0) # defaults to midnight # virtualenv - if mkvirtualenv: - python.mkvirtualenv(envname, python='/usr/bin/python3', upgrade_setuptools=False) envpath = '/srv/envs/{}'.format(envname) + if mkvirtualenv and not exists(envpath): + python.mkvirtualenv(envname, python=python_exe, upgrade_setuptools=False, + runas_user=user) sudo('chown -R {}: {}'.format(user, envpath)) with cd(envpath): mkdir('src', owner=user) - sudo('bin/pip install --upgrade pip', user=user) + sudo('bin/pip install --upgrade pip setuptools wheel', user=user) # rattail if not exists('src/rattail'): @@ -81,10 +97,21 @@ def deploy_backup_app(deploy, envname, mkvirtualenv=True, user='rattail', # config sudo('bin/rattail make-appdir', user=user) - deploy(config, 'app/rattail.conf', owner=user, mode='0600') + config_context = { + 'user': user, + } + deploy(config, 'app/rattail.conf', owner=user, mode='0600', context=config_context) sudo('bin/rattail -c app/rattail.conf make-config -T quiet -O app/', user=user) sudo('bin/rattail -c app/rattail.conf make-config -T silent -O app/', user=user) + # rattail-backup script + script_context = {'envname': envname} + if rattail_backup_script: + deploy(rattail_backup_script, '/usr/local/bin/rattail-backup', mode='0700', + context=script_context) + else: + deploy_rattail_backup_script(**script_context) + # backup-everything script everything_context = { 'envname': envname, diff --git a/rattail_fabric/core.py b/rattail_fabric/core.py index f409f15..0cf8a9e 100644 --- a/rattail_fabric/core.py +++ b/rattail_fabric/core.py @@ -137,10 +137,14 @@ def upload_mako_template(local_path, remote_path, context={}, encoding='utf_8', """ template = Template(filename=local_path) + # make copy of context; add env to it + context = dict(context) + context['env'] = env + temp_dir = tempfile.mkdtemp(prefix='rattail-fabric.') temp_path = os.path.join(temp_dir, os.path.basename(local_path)) with open(temp_path, 'wb') as f: - text = template.render(env=env, **context) + text = template.render(**context) f.write(text.encode(encoding)) os.chmod(temp_path, os.stat(local_path).st_mode) diff --git a/rattail_fabric/deploy/backup/backup-everything.mako b/rattail_fabric/deploy/backup/backup-everything.mako index 0f37d8b..cd30941 100755 --- a/rattail_fabric/deploy/backup/backup-everything.mako +++ b/rattail_fabric/deploy/backup/backup-everything.mako @@ -26,7 +26,7 @@ if [ "$(sudo -u ${user} git status --porcelain)" != '' ]; then exit 1 fi sudo -u ${user} git pull $QUIET -sudo -u ${user} find . -name '*.pyc' -delete +sudo find . -name '*.pyc' -delete $PIP install $QUIET --upgrade --upgrade-strategy eager --editable . # run backup diff --git a/rattail_fabric/deploy/backup/crontab.mako b/rattail_fabric/deploy/backup/crontab.mako index 43b10e5..537b128 100644 --- a/rattail_fabric/deploy/backup/crontab.mako +++ b/rattail_fabric/deploy/backup/crontab.mako @@ -1,4 +1,11 @@ # -*- mode: conf; -*- # backup everything of importance at ${pretty_time} +% if hasattr(env, 'machine_is_live'): +${'' if env.machine_is_live else '# '}${cron_time} * * * root /usr/local/bin/backup-everything +## TODO: should somehow deprecate / remove this? +% elif hasattr(env, 'server_is_live'): ${'' if env.server_is_live else '# '}${cron_time} * * * root /usr/local/bin/backup-everything +% else: +# ${cron_time} * * * root /usr/local/bin/backup-everything +% endif diff --git a/rattail_fabric/deploy/backup/rattail-backup.mako b/rattail_fabric/deploy/backup/rattail-backup.mako new file mode 100755 index 0000000..f88b6f0 --- /dev/null +++ b/rattail_fabric/deploy/backup/rattail-backup.mako @@ -0,0 +1,13 @@ +#!/bin/sh -e + +if [ "$1" = "-v" -o "$1" = "--verbose" ]; then + VERBOSE='--verbose' + CONFIG='/srv/envs/${envname}/app/rattail.conf' +else + VERBOSE= + CONFIG='/srv/envs/${envname}/app/silent.conf' +fi + +cd /srv/envs/${envname} + +bin/rattail -c $CONFIG $VERBOSE backup $* diff --git a/rattail_fabric/deploy/check-supervisor-process b/rattail_fabric/deploy/check-supervisor-process new file mode 100755 index 0000000..8a7323a --- /dev/null +++ b/rattail_fabric/deploy/check-supervisor-process @@ -0,0 +1,57 @@ +#!/bin/sh +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2018 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +# +# This is a simple script which will output a single line of info, and exit +# with a return code which indicates status of a given supervisor process. It +# was designed for use with Shinken monitoring. Invoke like so: +# +# check-supervisor-process NAME +# +# Where NAME refers to the process, e.g. 'rattail:datasync'. Exit code may be: +# +# * 0 = process is confirmed running +# * 2 = process is confirmed *not* running +# * 3 = unknown state +# +################################################################################ + +NAME="$1" +if [ "$NAME" = "" ]; then + echo "Usage: check-supervisor-process NAME" + exit 3 +fi + +STATUS=$(sudo supervisorctl status $NAME | awk -F ' +' '{print $2}') + +if [ $STATUS = "RUNNING" ]; then + echo "supervisor reports RUNNING status" + exit 0 +fi + +if [ "$STATUS" = "" ]; then + echo "unable to perform status check!" + exit 3 +fi + +echo "supervisor reports $STATUS status" +exit 2 diff --git a/rattail_fabric/freetds.py b/rattail_fabric/freetds.py index 74ba6b6..acca7a3 100644 --- a/rattail_fabric/freetds.py +++ b/rattail_fabric/freetds.py @@ -43,6 +43,7 @@ def install_from_source(user='rattail'): 'automake', 'autoconf', 'gettext', + 'gperf', 'pkg-config', ) diff --git a/rattail_fabric/mssql.py b/rattail_fabric/mssql.py new file mode 100644 index 0000000..5fab96b --- /dev/null +++ b/rattail_fabric/mssql.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2019 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Fabric Library for MS SQL Server +""" + +from __future__ import unicode_literals, absolute_import + +from fabric.api import sudo + +from rattail_fabric import apt + + +def install_mssql_odbc(): + """ + Install the MS SQL Server ODBC driver + + https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-2017 + """ + apt.install('apt-transport-https') + sudo('curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -') + sudo('curl https://packages.microsoft.com/config/debian/9/prod.list > /etc/apt/sources.list.d/mssql-release.list') + apt.update() + sudo('ACCEPT_EULA=Y apt-get --assume-yes install msodbcsql17') diff --git a/rattail_fabric/nodejs.py b/rattail_fabric/nodejs.py new file mode 100644 index 0000000..2a74b3d --- /dev/null +++ b/rattail_fabric/nodejs.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2019 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Fabric library for Node.js +""" + +import os + +from fabric.api import sudo, run +from fabric.contrib.files import append, exists + +from rattail_fabric.util import get_home_path + + +def install(version=None, user=None): + """ + Install nvm and node.js for given user, or else "connection" user. + """ + home = get_home_path(user) + nvm = os.path.join(home, '.nvm') + + if not exists(nvm): + cmd = "bash -c 'curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.5/install.sh | bash'" + if user: + sudo(cmd, user=user) + else: + run(cmd) + + profile = os.path.join(home, '.profile') + kwargs = {'use_sudo': bool(user)} + append(profile, 'export NVM_DIR="{}"'.format(nvm), **kwargs) + append(profile, '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"', **kwargs) + append(profile, '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"', **kwargs) + + node = version or 'node' + cmd = "bash -l -c 'nvm install {}'".format(node) + if user: + sudo(cmd, user=user) + else: + run(cmd) diff --git a/rattail_fabric/postgresql.py b/rattail_fabric/postgresql.py index ef9ba06..7214d5a 100644 --- a/rattail_fabric/postgresql.py +++ b/rattail_fabric/postgresql.py @@ -56,10 +56,10 @@ def sql(sql, database='', port=None): """ Execute some SQL as the 'postgres' user. """ - cmd = 'sudo -u postgres psql {port} --tuples-only --no-align --command="{sql}" {database}'.format( + cmd = 'psql {port} --tuples-only --no-align --command="{sql}" {database}'.format( port='--port={}'.format(port) if port else '', sql=sql, database=database) - return sudo(cmd, shell=False) + return sudo(cmd, user='postgres', shell=False) def script(path, database='', port=None, user=None, password=None): @@ -75,7 +75,7 @@ def script(path, database='', port=None, user=None, password=None): else: # run as postgres kw = dict(port=port, path=path, db=database) - return sudo("sudo -u postgres psql {port} --file='{path}' {db}".format(**kw), shell=False) + return sudo("psql {port} --file='{path}' {db}".format(**kw), user='postgres', shell=False) def user_exists(name, port=None): @@ -91,10 +91,11 @@ def create_user(name, password=None, port=None, checkfirst=True, createdb=False) Create a PostgreSQL user account. """ if not checkfirst or not user_exists(name, port=port): - sudo('sudo -u postgres createuser {port} {createdb} --no-createrole --no-superuser {name}'.format( + cmd = 'createuser {port} {createdb} --no-createrole --no-superuser {name}'.format( port='--port={}'.format(port) if port else '', createdb='--{}createdb'.format('' if createdb else 'no-'), - name=name)) + name=name) + sudo(cmd, user='postgres', shell=False) if password: set_user_password(name, password, port=port) @@ -120,11 +121,11 @@ def create_db(name, owner=None, port=None, checkfirst=True): Create a PostgreSQL database. """ if not checkfirst or not db_exists(name, port=port): - cmd = 'sudo -u postgres createdb {port} {owner} {name}'.format( + cmd = 'createdb {port} {owner} {name}'.format( port='--port={}'.format(port) if port else '', owner='--owner={}'.format(owner) if owner else '', name=name) - sudo(cmd, shell=False) + sudo(cmd, user='postgres', shell=False) def create_schema(name, dbname, owner='rattail', port=None): @@ -140,7 +141,7 @@ def drop_db(name, checkfirst=True): Drop a PostgreSQL database. """ if not checkfirst or db_exists(name): - sudo('sudo -u postgres dropdb {0}'.format(name), shell=False) + sudo('dropdb {}'.format(name), user='postgres', shell=False) def download_db(name, destination=None, port=None, exclude_tables=None): @@ -151,11 +152,11 @@ def download_db(name, destination=None, port=None, exclude_tables=None): destination = './{0}.sql.gz'.format(name) run('touch {0}.sql'.format(name)) run('chmod 0666 {0}.sql'.format(name)) - sudo('sudo -u postgres pg_dump {port} {exclude_tables} --file={name}.sql {name}'.format( + cmd = 'pg_dump {port} {exclude_tables} --file={name}.sql {name}'.format( name=name, port='--port={}'.format(port) if port else '', - exclude_tables='--exclude-table-data={}'.format(exclude_tables) if exclude_tables else '', - ), shell=False) + exclude_tables='--exclude-table-data={}'.format(exclude_tables) if exclude_tables else '') + sudo(cmd, user='postgres', shell=False) run('gzip --force {0}.sql'.format(name)) get('{0}.sql.gz'.format(name), destination) run('rm {0}.sql.gz'.format(name)) @@ -192,5 +193,5 @@ def clone_db(name, owner, download, user='rattail', force=False, workdir=None): # restore database on target server run('gunzip --force {}.sql.gz'.format(name)) - sudo('sudo -u postgres psql --echo-errors --file={0}.sql {0}'.format(name), shell=False) + sudo('psql --echo-errors --file={0}.sql {0}'.format(name), user='postgres', shell=False) run('rm {}.sql'.format(name)) diff --git a/rattail_fabric/python.py b/rattail_fabric/python.py index 5dbecb8..4fc1b3a 100644 --- a/rattail_fabric/python.py +++ b/rattail_fabric/python.py @@ -37,6 +37,32 @@ from fabric.contrib.files import exists, append from rattail_fabric import apt, mkdir +def install_pythonz(): + """ + Install the 'pythonz' utility, for installing arbitrary versions of python. + + Note that this uses 'curl' so that should already be installed. + + https://github.com/saghul/pythonz/blob/master/README.rst#installation + """ + if not exists('/usr/local/pythonz'): + if not exists('/usr/local/src/pythonz'): + mkdir('/usr/local/src/pythonz') + if not exists('/usr/local/src/pythonz/pythonz-install'): + sudo('curl -kL -o /usr/local/src/pythonz/pythonz-install https://raw.github.com/saghul/pythonz/master/pythonz-install') + sudo('chmod +x /usr/local/src/pythonz/pythonz-install') + sudo('/usr/local/src/pythonz/pythonz-install') + + +def install_python(version, verbose=False): + """ + Install a specific version of python, via pythonz. + """ + if not exists('/usr/local/pythonz/pythons/CPython-{}'.format(version)): + verbose = '--verbose' if verbose else '' + sudo('pythonz install {} {}'.format(verbose, version)) + + def install_pip(use_apt=False, eager=True): """ Install/upgrade the Pip installer for Python. diff --git a/rattail_fabric/rattail.py b/rattail_fabric/rattail.py index a6ba9dc..66a56e9 100644 --- a/rattail_fabric/rattail.py +++ b/rattail_fabric/rattail.py @@ -51,6 +51,7 @@ def bootstrap_rattail(home='/var/lib/rattail', uid=None, shell='/bin/bash'): mkdir('/srv/rattail/init') deploy('daemon', '/srv/rattail/init/daemon') deploy('check-rattail-daemon', '/usr/local/bin/check-rattail-daemon') + deploy('check-supervisor-process', '/usr/local/bin/check-supervisor-process', mode='0755') deploy('luigid', '/srv/rattail/init/luigid') deploy('soffice', '/srv/rattail/init/soffice') # TODO: deprecate / remove these diff --git a/rattail_fabric/ssh.py b/rattail_fabric/ssh.py index feaab86..cc8d4e5 100644 --- a/rattail_fabric/ssh.py +++ b/rattail_fabric/ssh.py @@ -29,7 +29,7 @@ from __future__ import unicode_literals, absolute_import import warnings from fabric.api import sudo, cd, settings -from fabric.contrib.files import exists, sed, append +from fabric.contrib.files import exists, sed from rattail_fabric import mkdir, agent_sudo from rattail_fabric.python import cdvirtualenv @@ -70,15 +70,14 @@ def configure(allow_root=False): """ Configure the OpenSSH service """ - path = '/etc/ssh/sshd_config' + # PermitRootLogin no (or without-password) + value = 'without-password' if allow_root else 'no' + sed('/etc/ssh/sshd_config', r'^#?PermitRootLogin .*', 'PermitRootLogin {}'.format(value), use_sudo=True) + sed('/etc/ssh/sshd_config', r'^PermitRootLogin .*', 'PermitRootLogin {}'.format(value), use_sudo=True) - entry = 'PermitRootLogin {}'.format('without-password' if allow_root else 'no') - sed(path, r'^PermitRootLogin\s+.*', entry, use_sudo=True) - append(path, entry, use_sudo=True) - - entry = 'PasswordAuthentication no' - sed(path, r'^PasswordAuthentication\s+.*', entry, use_sudo=True) - append(path, entry, use_sudo=True) + # PasswordAuthentication no + sed('/etc/ssh/sshd_config', r'^#?PasswordAuthentication .*', 'PasswordAuthentication no', use_sudo=True) + sed('/etc/ssh/sshd_config', r'^PasswordAuthentication .*', 'PasswordAuthentication no', use_sudo=True) restart() diff --git a/rattail_fabric/util.py b/rattail_fabric/util.py new file mode 100644 index 0000000..28331a6 --- /dev/null +++ b/rattail_fabric/util.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2019 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Misc. Utilities +""" + +from fabric.api import env, run + + +def get_home_path(user=None): + """ + Retrieve the path to the home folder for the given user, or else the + "connection" user. + """ + user = user or env.user + home = run('getent passwd {} | cut -d: -f6'.format(user)).stdout.strip() + home = home.rstrip('/') + return home diff --git a/setup.py b/setup.py index cfa12d9..21c8182 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) exec(open(os.path.join(here, 'rattail_fabric', '_version.py')).read()) -README = open(os.path.join(here, 'README.rst')).read() +README = open(os.path.join(here, 'README.md')).read() requires = [ @@ -61,7 +61,7 @@ requires = [ # # package # low high - 'Fabric', # 1.14.0 + 'Fabric<2.0', # 1.14.0 'invoke', # 0.22.1 ] @@ -73,7 +73,7 @@ setup( author_email = "lance@edbob.org", url = "https://rattailproject.org/", license = "GNU GPL v3", - description = "Fabric Utilities for Rattail", + description = "Fabric (v1) Utilities for Rattail", long_description = README, classifiers = [