feat: initial package, with apt and sync modules

.gitignore

@@ -0,0 +1,4 @@
+*~
+*.pyc
+.coverage
+docs/_build/

README.md

@@ -0,0 +1,6 @@
+
+# WuttaMess
+
+Fabric Automation Helpers
+
+See docs at https://rattailproject.org/docs/wuttamess/

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/api/wuttamess.apt.rst

@@ -0,0 +1,6 @@
+
+``wuttamess.apt``
+=================
+
+.. automodule:: wuttamess.apt
+   :members:

docs/api/wuttamess.rst

@@ -0,0 +1,6 @@
+
+``wuttamess``
+=============
+
+.. automodule:: wuttamess
+   :members:

docs/api/wuttamess.sync.rst

@@ -0,0 +1,6 @@
+
+``wuttamess.sync``
+==================
+
+.. automodule:: wuttamess.sync
+   :members:

docs/conf.py

@@ -0,0 +1,39 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+from importlib.metadata import version as get_version
+
+project = 'WuttaMess'
+copyright = '2024, Lance Edgar'
+author = 'Lance Edgar'
+release = get_version('WuttaMess')
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.todo',
+]
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+intersphinx_mapping = {
+    'fabsync': ('https://fabsync.ignorare.dev/', None),
+    'invoke': ('https://docs.pyinvoke.org/en/stable/', None),
+}
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'furo'
+html_static_path = ['_static']

docs/index.rst

@@ -0,0 +1,34 @@
+
+WuttaMess
+=========
+
+This package provides various helpers for use with `Fabric
+<https://www.fabfile.org>`_ automation.
+
+It can be used to deploy custom apps built with the `Wutta framework
+<https://wuttaproject.org>`_, but is not specific to that use case.
+It can also be used for general server setup etc.
+
+However it is only intended for use with Linux (and similar) for the
+target machine.
+
+Good documentation and 100% `test coverage`_ are priorities for this
+project.
+
+.. _test coverage: https://buildbot.rattailproject.org/coverage/wuttamess/
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Documentation:
+
+   narr/install
+   narr/usage
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Package API:
+
+   api/wuttamess
+   api/wuttamess.apt
+   api/wuttamess.sync

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/narr/install.rst

@@ -0,0 +1,14 @@
+
+Installation
+============
+
+Just the usual for a Python package:
+
+.. code-block:: sh
+
+   pip install WuttaMess
+
+Note that this will also install these dependencies:
+
+* `fabric <https://pypi.org/project/fabric/>`_
+* `fabsync <https://pypi.org/project/fabsync/>`_

docs/narr/usage.rst

@@ -0,0 +1,209 @@
+
+Usage
+=====
+
+The expected use case is as follows:
+
+Let's say you have a Linux machine "myserver" and you want to setup
+these software systems on it:
+
+* `Postfix <https://www.postfix.org/>`_
+* `PostgreSQL <https://www.postgresql.org/>`_
+* `collectd <https://www.collectd.org/>`_
+
+Create a folder to contain the ``fabfile.py`` etc.  Let's also assume
+you will have other machines to setup, and you want to commit all this
+to source control.
+
+Recommended project structure is like:
+
+.. code-block:: none
+
+   myproject
+   └── machines
+       └── myserver
+           ├── fabfile.py
+           ├── files
+           │   └── etc
+           │       ├── collectd
+           │       │   └── collectd.conf
+           │       └── postfix
+           │           └── main.cf
+           └── Vagrantfile
+
+More details on these below.
+
+
+.. _fabfile-example:
+
+``fabfile.py``
+--------------
+
+This is a "typical" fabfile, to the extent there is such a thing.
+This file contains Fabric "tasks" which may be executed on the target
+machine via SSH.  For more on that concept see
+:ref:`invoke:defining-and-running-task-functions`.
+
+In this example we define "bootstrap" tasks for the setup, but that is
+merely a personal convention.  You can define tasks however you need::
+
+   """
+   Fabric script for myserver
+   """
+
+   from fabric import task
+   from wuttamess import apt, sync
+
+
+   # nb. this is used below, for file sync
+   root = sync.make_root('files')
+
+
+   @task
+   def bootstrap_all(c):
+       """
+       Bootstrap all aspects of the server
+       """
+       bootstrap_base(c)
+       bootstrap_postgresql(c)
+       bootstrap_collectd(c)
+
+
+   @task
+   def bootstrap_base(c):
+       """
+       Bootstrap the base system
+       """
+       apt.dist_upgrade(c)
+
+       # postfix
+       apt.install(c, 'postfix')
+       if sync.check_isync(c, root, 'etc/postfix'):
+           c.run('systemctl restart postfix')
+
+
+   @task
+   def bootstrap_postgresql(c):
+       """
+       Bootstrap the PostgreSQL service
+       """
+       apt.install(c, 'postgresql', 'libpq-dev')
+
+
+   @task
+   def bootstrap_collectd(c):
+       """
+       Bootstrap the collectd service
+       """
+       apt.install(c, 'collectd')
+       if sync.check_isync(c, root, 'etc/collectd'):
+           c.run('systemctl restart collectd')
+
+Above you can see how WuttaMess is actually used; it simply provides
+convenience functions which can be called from a Fabric task.
+
+But `Fabric <https://www.fabfile.org>`_ (and `fabsync
+<https://fabsync.ignorare.dev/>`_ for file sync operations) are doing
+the heavy lifting.  The goal for WuttaMess is to further abstract
+common operations and keep the task logic as "clean" as possible.
+
+See also these functions which are used above:
+
+* :func:`wuttamess.apt.dist_upgrade()`
+* :func:`wuttamess.apt.install()`
+* :func:`wuttamess.sync.make_root()`
+* :func:`wuttamess.sync.check_isync()`
+
+
+``files``
+---------
+
+This folder contains all files which must be synced to the target
+machine as part of setup.  As shown in the example above, the
+``files`` structure should "mirror" the target machine file system.
+
+The :func:`~wuttamess.sync.check_isync()` function may be called with
+a "subpath" to sync just a portion of the file system.  It returns
+``True`` if any files were modified, so we can check for that and
+avoid restarting services if nothing changed.
+
+Note that in global module scope, we create the "root" object for use
+with file sync.  This is then passed to the various sync functions.
+
+This uses the ``fabsync`` library under the hood; for more on how that
+works see :doc:`fabsync:index`.
+
+
+``Vagrantfile``
+---------------
+
+This file is optional but may be useful for testing deployment on a
+local VM using `Vagrant <https://www.vagrantup.com/>`_.  For example:
+
+.. code-block:: ruby
+
+   Vagrant.configure("2") do |config|
+
+     # live machine runs Debian 12 "bookworm"
+     config.vm.box = "debian/bookworm64"
+
+   end
+
+For more info see docs for `Vagrantfile
+<https://developer.hashicorp.com/vagrant/docs/vagrantfile>`_.
+
+
+.. _running-tasks:
+
+Running Tasks via CLI
+---------------------
+
+With the above setup, first make sure you are in the right working
+directory (wherever ``fabfile.py`` lives):
+
+.. code-block:: sh
+
+   cd myproject/machines/myserver
+
+Then run whichever tasks you need, specifying the connection info for
+target machine like so:
+
+.. code-block:: sh
+
+   fab -e -H root@myserver.example.com bootstrap-all
+
+Fabric uses SSH to connect to the target machine
+(myserver.example.com) and runs the specified task on that machine.
+
+Testing with a Vagrant VM will likely require a more "complicated"
+command line.  See output from ``vagrant ssh-config`` for details
+specific to your VM, but the command may be something like:
+
+.. code-block:: sh
+
+   fab -e -H root@192.168.121.42 -i .vagrant/machines/default/libvirt/private_key bootstrap-all
+
+
+Troubleshooting SSH
+-------------------
+
+In some cases troubleshooting the SSH connection can be tricky.  A rule of
+thumb is to first make sure it works without Fabric.
+
+Try a basic connection with the same args using SSH only:
+
+.. code-block:: sh
+
+   ssh root@myserver.example.com
+
+Or for a Vagrant VM:
+
+.. code-block:: sh
+
+   ssh root@192.168.121.42 -i .vagrant/machines/default/libvirt/private_key
+
+You may want to edit your ``~/.ssh/config`` file as needed.  However
+this usually is done for "normal" machines only, not for Vagrant VM.
+
+Once that works, then the ``fab`` command *should* also work using the
+same args...

pyproject.toml

@@ -0,0 +1,57 @@
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+
+[project]
+name = "WuttaMess"
+version = "0.0.0"
+description = "Fabric Automation Helpers"
+readme = "README.md"
+authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]
+license = {text = "GNU GPL v3+"}
+classifiers = [
+        "Development Status :: 4 - Beta",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Natural Language :: English",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Topic :: System :: Systems Administration",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">= 3.8"
+dependencies = [
+        "fabric",
+        "fabsync",
+]
+
+
+[project.optional-dependencies]
+docs = ["Sphinx", "furo"]
+tests = ["pytest-cov", "tox"]
+
+
+[project.urls]
+Homepage = "https://wuttaproject.org/"
+Repository = "https://forgejo.wuttaproject.org/wutta/wuttamess"
+Changelog = "https://forgejo.wuttaproject.org/wutta/wuttamess/src/branch/master/CHANGELOG.md"
+
+
+[tool.commitizen]
+version_provider = "pep621"
+tag_format = "v$version"
+update_changelog_on_bump = true
+
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "htmlcov/",
+]

src/wuttamess/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  WuttaMess -- Fabric Automation Helpers
+#  Copyright © 2024 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+WuttaMess - base package
+"""
+
+from ._version import __version__

src/wuttamess/_version.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8; -*-
+
+from importlib.metadata import version
+
+
+__version__ = version('WuttaMess')

src/wuttamess/apt.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  WuttaMess -- Fabric Automation Helpers
+#  Copyright © 2024 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+APT package management
+"""
+
+
+def dist_upgrade(c, frontend='noninteractive'):
+    """
+    Run a full dist-upgrade for APT.  Essentially this runs:
+
+    .. code-block:: sh
+
+       apt update
+       apt dist-upgrade
+    """
+    update(c)
+    upgrade(c, dist_upgrade=True, frontend=frontend)
+
+
+def install(c, *packages, **kwargs):
+    """
+    Install some package(s) via APT.  Essentially this runs:
+
+    .. code-block:: sh
+
+       apt install PKG [PKG ...]
+    """
+    frontend = kwargs.pop('frontend', 'noninteractive')
+    packages = ' '.join(packages)
+    return c.run(f'DEBIAN_FRONTEND={frontend} apt-get --assume-yes install {packages}')
+
+
+def update(c):
+    """
+    Update the APT package lists.  Essentially this runs:
+
+    .. code-block:: sh
+
+       apt update
+    """
+    c.run('apt-get update')
+
+
+def upgrade(c, dist_upgrade=False, frontend='noninteractive'):
+    """
+    Upgrade packages via APT.  Essentially this runs:
+
+    .. code-block:: sh
+
+       apt upgrade
+
+       # ..or..
+
+       apt dist-upgrade
+    """
+    options = ''
+    if frontend == 'noninteractive':
+        options = '--option Dpkg::Options::="--force-confdef" --option Dpkg::Options::="--force-confold"'
+    upgrade = 'dist-upgrade' if dist_upgrade else 'upgrade'
+    c.run(f'DEBIAN_FRONTEND={frontend} apt-get --assume-yes {options} {upgrade}')

src/wuttamess/sync.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  WuttaMess -- Fabric Automation Helpers
+#  Copyright © 2024 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Synchronize Files
+
+See :doc:`/narr/usage` for a basic example.
+"""
+
+import fabsync
+
+
+def make_root(path, dest='/'):
+    """
+    Make and return a "root" object for use with future sync calls.
+
+    This is a convenience wrapper around
+    :func:`fabsync:fabsync.load()`.
+
+    :param path: Path to local file tree.  Usually this is relative to
+       the ``fabfile.py`` location, otherwise should be absolute.
+
+    :param dest: Path for target file tree.
+    """
+    return fabsync.load(path, dest)
+
+
+def isync(c, root, selector=None, echo=True, **kwargs):
+    """
+    Sync files, yielding the result for each as it goes.
+
+    This is a convenience wrapper around
+    :func:`fabsync:fabsync.isync()`.
+
+    :param c: Connection object.
+
+    :param root: File tree "root" object as obtained from
+       :func:`make_root()`.
+
+    :param selector: This can be a simple "subpath" string, indicating
+       a section of the file tree.  For instance: ``'etc/postfix'``
+
+    :param echo: Flag indicating whether the path for each file synced
+       should be echoed to stdout.  Generally thought to be useful but
+       may be disabled.
+
+    :param \**kwargs: Any remaining kwargs are passed as-is to
+       :func:`fabsync:fabsync.isync()`.
+    """
+    if selector:
+        if not isinstance(selector, fabsync.ItemSelector):
+            selector = fabsync.ItemSelector.new(selector)
+        kwargs['selector'] = selector
+
+    for result in fabsync.isync(c, root, **kwargs):
+        if echo:
+            print(f"{result.path}{' [modified]' if result.modified else ''}")
+        yield result
+
+
+def check_isync(c, root, selector=None, **kwargs):
+    """
+    Sync all files and return boolean indicating whether any actual
+    modifications were made.
+
+    Arguments are the same as for :func:`isync()`, which this calls.
+
+    :returns: ``True`` if any sync result indicates a file was
+       modified; otherwise ``False``.
+    """
+    return any([result.modified
+                for result in isync(c, root, selector, **kwargs)])

tests/files/foo

@@ -0,0 +1 @@
+foo

tests/test_apt.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8; -*-
+
+from unittest import TestCase
+from unittest.mock import patch, MagicMock
+
+from wuttamess import apt as mod
+
+
+class TestDistUpgrade(TestCase):
+
+    def test_basic(self):
+        c = MagicMock()
+        with patch.object(mod, 'update') as update:
+            with patch.object(mod, 'upgrade') as upgrade:
+                mod.dist_upgrade(c, frontend='whatever')
+                update.assert_called_once_with(c)
+                upgrade.assert_called_once_with(c, dist_upgrade=True, frontend='whatever')
+
+
+class TestInstall(TestCase):
+
+    def test_basic(self):
+        c = MagicMock()
+        mod.install(c, 'postfix')
+        c.run.assert_called_once_with('DEBIAN_FRONTEND=noninteractive apt-get --assume-yes install postfix')
+
+
+class TestUpdate(TestCase):
+
+    def test_basic(self):
+        c = MagicMock()
+        mod.update(c)
+        c.run.assert_called_once_with('apt-get update')
+
+
+class TestUpgrade(TestCase):
+
+    def test_basic(self):
+        c = MagicMock()
+        mod.upgrade(c)
+        c.run.assert_called_once_with('DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --option Dpkg::Options::="--force-confdef" --option Dpkg::Options::="--force-confold" upgrade')

tests/test_sync.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8; -*-
+
+from pathlib import Path
+from unittest import TestCase
+from unittest.mock import patch, MagicMock
+
+from fabsync import SyncedRoot, ItemSelector
+
+from wuttamess import sync as mod
+
+
+class TestMakeRoot(TestCase):
+
+    def test_basic(self):
+        root = mod.make_root('files')
+        self.assertIsInstance(root, SyncedRoot)
+        self.assertEqual(root.src, Path('files'))
+        self.assertEqual(root.dest, Path('/'))
+
+
+class TestIsync(TestCase):
+
+    def test_basic(self):
+        c = MagicMock()
+        root = mod.make_root('files')
+        with patch.object(mod, 'fabsync') as fabsync:
+            fabsync.ItemSelector = ItemSelector
+
+            # nothing to sync
+            fabsync.isync.return_value = []
+            results = list(mod.isync(c, root))
+            self.assertEqual(results, [])
+            fabsync.isync.assert_called_once_with(c, root)
+
+            # sync one file
+            fabsync.isync.reset_mock()
+            result = MagicMock(path='/foo', modified=True)
+            fabsync.isync.return_value = [result]
+            results = list(mod.isync(c, root))
+            self.assertEqual(results, [result])
+            fabsync.isync.assert_called_once_with(c, root)
+
+            # sync with selector
+            fabsync.isync.reset_mock()
+            result = MagicMock(path='/foo', modified=True)
+            fabsync.isync.return_value = [result]
+            results = list(mod.isync(c, root, 'foo'))
+            self.assertEqual(results, [result])
+            fabsync.isync.assert_called_once_with(c, root, selector=fabsync.ItemSelector.new('foo'))
+
+
+class TestCheckIsync(TestCase):
+
+    def test_basic(self):
+        c = MagicMock()
+        root = mod.make_root('files')
+        with patch.object(mod, 'isync') as isync:
+
+            # file(s) modified
+            result = MagicMock(path='/foo', modified=True)
+            isync.return_value = [result]
+            self.assertTrue(mod.check_isync(c, root))
+
+            # not modified
+            result = MagicMock(path='/foo', modified=False)
+            isync.return_value = [result]
+            self.assertFalse(mod.check_isync(c, root))

tox.ini

@@ -0,0 +1,17 @@
+
+[tox]
+envlist = py38, py39, py310, py311
+
+[testenv]
+extras = tests
+commands = pytest {posargs}
+
+[testenv:coverage]
+basepython = python3.11
+commands = pytest --cov=wuttamess --cov-report=html --cov-fail-under=100
+
+[testenv:docs]
+basepython = python3.11
+extras = docs
+changedir = docs
+commands = sphinx-build -b html -d {envtmpdir}/doctrees -W -T . {envtmpdir}/docs