From 98d696137008ffbb3698e3fd714df7465ab5e66f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 7 Jan 2021 16:48:14 -0600 Subject: [PATCH] Add most of the structure! plus several Base Layer docs --- .gitignore | 1 + docs/backup/borg.rst | 5 ++ docs/backup/index.rst | 10 +++ docs/backup/overview.rst | 5 ++ docs/base/appdir.rst | 136 +++++++++++++++++++++++++++++++ docs/base/commands.rst | 23 ++++++ docs/base/config/db.rst | 86 +++++++++++++++++++ docs/base/config/defined.rst | 11 +++ docs/base/config/generate.rst | 21 +++++ docs/base/config/index.rst | 16 ++++ docs/base/config/inheritance.rst | 72 ++++++++++++++++ docs/base/config/logging.rst | 32 ++++++++ docs/base/config/overview.rst | 31 +++++++ docs/base/config/paths.rst | 67 +++++++++++++++ docs/base/config/syntax.rst | 59 ++++++++++++++ docs/base/email.rst | 53 ++++++++++++ docs/base/filemon.rst | 5 ++ docs/base/handlers.rst | 5 ++ docs/base/index.rst | 19 +++++ docs/base/install.rst | 49 +++++++++++ docs/base/reqs.rst | 14 ++++ docs/base/scripts.rst | 17 ++++ docs/base/supervisor.rst | 11 +++ docs/base/venv.rst | 121 +++++++++++++++++++++++++++ docs/conf.py | 4 +- docs/data/batches.rst | 5 ++ docs/data/datasync.rst | 5 ++ docs/data/db.rst | 5 ++ docs/data/importing.rst | 5 ++ docs/data/index.rst | 18 ++++ docs/data/multinode.rst | 5 ++ docs/data/other.rst | 5 ++ docs/data/patterns.rst | 5 ++ docs/data/tasks.rst | 5 ++ docs/data/tempmon.rst | 5 ++ docs/data/trainwreck.rst | 5 ++ docs/deploy/fabric.rst | 5 ++ docs/deploy/index.rst | 10 +++ docs/deploy/overview.rst | 5 ++ docs/index.rst | 35 ++++++-- docs/luigi/index.rst | 10 +++ docs/luigi/overview.rst | 5 ++ docs/readfirst.rst | 16 ++++ docs/web/index.rst | 13 +++ docs/web/overview.rst | 5 ++ docs/web/spa.rst | 5 ++ docs/web/webapi.rst | 17 ++++ docs/web/webmain.rst | 17 ++++ 48 files changed, 1076 insertions(+), 8 deletions(-) create mode 100644 .gitignore create mode 100644 docs/backup/borg.rst create mode 100644 docs/backup/index.rst create mode 100644 docs/backup/overview.rst create mode 100644 docs/base/appdir.rst create mode 100644 docs/base/commands.rst create mode 100644 docs/base/config/db.rst create mode 100644 docs/base/config/defined.rst create mode 100644 docs/base/config/generate.rst create mode 100644 docs/base/config/index.rst create mode 100644 docs/base/config/inheritance.rst create mode 100644 docs/base/config/logging.rst create mode 100644 docs/base/config/overview.rst create mode 100644 docs/base/config/paths.rst create mode 100644 docs/base/config/syntax.rst create mode 100644 docs/base/email.rst create mode 100644 docs/base/filemon.rst create mode 100644 docs/base/handlers.rst create mode 100644 docs/base/index.rst create mode 100644 docs/base/install.rst create mode 100644 docs/base/reqs.rst create mode 100644 docs/base/scripts.rst create mode 100644 docs/base/supervisor.rst create mode 100644 docs/base/venv.rst create mode 100644 docs/data/batches.rst create mode 100644 docs/data/datasync.rst create mode 100644 docs/data/db.rst create mode 100644 docs/data/importing.rst create mode 100644 docs/data/index.rst create mode 100644 docs/data/multinode.rst create mode 100644 docs/data/other.rst create mode 100644 docs/data/patterns.rst create mode 100644 docs/data/tasks.rst create mode 100644 docs/data/tempmon.rst create mode 100644 docs/data/trainwreck.rst create mode 100644 docs/deploy/fabric.rst create mode 100644 docs/deploy/index.rst create mode 100644 docs/deploy/overview.rst create mode 100644 docs/luigi/index.rst create mode 100644 docs/luigi/overview.rst create mode 100644 docs/readfirst.rst create mode 100644 docs/web/index.rst create mode 100644 docs/web/overview.rst create mode 100644 docs/web/spa.rst create mode 100644 docs/web/webapi.rst create mode 100644 docs/web/webmain.rst diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f0ed21 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docs/_build/ diff --git a/docs/backup/borg.rst b/docs/backup/borg.rst new file mode 100644 index 0000000..7d1e855 --- /dev/null +++ b/docs/backup/borg.rst @@ -0,0 +1,5 @@ + +Borg +==== + +TODO diff --git a/docs/backup/index.rst b/docs/backup/index.rst new file mode 100644 index 0000000..6c1a27d --- /dev/null +++ b/docs/backup/index.rst @@ -0,0 +1,10 @@ + +Backup Layer +============ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + overview + borg diff --git a/docs/backup/overview.rst b/docs/backup/overview.rst new file mode 100644 index 0000000..45595c5 --- /dev/null +++ b/docs/backup/overview.rst @@ -0,0 +1,5 @@ + +Overview +======== + +TODO diff --git a/docs/base/appdir.rst b/docs/base/appdir.rst new file mode 100644 index 0000000..d2550b5 --- /dev/null +++ b/docs/base/appdir.rst @@ -0,0 +1,136 @@ + +.. highlight:: sh + +App Folder +========== + +Here we describe the "app folder" (aka. "app dir"), for instance at +``/srv/envs/poser/app``, which contains all config and data for the app. + +Rationale +--------- + +We'll use a typical Linux example here. Let's say we have a "poser" virtual +environment in the default location. By default the folder structure within +the virtual environment may look something like this: + +.. code-block:: none + + /srv/envs/poser + ├── bin + ├── include + ├── lib + │   └── python3.8 + ├── man + └── share + +Once the Python package(s) for our app have been installed to the virtual +environment, we will create an "app" folder in the environment root. This app +folder will contain all config, data and log files for the app. + +The idea here is that a proper (minimal) backup for the app, should only +*require* this "app dir" but the other virtual environment folders need not be +included in the backup (although they still can be, of course). + + +Making the App Dir +------------------ + +This may be done with or without activating your virtual environment. + +If your env is activated:: + + rattail make-appdir + +If your env is *not* activated:: + + cd /srv/envs/poser + bin/rattail make-appdir + + +Structure +--------- + +Keep in mind that the following is just an example. Each app is unique and +many of the folders described below can live elsewhere as long as the app is +configured appropriately. But this example is "typical" so its conventions +should be used unless there is a good reason not to. + +For the sake of completeness here we will assume a fairly robust app, which +uses several possible features and has been running a while. Here is what the +final virtual environment may look like: + +.. code-block:: none + + /srv/envs/poser + ├── app + │   ├── data + │   │   ├── batches + │   │   │   ├── inventory + │   │   │   ├── vendor_catalog + │   │   │   └── vendor_invoice + │   │   ├── exports + │   │   │   ├── instacart_product_export + │   │   │   └── report_output + │   │   ├── templates + │   │   ├── upgrades + │   │   └── uploads + │   ├── log + │   ├── luigi + │   │   ├── 2020 + │   │   ├── 2021 + │   │   └── log + │   ├── luigitasks + │   ├── sessions + │   │   ├── data + │   │   └── lock + │   └── work + │   ├── csv + │   ├── generated-projects + │   └── user-files + ├── bin + ├── include + ├── lib + │   └── python3.8 + ├── man + └── share + +First you may note that the above does not include a "config" folder. All +config files, and most scripts, will generally live directly within the "app" +folder itself. + +And now for a rundown of the "important" folders you do see above: + +``app/data`` ideally contains all "true" data for the app. What we mean by +that is, any data which the app requires to be 100% fully functional. Much of +this data is generated by the app itself, but should remain available for later +display within the app. + +``app/data/batches`` contains all data files used as input, or generated as +output, for various "batches" within the app. + +``app/data/exports`` contains all "export" files generated by the app, +e.g. when certain reports are ran etc. + +``app/data/templates`` may contain certain template files needed by the app +when it is generating various kinds of output. + +``app/data/upgrades`` contains the before and after package snapshots, and +command output etc. when an app upgrade is executed. + +``app/data/uploads`` is more of a temp folder, used to store files uploaded by +a user within the web app, until they are further processed. + +``app/log`` is where all log files go for the app proper. + +``app/luigi`` is where Luigi (if used) will keep track of which individual +tasks have already been ran for a given date. + +``app/luigi/log`` will contain log files for both the Luigi server and client. + +``app/luigitasks`` contains the task logic to be ran by Luigi. + +``app/sessions`` contains the "user session" data for the web app(s). + +``app/work`` is more of a temp folder; its contents may vary over time and are +not considered essential to the app. diff --git a/docs/base/commands.rst b/docs/base/commands.rst new file mode 100644 index 0000000..215f12c --- /dev/null +++ b/docs/base/commands.rst @@ -0,0 +1,23 @@ + +Running Commands +================ + +TODO + + +Typical Usage +------------- + +TODO + + +Running as System User +---------------------- + +TODO + + +Usage with ``sudo`` +------------------- + +TODO diff --git a/docs/base/config/db.rst b/docs/base/config/db.rst new file mode 100644 index 0000000..ce668cd --- /dev/null +++ b/docs/base/config/db.rst @@ -0,0 +1,86 @@ + +.. highlight:: ini + +Storing Config in DB +==================== + +We're getting ahead of ourselves a little here, if you're reading this manual +straight through. But for reference sake this probably belongs here. + +Settings Table +-------------- + +If you already have a Rattail DB, then it has a table named ``setting`` which +is designed to store config values. It's just a regular table so you can write +settings via SQL if you like: + +.. code-block:: sql + + insert into setting (name, value) values ('rattail.app_title', 'Poser'); + + update setting set value = 'Something Else' where name = 'rattail.app_title'; + +Although the main reason for putting settings in the DB is usually so you can +edit them via the web app. + + +Telling App to Read Config from DB +---------------------------------- + +Well first of all we must assume that the DB connection itself is configured. +More on that later but let's say you have this in place:: + + [rattail.db] + default.url = postgresql://user:password@localhost/poser + +Then you also must tell the Rattail config engine that a) it should read config +values from the DB at all, but probably also b) it should *prefer* values from +the DB over what was read from file. Do this within your config file:: + + [rattail.config] + usedb = true + preferdb = true + +With this in place, when the app requests a config value, it will come from the +DB if present, falling back to the file value if that exists. + + +File vs. DB Setting Names +------------------------- + +You may have noticed that the SQL examples above, and the examples used in +:doc:`syntax` are really for the same 'app_title' setting, but there is a key +difference in naming. + +So in a config file you might have this snippet to define a setting:: + + [rattail] + app_title = Poser + +But then that same setting is written in SQL as: + +.. code-block:: sql + + insert into setting (name, value) values ('rattail.app_title', 'Poser'); + +In other words the "section" and "option" names from the config file, are +joined together with a dot, for the DB setting name. + +When the app requests a config value, it must specify both a section and +option. The config engine then will auto-join them as needed when doing DB +lookups. + +.. code-block:: python + + config.get('rattail', 'app_title') + + +Avoiding the DB +--------------- + +The app can request config from "file only" if it needs to. It just has to +specify a flag when reading the value, for example: + +.. code-block:: python + + config.get('rattail', 'app_title', usedb=False) diff --git a/docs/base/config/defined.rst b/docs/base/config/defined.rst new file mode 100644 index 0000000..f03927e --- /dev/null +++ b/docs/base/config/defined.rst @@ -0,0 +1,11 @@ + +Defined Settings +================ + +There are a number of config settings which a given Poser/Rattail app may +leverage. Most if not all are optional. We'll try to describe them here. + +Note that we will present each in terms of a "section" and "option" pair, +i.e. using the naming convention found in a config file vs. that of DB. + +TODO! diff --git a/docs/base/config/generate.rst b/docs/base/config/generate.rst new file mode 100644 index 0000000..25e72c6 --- /dev/null +++ b/docs/base/config/generate.rst @@ -0,0 +1,21 @@ + +Generating Config Files +======================= + +Rattail is able to generate "starting point" config files for you, for any of +the types described in :doc:`paths`. Usually you will need to edit them +further, but the basic structure can be provided automatically. + +Run any of these commands from your env root (e.g. ``/srv/envs/poser``) +depending on which types of files you need: + +.. code-block:: sh + + bin/rattail make-config -O app/ -T rattail + bin/rattail make-config -O app/ -T quiet + bin/rattail make-config -O app/ -T web + bin/rattail make-config -O app/ -T datasync + bin/rattail make-config -O app/ -T filemon + +Each generated file should contain "TODO" comments directing your attention to +settings which may require adjustment. diff --git a/docs/base/config/index.rst b/docs/base/config/index.rst new file mode 100644 index 0000000..63f6dfd --- /dev/null +++ b/docs/base/config/index.rst @@ -0,0 +1,16 @@ + +Configuration +============= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + overview + syntax + paths + inheritance + generate + logging + db + defined diff --git a/docs/base/config/inheritance.rst b/docs/base/config/inheritance.rst new file mode 100644 index 0000000..2edbc28 --- /dev/null +++ b/docs/base/config/inheritance.rst @@ -0,0 +1,72 @@ + +.. highlight:: ini + +Config File Inheritance +======================= + +It may already be obvious, if you read :doc:`paths`, but it's possible for +config files to "inherit" from one another. The general idea is that all +config is collected from various files, when assembling the "final" config to +be used by the app. + +For a simple example let's assume you have just 2 typical config files in your +app dir: + +* ``/srv/envs/poser/app/rattail.conf`` +* ``/srv/envs/poser/app/quiet.conf`` + +Let's say that ``rattail.conf`` is a "complete" config file and may be used +directly, as-is. And that ``quiet.conf`` is not complete but "inherits" from +``rattail.conf`` (as is typical) so that it also may be used directly. + +In other words either of these commands should work when ran from +``/srv/envs/poser``: + +.. code-block:: sh + + bin/rattail -c app/rattail.conf make-uuid + bin/rattail -c app/quiet.conf make-uuid + +The contents of ``quiet.conf`` are usually quite minimal:: + + [rattail.config] + include = %(here)s/rattail.conf + + [handler_console] + level = INFO + +The "include" option within "rattail.config" section above, tells the Rattail +config parser to bring in the contents of ``rattail.conf`` whenever it is +reading ``quiet.conf`` - although any settings defined in ``quiet.conf`` will +override whatever was brought in from ``rattail.conf``. (In this example, +``quiet.conf`` only needs to set ``level = INFO`` to cut down on some logging +output on the console.) + +Caveats +------- + +There is a gotcha which can break the inheritance logic, but it can be avoided +if you follow one simple rule: + +The primary config file you reference when invoking the app +(e.g. ``bin/rattail -c qpp/quiet.conf ...``) must *not* contain a 'loggers' +section, i.e. it should *not* have a snippet like this:: + + [loggers] + keys = root, exc_logger, ... + +To be clear the gotcha only exists when: + +* config file which app is told to read, contains snippet like above +* config file has an ``include`` setting, meaning inheritance should happen +* config file also says to configure logging + +The reason it breaks is that we let Python standard ``logging`` module take +care of the logging configuration, but it will try to do so using the specified +config file (e.g. ``quiet.conf``) *only* instead of doing so with the combined +result. + +So again, just make sure there is no 'loggers' section in the config file you +present to your app. Or alternatively, you can make sure that same config file +*does* have all logging config within it, so e.g. inheritance would not affect +that part. diff --git a/docs/base/config/logging.rst b/docs/base/config/logging.rst new file mode 100644 index 0000000..4195f8a --- /dev/null +++ b/docs/base/config/logging.rst @@ -0,0 +1,32 @@ + +.. highlight:: ini + +Configuring Logging +=================== + +Rattail relies on Python standard ``logging`` module to configure logging. +However if :doc:`inheritance` is involved then Rattail will first combine all +applicable config files into a single file before handing that off to +`logging.config.fileConfig()`_. + +.. _logging.config.fileConfig(): https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig + +Rattail does not do any of this though, unless config says to. So if you want +it to configure logging in this way, specify this in your config file:: + + [rattail.config] + configure_logging = true + +Beyond that you must ensure your config file(s) contains appropriate settings +for logging. Rattail `has a sample`_ ``rattail.conf`` which includes a typical +logging section. (This is the source used to generate a new file when you run +``rattail make-config -T rattail`` command.) + +.. _has a sample: https://kallithea.rattailproject.org/rattail-project/rattail/files/master/rattail/data/config/rattail.conf + +See the Python docs for more info, in particular these sections: + +* `Configuration file format `_ +* `Logging Levels `_ +* `Useful Handlers `_ +* `Formatters `_ diff --git a/docs/base/config/overview.rst b/docs/base/config/overview.rst new file mode 100644 index 0000000..146e959 --- /dev/null +++ b/docs/base/config/overview.rst @@ -0,0 +1,31 @@ + +Overview +======== + +The basic idea of course is that the app needs to be configurable, so the +question is how to go about that. + +The short answer is, "We always use config files but can also store config in +the database where applicable." + +The advantage to config files is that they are easier to get started with, but +also we can restrict access at the file system level, which means we can (more +safely) store sensitive information in them. + +The advantage to storing config in DB, is that one can change config on-the-fly +using e.g. the web app. (Whereas changing a config file requires the app to be +restarted, so it will read the new file contents.) + +Generally speaking, certain config which is needed during app startup (e.g. DB +access credentials and logging config), or which is deemed "sensitive" +(passwords, API keys etc.), is kept in config files, and the remainder is kept +in the DB (if there is one). + +This behavior is itself configurable, but most typically, the app will not care +where a given setting is defined (file vs. DB) but if one is defined in both +places the DB value would win. But the app can request a config value from +"file only" and ignore the DB, where that is desirable (e.g. when reading +settings required for startup). + +We are still in the Base Layer docs at this point, so we'll focus on the config +files first, and come back to the DB near the end. diff --git a/docs/base/config/paths.rst b/docs/base/config/paths.rst new file mode 100644 index 0000000..18cd665 --- /dev/null +++ b/docs/base/config/paths.rst @@ -0,0 +1,67 @@ + +Typical File Paths +================== + +Here we'll describe some typical locations and filenames for config. + + +App-Wide +-------- + +Every app will need at least one config file. The convention is to name this +file ``rattail.conf`` and place it directly in your app dir, e.g. +``/srv/envs/poser/app/rattail.conf`` + +Many apps will benefit from having multiple config files, primarily for the +sake of organization and "separation of concerns". A robust app with several +features then might have the following, all in its app dir: + +``rattail.conf`` is considered the "core" config file for the app, which means +that no matter how you run the app, this file should be (in)directly referenced +somehow, so that it affects the runtime behavior. + +``quiet.conf`` is meant to be used for ad-hoc app commands which you run from +the console. It is a thin wrapper around ``rattail.conf`` and merely tries to +cut down on some of the output (logging) "noise" from commands. + +``cron.conf`` is also a thin wrapper, which cuts down on command output "noise" +and uses a custom logging file. + +``web.conf`` is meant to be used by the standard web app only; it defines the +config needed to run the web app and uses a custom logging file. + +``webapi.conf`` is meant to be used by the web API only; it defines the config +needed to run the web API and uses a custom logging file. + +``datasync.conf`` is meant to be used only by the ``datasync`` commands; it +defines the config needed to run datasync and uses a custom logging file. + +``filemon.conf`` is meant to be used only by the ``filemon`` commands; it +defines the config needed to run filemon and uses a custom logging file. + +``bouncer.conf`` is meant to be used only by the ``bouncer`` commands; it +defines the config needed to run bouncer and uses a custom logging file. + + +Machine-Wide +------------ + +If you have several apps running on a given machine, you may wish to share some +"common" config among all the apps. If so you can make a "machine-wide" config +file to store that common config. + +The convention here is to place the file at ``/etc/rattail/rattail.conf`` +although you could do whatever you like. + + +Site-Wide +--------- + +This is similar to the machine-wide scenario, but if you have multiple +*machines* which run apps, you may wish to share the common config in such a +way that all machines could access it. We call this a "site-wide" config file. + +How exactly you go about making this file available is beyond our scope here, +but a hint is to use Samba/CIFS. True location of the file would be anywhere +you like, on whichever machine you elected to be the provider of this common +config. diff --git a/docs/base/config/syntax.rst b/docs/base/config/syntax.rst new file mode 100644 index 0000000..41563a9 --- /dev/null +++ b/docs/base/config/syntax.rst @@ -0,0 +1,59 @@ + +.. highlight:: ini + +Config File Syntax +================== + +Rattail config files follow the traditional `INI file`_ syntax, e.g. here is a +snippet:: + + [rattail] + app_title = Poser + +.. _INI file: https://en.wikipedia.org/wiki/INI_file + +That example can be broken down into 3 parts: + +* section (the name in square brackets, "rattail") +* option (the name of the setting being defined, "app_title") +* value (the value for the setting, "Poser") + +When logic within the app needs to retrieve a value from config, it does so by +requesting both the section and option by name, for example: + +.. code-block:: python + + config.get('rattail', 'app_title') # returns "Poser" + +Under the hood we use Python's `configparser`_ module to parse config files. + +.. _configparser: https://docs.python.org/3/library/configparser.html + +If you think it would be handy for your app to have a new setting for some +reason, just create one and use it like so:: + + [poser] + foo = bar + +.. code-block:: python + + config.get('poser', 'foo') # returns "bar" + +So far our examples have been for simple string values. There is no way within +the standard INI file syntax, to define any data types other than string. +However the Rattail config parser/object can "coerce" values to a given type if +so requested, for example:: + + [poser] + foo_flag = true + foo_number = 42 + foo_entries = + first + second + third + +.. code-block:: python + + config.getbool('poser', 'foo_flag') # returns True + config.getint('poser', 'foo_number') # returns 42 + config.getlist('poser', 'foo_entries') # returns ["first", "second", "third"] diff --git a/docs/base/email.rst b/docs/base/email.rst new file mode 100644 index 0000000..1c1bf29 --- /dev/null +++ b/docs/base/email.rst @@ -0,0 +1,53 @@ + +Sending Email +============= + +TODO + + +Postfix Setup +------------- + +TODO + + +Types of Emails +--------------- + +TODO + + +Configuring SMTP +---------------- + +TODO + + +Configuring Recipients etc. +--------------------------- + +TODO + + +Customizing Templates +--------------------- + +TODO + + +How to programmatically send email +---------------------------------- + +TODO + + +Nod to email via error logging +------------------------------ + +TODO + + +Handling Email Bounces +---------------------- + +TODO diff --git a/docs/base/filemon.rst b/docs/base/filemon.rst new file mode 100644 index 0000000..8c88f81 --- /dev/null +++ b/docs/base/filemon.rst @@ -0,0 +1,5 @@ + +File Monitoring +=============== + +TODO diff --git a/docs/base/handlers.rst b/docs/base/handlers.rst new file mode 100644 index 0000000..4993d69 --- /dev/null +++ b/docs/base/handlers.rst @@ -0,0 +1,5 @@ + +App Handlers +============ + +TODO diff --git a/docs/base/index.rst b/docs/base/index.rst new file mode 100644 index 0000000..c92eb39 --- /dev/null +++ b/docs/base/index.rst @@ -0,0 +1,19 @@ + +Base Layer +========== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + reqs + venv + install + appdir + config/index + commands + scripts + email + supervisor + filemon + handlers diff --git a/docs/base/install.rst b/docs/base/install.rst new file mode 100644 index 0000000..1207245 --- /dev/null +++ b/docs/base/install.rst @@ -0,0 +1,49 @@ + +.. highlight:: sh + +Installation +============ + +How exactly you install an app to your virtual environment can vary. + +However they all involve using ``pip install`` in some way or another. And +therefore your virtual environment should be "activated" when running the +commands shown below. + + +Installing from PyPI +-------------------- + +If the app is published to the `Python Package Index`_ (PyPI) then you can just +``pip install`` it. For instance that is the case for `Theo`_, so if that's +what you're after then:: + + pip install tailbone-theo + +.. _Python Package Index: https://pypi.org/ +.. _Theo: https://pypi.org/project/tailbone-theo/ + + +Installing from Source +---------------------- + +If you need (or want) to install from source, then of course the first step is +to obtain the source code for the app. Here we'll again use Theo as our example:: + + git clone https://kallithea.rattailproject.org/rattail-project/theo + +Then you would install that to your virtual environment like so:: + + pip install -e ./theo + + +Sanity Check +------------ + +Before we do anything else let's confirm that you have the 'rattail' package +installed, at the very least. With your env still activated try this:: + + rattail --version + +That of course should spit out a version number; if you see anything else then +you may need to troubleshoot that first. diff --git a/docs/base/reqs.rst b/docs/base/reqs.rst new file mode 100644 index 0000000..63878fe --- /dev/null +++ b/docs/base/reqs.rst @@ -0,0 +1,14 @@ + +Requirements +============ + +The only "hard" requirement is a relatively modern Python 3. + +For best results, as of this writing that should be 3.6+ although generally +speaking, the newer the better. + +You should also know that more attention by far has been given to Linux over +the years. We'll try to ensure this manual works for Windows too but FYI that +it's possible we miss something. Thus far all production installs are done on +Debian or Ubuntu Linux so if you can use those then your experience may be +better. diff --git a/docs/base/scripts.rst b/docs/base/scripts.rst new file mode 100644 index 0000000..a74882f --- /dev/null +++ b/docs/base/scripts.rst @@ -0,0 +1,17 @@ + +Writing Scripts +=============== + +TODO + + +Python Scripts +-------------- + +TODO + + +Shell Scripts +------------- + +TODO diff --git a/docs/base/supervisor.rst b/docs/base/supervisor.rst new file mode 100644 index 0000000..d99e6bd --- /dev/null +++ b/docs/base/supervisor.rst @@ -0,0 +1,11 @@ + +Supervisord +=========== + +TODO + + +Linux Only... +------------- + +TODO diff --git a/docs/base/venv.rst b/docs/base/venv.rst new file mode 100644 index 0000000..a35ca41 --- /dev/null +++ b/docs/base/venv.rst @@ -0,0 +1,121 @@ + +.. highlight:: sh + +Virtual Environment +=================== + +Regardless of platform, you are strongly encouraged to use a `virtual +environment`_ for your app. This allows you to experiment with installation +without affecting the rest of your system. + +.. _virtual environment: https://packaging.python.org/glossary/#term-Virtual-Environment + + +Choosing a Location +------------------- + +It can be helpful to standardize the location of all virtual environments +regardless of their purpose. The tool you use to create a virtual environment +may (or may not) have opinions on that, but so do we: + +This manual will assume one of the following based on platform: + +* Linux - ``/srv/envs`` +* Windows - ``C:\envs`` + +So for instance if you run Linux and make a new virtual environment named +"poser" then it would live in ``/srv/envs/poser`` according to the above. + +Note that you may need to consider file permissions etc. when choosing your +actual location, but if possible you're encouraged to use the examples shown +here. + + +Choosing a User +--------------- + +To be thorough you may need to consider which user should "own" the virtual +environment and installed app. If you're just getting started then you can +skip this section for now and run all commands as yourself, then revisit the +issue later as needed. + +Why the User Matters +^^^^^^^^^^^^^^^^^^^^ + +The virtual environment and installed app, its config and data files etc. must +be owned by someone after all. So at the most basic level the user matters +simply because it can't be "nobody" - a choice must be made. + +Some config, data and/or log files may contain sensitive information (passwords +etc.) and should be "locked down" in some way to prevent unauthorized access. +So then the "owner" would have access to such files but perhaps no other users +would. + +When app commands are ran, by yourself in the console or e.g. via cron job, the +user which the command "runs as" will matter, in the sense that this user will +need access to any "restricted" (e.g. config) files. So typically all commands +would be ran as the same user who "owns" the app. + +User Options +^^^^^^^^^^^^ + +**"yourself"** - For instance my own username is 'lance' and so for convenience +in development, I might just run all commands as myself, and let all files be +owned by myself. This is the simplest option and most commands in this manual +will work as-is for this option. + +**"root"** - When deploying the app to a server, maybe I am connecting to it +via SSH as the 'lance' user, but let's say I am just one of several users who +needs to connect, and so it doesn't make for 'lance' to be the file owner, or +to run app commands as 'lance'. You can, if you really want to, use 'root' as +the app owner/user, although you are encouraged to use one of the other options +below instead. + +**"admin"** - In some organizations there is a dedicated "admin" user defined +within LDAP etc. If such a user is already present in the system then there +may be no reason not to use this for the app owner/user. + +**"rattail"** - This option is my personal preference. Here we create a +dedicated system user whose sole purpose is to be the app owner/user. I always +name this user 'rattail' and you can think of it like the 'www-data' or +'postgres' user accounts. Basically this is a "true" system user meaning it +doesn't correspond to any person. But it can be defined on many machines for +automation's sake, e.g. if SSH keys are shared too then 'rattail' user on one +machine can effectively run 'rattail' commands on any other machine. + + +Creating a Virtual Environment +------------------------------ + +Please also see `Creating a virtual environment`_ in the Python Packaging User +Guide. + +.. _Creating a virtual environment: https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment + +For our purposes, on Linux you can do this:: + + python3 -m venv /srv/envs/poser + + +Using a Virtual Environment +--------------------------- + +If you're new to virtual environments then you're encouraged to read over the +following, from the Python Packaging User Guide: + +* `Activating a virtual environment `_ +* `Leaving the virtual environment `_ + +So for our Linux example you might activate with:: + + source /srv/envs/poser/bin/activate + +*All* commands in this manual will assume you have a virtual environment, but +*most* of them will not require it to be "activated" to run the command. The +main exception to that is ``pip`` commands, which *do* assume the virtual +environment is activated. + +It may be helpful to ensure your new virtual environment has the lastest pip +and friends. Note that your env should be activated when running this:: + + pip install -U pip setuptools wheel diff --git a/docs/conf.py b/docs/conf.py index 44dd17b..62b7cce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- -project = 'rattail-guide' +project = 'rattail-manual' copyright = '2021, Lance Edgar' author = 'Lance Edgar' @@ -49,4 +49,4 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +html_static_path = ['_static'] diff --git a/docs/data/batches.rst b/docs/data/batches.rst new file mode 100644 index 0000000..39c38f1 --- /dev/null +++ b/docs/data/batches.rst @@ -0,0 +1,5 @@ + +Data Batch Processing +===================== + +TODO diff --git a/docs/data/datasync.rst b/docs/data/datasync.rst new file mode 100644 index 0000000..2017d51 --- /dev/null +++ b/docs/data/datasync.rst @@ -0,0 +1,5 @@ + +Data Synchronization +==================== + +TODO diff --git a/docs/data/db.rst b/docs/data/db.rst new file mode 100644 index 0000000..5d712e6 --- /dev/null +++ b/docs/data/db.rst @@ -0,0 +1,5 @@ + +Rattail Database +================ + +TODO diff --git a/docs/data/importing.rst b/docs/data/importing.rst new file mode 100644 index 0000000..3afb840 --- /dev/null +++ b/docs/data/importing.rst @@ -0,0 +1,5 @@ + +Data Import / Export +==================== + +TODO diff --git a/docs/data/index.rst b/docs/data/index.rst new file mode 100644 index 0000000..0826550 --- /dev/null +++ b/docs/data/index.rst @@ -0,0 +1,18 @@ + +Data Layer +========== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + db + trainwreck + other + importing + datasync + batches + multinode + tempmon + tasks + patterns diff --git a/docs/data/multinode.rst b/docs/data/multinode.rst new file mode 100644 index 0000000..e2684ee --- /dev/null +++ b/docs/data/multinode.rst @@ -0,0 +1,5 @@ + +Multiple App Nodes +================== + +TODO diff --git a/docs/data/other.rst b/docs/data/other.rst new file mode 100644 index 0000000..3e948a5 --- /dev/null +++ b/docs/data/other.rst @@ -0,0 +1,5 @@ + +POS and Other Data Stores +========================= + +TODO diff --git a/docs/data/patterns.rst b/docs/data/patterns.rst new file mode 100644 index 0000000..1340338 --- /dev/null +++ b/docs/data/patterns.rst @@ -0,0 +1,5 @@ + +Common Patterns +=============== + +TODO diff --git a/docs/data/tasks.rst b/docs/data/tasks.rst new file mode 100644 index 0000000..bec1dce --- /dev/null +++ b/docs/data/tasks.rst @@ -0,0 +1,5 @@ + +Common Tasks +============ + +TODO diff --git a/docs/data/tempmon.rst b/docs/data/tempmon.rst new file mode 100644 index 0000000..2af78e6 --- /dev/null +++ b/docs/data/tempmon.rst @@ -0,0 +1,5 @@ + +Temperature Monitoring +====================== + +TODO diff --git a/docs/data/trainwreck.rst b/docs/data/trainwreck.rst new file mode 100644 index 0000000..ec69e2b --- /dev/null +++ b/docs/data/trainwreck.rst @@ -0,0 +1,5 @@ + +Trainwreck Database +=================== + +TODO diff --git a/docs/deploy/fabric.rst b/docs/deploy/fabric.rst new file mode 100644 index 0000000..bca2744 --- /dev/null +++ b/docs/deploy/fabric.rst @@ -0,0 +1,5 @@ + +Fabric +====== + +TODO diff --git a/docs/deploy/index.rst b/docs/deploy/index.rst new file mode 100644 index 0000000..49fdbfc --- /dev/null +++ b/docs/deploy/index.rst @@ -0,0 +1,10 @@ + +Deployment Layer +================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + overview + fabric diff --git a/docs/deploy/overview.rst b/docs/deploy/overview.rst new file mode 100644 index 0000000..45595c5 --- /dev/null +++ b/docs/deploy/overview.rst @@ -0,0 +1,5 @@ + +Overview +======== + +TODO diff --git a/docs/index.rst b/docs/index.rst index c920e88..20b38a6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,15 +1,38 @@ -.. rattail-guide documentation master file, created by - sphinx-quickstart on Wed Jan 6 15:33:34 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. -Welcome to rattail-guide's documentation! -========================================= +Rattail Manual +============== + +This is "the reference manual" for Rattail, intended primarily for developers. +It assumes the reader is involved with the creation, customization and/or +maintenance of a Poser/Rattail app. It tries to explain the concepts involved +and hopefully tie them all together. + +If you prefer step-by-step instructions for creating your own app then see also +the `Rattail Tutorial`_. + +.. _Rattail Tutorial: https://rattailproject.org/docs/rattail-tutorial/ + +This manual is primarily organized according to the possible "layers" of a +Poser/Rattail app: + +* :doc:`base/index` - general app install, command line, plus filemon etc. +* :doc:`data/index` - when a DB or data import/export come into it +* :doc:`web/index` - all things web +* :doc:`luigi/index` - using Luigi for better overnight automation +* :doc:`deploy/index` - deals with production deployment +* :doc:`backup/index` - backing up the production system .. toctree:: :maxdepth: 2 :caption: Contents: + readfirst + base/index + data/index + web/index + luigi/index + deploy/index + backup/index Indices and tables diff --git a/docs/luigi/index.rst b/docs/luigi/index.rst new file mode 100644 index 0000000..7e102ff --- /dev/null +++ b/docs/luigi/index.rst @@ -0,0 +1,10 @@ + +Luigi Layer +=========== + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + overview diff --git a/docs/luigi/overview.rst b/docs/luigi/overview.rst new file mode 100644 index 0000000..45595c5 --- /dev/null +++ b/docs/luigi/overview.rst @@ -0,0 +1,5 @@ + +Overview +======== + +TODO diff --git a/docs/readfirst.rst b/docs/readfirst.rst new file mode 100644 index 0000000..2fab56b --- /dev/null +++ b/docs/readfirst.rst @@ -0,0 +1,16 @@ + +Read This First +=============== + +Here we just want to touch on some conventions used by this manual. + +We use the name "Poser" (or "poser") to refer to *any* app based on Rattail. +For example this can refer to: + +* `Theo `_ +* `Rattail Demo `_ +* *your* custom app + +Poser is not a real thing in other words; that name is made up and you should +replace it (mentally and literally) when reading these docs and running +commands etc. diff --git a/docs/web/index.rst b/docs/web/index.rst new file mode 100644 index 0000000..2a10ee8 --- /dev/null +++ b/docs/web/index.rst @@ -0,0 +1,13 @@ + +Web Layer +========= + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + overview + webmain + webapi + spa diff --git a/docs/web/overview.rst b/docs/web/overview.rst new file mode 100644 index 0000000..45595c5 --- /dev/null +++ b/docs/web/overview.rst @@ -0,0 +1,5 @@ + +Overview +======== + +TODO diff --git a/docs/web/spa.rst b/docs/web/spa.rst new file mode 100644 index 0000000..e6b672b --- /dev/null +++ b/docs/web/spa.rst @@ -0,0 +1,5 @@ + +Single-Page Apps +================ + +TODO diff --git a/docs/web/webapi.rst b/docs/web/webapi.rst new file mode 100644 index 0000000..27c3891 --- /dev/null +++ b/docs/web/webapi.rst @@ -0,0 +1,17 @@ + +Web API +======= + +TODO + + +Common Tasks +------------ + +TODO + + +Common Patterns +--------------- + +TODO diff --git a/docs/web/webmain.rst b/docs/web/webmain.rst new file mode 100644 index 0000000..372dcf8 --- /dev/null +++ b/docs/web/webmain.rst @@ -0,0 +1,17 @@ + +Default Web App +=============== + +TODO + + +Common Tasks +------------ + +TODO + + +Common Patterns +--------------- + +TODO