wutta/wuttjamaican
1
0
Fork 0
Code Activity

Add make_engine_from_config() method for AppHandler

Browse source 
and other misc. tweaks needed to get this incorporated into Rattail
This commit is contained in:
Lance Edgar 2023-11-19 20:36:51 -06:00
parent afd2d005a3
commit b458272207
8 changed files with 177 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
src/wuttjamaican/app.py
View file

@ -24,7 +24,7 @@
WuttJamaican - app handler
"""
from wuttjamaican.util import load_entry_points
from wuttjamaican.util import load_entry_points, load_object, parse_bool
class AppHandler:
@ -47,6 +47,60 @@ class AppHandler:
self.config = config
self.handlers = {}
def make_engine_from_config(
self,
config_dict,
prefix='sqlalchemy.',
**kwargs):
"""
Construct a new DB engine from configuration dict.
This is a wrapper around upstream
:func:`sqlalchemy:sqlalchemy.engine_from_config()`.
The purpose of the customization is to allow certain
attributes of the engine to be driven by config, whereas the
upstream function is more limited in that regard. The
following in particular:
* ``poolclass``
* ``pool_pre_ping``
If these options are present in the configuration dict, they
will be coerced to appropriate Python equivalents and then
passed as kwargs to the upstream function.
An example config file leveraging this feature:
.. code-block:: ini
[wutta.db]
default.url = sqlite:///tmp/default.sqlite
default.poolclass = sqlalchemy.pool:NullPool
default.pool_pre_ping = true
Note that if present, the ``poolclass`` value must be a "spec"
string, as required by
:func:`~wuttjamaican.util.load_object()`.
"""
import sqlalchemy as sa
config_dict = dict(config_dict)
# convert 'poolclass' arg to actual class
key = f'{prefix}poolclass'
if key in config_dict and 'poolclass' not in kwargs:
kwargs['poolclass'] = load_object(config_dict.pop(key))
# convert 'pool_pre_ping' arg to boolean
key = f'{prefix}pool_pre_ping'
if key in config_dict and 'pool_pre_ping' not in kwargs:
kwargs['pool_pre_ping'] = parse_bool(config_dict.pop(key))
engine = sa.engine_from_config(config_dict, prefix, **kwargs)
return engine
def make_session(self, **kwargs):
"""
Creates a new SQLAlchemy session for the app DB. By default

42
src/wuttjamaican/conf.py
View file

@ -195,6 +195,15 @@ class WuttaConfig:
path = path % {'here': here}
self._load_ini_configs(path, configs, require=False)
def get_prioritized_files(self):
"""
Returns list of config files in order of priority.
By default, :attr:`files_read` should already be in the
correct order, but this is to make things more explicit.
"""
return self.files_read
def setdefault(
self,
key,
@ -222,6 +231,7 @@ class WuttaConfig:
key,
default=UNSPECIFIED,
require=False,
ignore_ambiguous=False,
message=None,
usedb=None,
preferdb=None,
@ -283,6 +293,12 @@ class WuttaConfig:
Note that it is an error to specify a default value if you
also specify ``require=True``.
:param ignore_ambiguous: By default this method will log a
warning if an ambiguous value is detected (as described
above). Pass a true value for this flag to avoid the
warnings. Should use with caution, as the warnings are
there for a reason.
:param message: Optional first part of message to be used,
when raising a "value not found" error. If not specified,
a default error message will be generated.
@ -329,7 +345,8 @@ class WuttaConfig:
if not isinstance(value, configuration.Configuration):
return value
log.warning("ambiguous config key '%s' returns: %s", key, value)
if not ignore_ambiguous:
log.warning("ambiguous config key '%s' returns: %s", key, value)
# read from db last if so requested
if usedb and not preferdb:
@ -380,6 +397,29 @@ class WuttaConfig:
value = self.get(*args, **kwargs)
return parse_bool(value)
def get_int(self, *args, **kwargs):
"""
Retrieve an integer value from config.
Accepts same params as :meth:`get()` but if a value is found,
it will be coerced to integer via the :class:`python:int()`
constructor.
"""
value = self.get(*args, **kwargs)
if value is not None:
return int(value)
def get_list(self, *args, **kwargs):
"""
Retrieve a list value from config.
Accepts same params as :meth:`get()` but if a value is found,
it will be coerced to list via
:func:`~wuttjamaican.util.parse_list()`.
"""
value = self.get(*args, **kwargs)
return parse_list(value)
def get_dict(self, prefix):
"""
Retrieve a particular group of values, as a dictionary.

57
src/wuttjamaican/db/conf.py
View file

@ -28,56 +28,7 @@ from collections import OrderedDict
import sqlalchemy as sa
from wuttjamaican.util import load_object, parse_bool, parse_list
def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
"""
Construct a new DB engine from configuration.
This is a wrapper around upstream
:func:`sqlalchemy:sqlalchemy.engine_from_config()`.
The purpose of the customization is to allow certain attributes of
the engine to be driven by config, whereas the upstream function
is more limited in that regard. The following in particular:
* ``poolclass``
* ``pool_pre_ping``
If these options are present in the configuration dictionary, they
will be coerced to appropriate Python equivalents and then passed
as kwargs to the upstream function.
An example config file leveraging this feature:
.. code-block:: ini
[wutta.db]
sqlalchemy.url = sqlite:///tmp/default.sqlite
sqlalchemy.poolclass = sqlalchemy.pool:NullPool
sqlalchemy.pool_pre_ping = true
Note that if present, the ``poolclass`` value must be a "spec"
string, as required by :func:`~wuttjamaican.util.load_object()`.
"""
config_dict = dict(configuration)
# convert 'poolclass' arg to actual class
key = f'{prefix}poolclass'
if key in config_dict:
kwargs.setdefault('poolclass', load_object(config_dict[key]))
del config_dict[key]
# convert 'pool_pre_ping' arg to boolean
key = f'{prefix}pool_pre_ping'
if key in config_dict:
kwargs.setdefault('pool_pre_ping', parse_bool(config_dict[key]))
del config_dict[key]
engine = sa.engine_from_config(config_dict, prefix, **kwargs)
return engine
from wuttjamaican.util import parse_list
def get_engines(config, prefix):
@ -111,6 +62,8 @@ def get_engines(config, prefix):
:returns: A dictionary of SQLAlchemy engines, with keys matching
those found in config.
"""
app = config.get_app()
keys = config.get(f'{prefix}.keys', usedb=False)
if keys:
keys = parse_list(keys)
@ -122,11 +75,11 @@ def get_engines(config, prefix):
for key in keys:
key = key.strip()
try:
engines[key] = engine_from_config(cfg, prefix=f'{key}.')
engines[key] = app.make_engine_from_config(cfg, prefix=f'{key}.')
except KeyError:
if key == 'default':
try:
engines[key] = engine_from_config(cfg, prefix='sqlalchemy.')
engines[key] = app.make_engine_from_config(cfg, prefix='sqlalchemy.')
except KeyError:
pass
return engines

2
src/wuttjamaican/util.py
View file

@ -150,6 +150,8 @@ def parse_list(value):
"""
if value is None:
return []
if isinstance(value, list):
return value
parser = shlex.shlex(value)
parser.whitespace += ','
parser.whitespace_split = True

37
tests/db/test_conf.py
View file

@ -7,48 +7,11 @@ from unittest import TestCase
import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.engine import Engine
from sqlalchemy.pool import NullPool
from wuttjamaican.db import conf
from wuttjamaican.conf import WuttaConfig
class TestEngineFromConfig(TestCase):
def test_basic(self):
engine = conf.engine_from_config({
'sqlalchemy.url': 'sqlite://',
})
self.assertIsInstance(engine, Engine)
def test_poolclass(self):
engine = conf.engine_from_config({
'sqlalchemy.url': 'sqlite://',
})
self.assertNotIsInstance(engine.pool, NullPool)
engine = conf.engine_from_config({
'sqlalchemy.url': 'sqlite://',
'sqlalchemy.poolclass': 'sqlalchemy.pool:NullPool',
})
self.assertIsInstance(engine.pool, NullPool)
def test_pool_pre_ping(self):
engine = conf.engine_from_config({
'sqlalchemy.url': 'sqlite://',
})
self.assertFalse(engine.pool._pre_ping)
engine = conf.engine_from_config({
'sqlalchemy.url': 'sqlite://',
'sqlalchemy.pool_pre_ping': 'true',
})
self.assertTrue(engine.pool._pre_ping)
class TestGetEngines(TestCase):
def setUp(self):

34
tests/test_app.py
View file

@ -5,6 +5,8 @@ from unittest.mock import patch, MagicMock
import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.engine import Engine
from sqlalchemy.pool import NullPool
from wuttjamaican import app, db
@ -19,6 +21,38 @@ class TestAppHandler(TestCase):
self.assertIs(self.app.config, self.config)
self.assertEqual(self.app.handlers, {})
def test_make_engine_from_config_basic(self):
engine = self.app.make_engine_from_config({
'sqlalchemy.url': 'sqlite://',
})
self.assertIsInstance(engine, Engine)
def test_make_engine_from_config_poolclass(self):
engine = self.app.make_engine_from_config({
'sqlalchemy.url': 'sqlite://',
})
self.assertNotIsInstance(engine.pool, NullPool)
engine = self.app.make_engine_from_config({
'sqlalchemy.url': 'sqlite://',
'sqlalchemy.poolclass': 'sqlalchemy.pool:NullPool',
})
self.assertIsInstance(engine.pool, NullPool)
def test_make_engine_from_config_pool_pre_ping(self):
engine = self.app.make_engine_from_config({
'sqlalchemy.url': 'sqlite://',
})
self.assertFalse(engine.pool._pre_ping)
engine = self.app.make_engine_from_config({
'sqlalchemy.url': 'sqlite://',
'sqlalchemy.pool_pre_ping': 'true',
})
self.assertTrue(engine.pool._pre_ping)
def test_make_session(self):
session = self.app.make_session()
self.assertIsInstance(session, db.Session.class_)

35
tests/test_conf.py
View file

@ -117,6 +117,23 @@ baz = B
self.assertIsNone(config.get('foo.bar'))
self.assertEqual(config.get('foo.baz'), 'B')
def test_prioritized_files(self):
first = self.write_file('first.conf', """\
[foo]
bar = 1
""")
second = self.write_file('second.conf', """\
[wutta.config]
require = %(here)s/first.conf
""")
config = conf.WuttaConfig(files=[second])
files = config.get_prioritized_files()
self.assertEqual(len(files), 2)
self.assertEqual(files[0], second)
self.assertEqual(files[1], first)
def test_constructor_defaults(self):
config = conf.WuttaConfig()
self.assertEqual(config.defaults, {})
@ -351,6 +368,24 @@ configure_logging = true
config = conf.WuttaConfig()
self.assertRaises(ConfigurationError, config.require, 'foo')
def test_get_bool(self):
config = conf.WuttaConfig()
self.assertFalse(config.get_bool('foo.bar'))
config.setdefault('foo.bar', 'true')
self.assertTrue(config.get_bool('foo.bar'))
def test_get_int(self):
config = conf.WuttaConfig()
self.assertIsNone(config.get_int('foo.bar'))
config.setdefault('foo.bar', '42')
self.assertEqual(config.get_int('foo.bar'), 42)
def test_get_list(self):
config = conf.WuttaConfig()
self.assertEqual(config.get_list('foo.bar'), [])
config.setdefault('foo.bar', 'hello world')
self.assertEqual(config.get_list('foo.bar'), ['hello', 'world'])
class TestGenericDefaultFiles(TestCase):

5
tests/test_util.py
View file

@ -190,6 +190,11 @@ class TestParseList(TestCase):
self.assertIsInstance(value, list)
self.assertEqual(len(value), 0)
def test_list_instance(self):
mylist = []
value = util.parse_list(mylist)
self.assertIs(value, mylist)
def test_single_value(self):
value = util.parse_list('foo')
self.assertEqual(len(value), 1)