rattail/tailbone
2
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

OMG a ridiculous commit to overhaul import handler config etc.

Browse source 
- add `MasterView.configurable` concept, `/configure.mako` template
- add new master view for DataSync Threads (needs content)
- tweak view config for DataSync Changes accordingly
- update the Configure DataSync page per `configurable` concept
- add new Configure Import/Export page, per `configurable`
- add basic views for Raw Permissions
This commit is contained in:
Lance Edgar 2021-12-06 20:04:34 -06:00
parent 282185c5af
commit cc4b2278e7
10 changed files with 735 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

160
tailbone/views/datasync.py
View file

@ -32,54 +32,44 @@ import logging
from rattail.db import model
from rattail.datasync.config import load_profiles
from rattail.datasync.util import get_lastrun, purge_datasync_settings
from rattail.datasync.util import purge_datasync_settings
from tailbone.views import MasterView
from tailbone.util import csrf_token
log = logging.getLogger(__name__)
class DataSyncChangeView(MasterView):
class DataSyncThreadView(MasterView):
"""
Master view for the DataSyncChange model.
Master view for DataSync itself.
This should (eventually) show all running threads in the main
index view, with status for each, sort of akin to "dashboard".
For now it only serves the config view.
"""
model_class = model.DataSyncChange
url_prefix = '/datasync/changes'
permission_prefix = 'datasync'
normalized_model_name = 'datasyncthread'
model_title = "DataSync Thread"
model_key = 'key'
route_prefix = 'datasync'
url_prefix = '/datasync'
viewable = False
creatable = False
editable = False
bulk_deletable = True
deletable = False
filterable = False
pageable = False
labels = {
'batch_id': "Batch ID",
}
configurable = True
config_title = "DataSync"
grid_columns = [
'source',
'batch_id',
'batch_sequence',
'payload_type',
'payload_key',
'deletion',
'obtained',
'consumer',
'key',
]
def configure_grid(self, g):
super(DataSyncChangeView, self).configure_grid(g)
# batch_sequence
g.set_label('batch_sequence', "Batch Seq.")
g.filters['batch_sequence'].label = "Batch Sequence"
g.set_sort_defaults('obtained')
g.set_type('obtained', 'datetime')
def template_kwargs_index(self, **kwargs):
kwargs['allow_filemon_restart'] = bool(self.rattail_config.get('tailbone', 'filemon.restart'))
return kwargs
def get_data(self, session=None):
data = []
return data
def restart(self):
cmd = self.rattail_config.getlist('tailbone', 'datasync.restart',
@ -93,23 +83,7 @@ class DataSyncChangeView(MasterView):
self.request.session.flash("DataSync daemon could not be restarted; result was: {}".format(result), 'error')
return self.redirect(self.request.get_referrer(default=self.request.route_url('datasyncchanges')))
def configure(self):
"""
View for configuring the DataSync daemon.
"""
if self.request.method == 'POST':
# if self.request.is_xhr and not self.request.POST:
if self.request.POST.get('purge_settings'):
self.delete_settings()
self.request.session.flash("Settings have been removed.")
return self.redirect(self.request.current_route_url())
else:
data = self.request.json_body
self.save_settings(data)
self.request.session.flash("Settings have been saved. "
"You should probably restart DataSync now.")
return self.json_response({'success': True})
def configure_get_context(self):
profiles = load_profiles(self.rattail_config,
include_disabled=True,
ignore_problems=True)
@ -148,27 +122,21 @@ class DataSyncChangeView(MasterView):
profiles_data.append(data)
return {
'master': self,
# TODO: really only buefy themes are supported here
'use_buefy': self.get_use_buefy(),
'index_title': "DataSync Changes",
'index_url': self.get_index_url(),
'profiles': profiles,
'profiles_data': profiles_data,
'restart_command': self.rattail_config.get('tailbone', 'datasync.restart'),
'system_user': getpass.getuser(),
}
def save_settings(self, data):
model = self.model
# collect new settings
def configure_gather_settings(self, data):
settings = []
watch = []
for profile in data['profiles']:
pkey = profile['key']
if profile['enabled']:
watch.append(pkey)
settings.extend([
{'name': 'rattail.datasync.{}.watcher'.format(pkey),
'value': profile['watcher_spec']},
@ -183,10 +151,12 @@ class DataSyncChangeView(MasterView):
{'name': 'rattail.datasync.{}.consumers.runas'.format(pkey),
'value': profile['watcher_default_runas']},
])
consumers = []
if profile['watcher_consumes_self']:
consumers = ['self']
else:
for consumer in profile['consumers_data']:
ckey = consumer['key']
if consumer['enabled']:
@ -205,10 +175,12 @@ class DataSyncChangeView(MasterView):
{'name': 'rattail.datasync.{}.consumer.{}.runas'.format(pkey, ckey),
'value': consumer['consumer_runas']},
])
settings.extend([
{'name': 'rattail.datasync.{}.consumers'.format(pkey),
'value': ', '.join(consumers)},
])
settings.extend([
{'name': 'rattail.datasync.watch',
'value': ', '.join(watch)},
@ -216,15 +188,9 @@ class DataSyncChangeView(MasterView):
'value': data['restart_command']},
])
# delete all current settings
self.delete_settings()
return settings
# create all new settings
for setting in settings:
self.Session.add(model.Setting(name=setting['name'],
value=setting['value']))
def delete_settings(self):
def configure_remove_settings(self):
purge_datasync_settings(self.rattail_config, self.Session())
@classmethod
@ -235,33 +201,65 @@ class DataSyncChangeView(MasterView):
@classmethod
def _datasync_defaults(cls, config):
permission_prefix = cls.get_permission_prefix()
route_prefix = cls.get_route_prefix()
url_prefix = cls.get_url_prefix()
# fix permission group title
config.add_tailbone_permission_group(permission_prefix, label="DataSync")
# restart datasync
# restart
config.add_tailbone_permission(permission_prefix,
'{}.restart'.format(permission_prefix),
label="Restart the DataSync daemon")
config.add_route('datasync.restart', '/datasync/restart',
config.add_route('{}.restart'.format(route_prefix),
'{}/restart'.format(url_prefix),
request_method='POST')
config.add_view(cls, attr='restart',
route_name='datasync.restart',
route_name='{}.restart'.format(route_prefix),
permission='{}.restart'.format(permission_prefix))
# configure datasync
config.add_tailbone_permission(permission_prefix,
'{}.configure'.format(permission_prefix),
label="Configure the DataSync daemon")
config.add_route('datasync.configure', '/datasync/configure')
config.add_view(cls, attr='configure',
route_name='datasync.configure',
permission='{}.configure'.format(permission_prefix),
renderer='/datasync/configure.mako')
class DataSyncChangeView(MasterView):
"""
Master view for the DataSyncChange model.
"""
model_class = model.DataSyncChange
url_prefix = '/datasync/changes'
permission_prefix = 'datasync_changes'
creatable = False
editable = False
bulk_deletable = True
labels = {
'batch_id': "Batch ID",
}
grid_columns = [
'source',
'batch_id',
'batch_sequence',
'payload_type',
'payload_key',
'deletion',
'obtained',
'consumer',
]
def configure_grid(self, g):
super(DataSyncChangeView, self).configure_grid(g)
# batch_sequence
g.set_label('batch_sequence', "Batch Seq.")
g.filters['batch_sequence'].label = "Batch Sequence"
g.set_sort_defaults('obtained')
g.set_type('obtained', 'datetime')
def template_kwargs_index(self, **kwargs):
kwargs['allow_filemon_restart'] = bool(self.rattail_config.get('tailbone', 'filemon.restart'))
return kwargs
# TODO: deprecate / remove this
DataSyncChangesView = DataSyncChangeView
def includeme(config):
DataSyncThreadView.defaults(config)
DataSyncChangeView.defaults(config)

156
tailbone/views/importing.py
View file

@ -35,6 +35,7 @@ import time
import json
import six
import sqlalchemy as sa
from rattail.exceptions import ConfigurationError
from rattail.threads import Thread
@ -66,14 +67,19 @@ class ImportingView(MasterView):
filterable = False
pageable = False
configurable = True
config_title = "Import / Export"
labels = {
'host_title': "Data Source",
'local_title': "Data Target",
'direction_display': "Direction",
}
grid_columns = [
'host_title',
'local_title',
'direction_display',
'handler_spec',
]
@ -84,6 +90,7 @@ class ImportingView(MasterView):
'handler_spec',
'host_title',
'local_title',
'direction_display',
'models',
]
@ -105,18 +112,14 @@ class ImportingView(MasterView):
app = self.get_rattail_app()
data = []
for Handler in app.all_import_handlers():
handler = Handler(self.rattail_config)
for handler in app.get_designated_import_handlers(
ignore_errors=True, sort=True):
data.append(self.normalize(handler))
data.sort(key=lambda handler: (handler['host_title'],
handler['local_title']))
return data
def normalize(self, handler):
Handler = handler.__class__
return {
'_handler': handler,
def normalize(self, handler, keep_handler=True):
data = {
'key': handler.get_key(),
'generic_title': handler.get_generic_title(),
'host_key': handler.host_key,
@ -124,7 +127,31 @@ class ImportingView(MasterView):
'local_key': handler.local_key,
'local_title': handler.get_generic_local_title(),
'handler_spec': handler.get_spec(),
}
'direction': handler.direction,
'direction_display': handler.direction.capitalize(),
}
if keep_handler:
data['_handler'] = handler
alternates = getattr(handler, 'alternate_handlers', None)
if alternates:
data['alternates'] = []
for alternate in alternates:
data['alternates'].append(self.normalize(
alternate, keep_handler=keep_handler))
cmd = self.get_cmd_for_handler(handler, ignore_errors=True)
if cmd:
data['cmd'] = ' '.join(cmd)
data['command'] = cmd[0]
data['subcommand'] = cmd[1]
runas = self.get_runas_for_handler(handler)
if runas:
data['default_runas'] = runas
return data
def configure_grid(self, g):
super(ImportingView, self).configure_grid(g)
@ -139,9 +166,9 @@ class ImportingView(MasterView):
"""
key = self.request.matchdict['key']
app = self.get_rattail_app()
for Handler in app.all_import_handlers():
if Handler.get_key() == key:
return self.normalize(Handler(self.rattail_config))
handler = app.get_designated_import_handler(key, ignore_errors=True)
if handler:
return self.normalize(handler)
raise self.notfound()
def get_instance_title(self, handler_info):
@ -206,8 +233,8 @@ class ImportingView(MasterView):
def cache_runjob_form_values(self, handler, form):
handler_key = handler.get_key()
def make_key(key):
return 'rattail.importing.{}.{}'.format(handler_key, key)
def make_key(field):
return 'rattail.importing.{}.{}'.format(handler_key, field)
for field in form.fields:
key = make_key(field)
@ -216,8 +243,8 @@ class ImportingView(MasterView):
def read_cached_runjob_values(self, handler, form):
handler_key = handler.get_key()
def make_key(key):
return 'rattail.importing.{}.{}'.format(handler_key, key)
def make_key(field):
return 'rattail.importing.{}.{}'.format(handler_key, field)
for field in form.fields:
key = make_key(field)
@ -331,8 +358,10 @@ class ImportingView(MasterView):
# invoke handler command via subprocess
try:
result = subprocess.run(cmd, check=True, capture_output=True)
output = result.stderr.decode('utf_8').strip()
result = subprocess.run(cmd, check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = result.stdout.decode('utf_8').strip()
except Exception as error:
log.warning("failed to invoke handler cmd: %s", cmd, exc_info=True)
@ -346,14 +375,14 @@ class ImportingView(MasterView):
{}
```
And here is the STDERR output:
And here is the output:
```
{}
```
""".format(handler.direction.capitalize(),
' '.join(cmd),
error.stderr.decode('utf_8').strip())
error.stdout.decode('utf_8').strip())
msg = markdown.markdown(msg, extensions=['fenced_code'])
msg = HTML.literal(msg)
msg = HTML.tag('div', class_='tailbone-markdown', c=[msg])
@ -394,21 +423,35 @@ And here is the STDERR output:
notes = HTML.literal(notes)
return HTML.tag('div', class_='tailbone-markdown', c=[notes])
def make_runjob_cmd(self, handler, form, typ, port=None):
def get_cmd_for_handler(self, handler, ignore_errors=False):
handler_key = handler.get_key()
option = '{}.cmd'.format(handler_key)
cmd = self.rattail_config.getlist('rattail.importing', option)
cmd = self.rattail_config.getlist('rattail.importing',
'{}.cmd'.format(handler_key))
if not cmd or len(cmd) != 2:
msg = ("Missing or invalid config; please set '{}' in the "
"[rattail.importing] section of your config file".format(option))
raise ConfigurationError(msg)
cmd = self.rattail_config.getlist('rattail.importing',
'{}.default_cmd'.format(handler_key))
command, subcommand = cmd
if not cmd or len(cmd) != 2:
msg = ("Missing or invalid config; please set '{}.default_cmd' in the "
"[rattail.importing] section of your config file".format(handler_key))
if ignore_errors:
return
raise ConfigurationError(msg)
option = '{}.runas'.format(handler_key)
runas = self.rattail_config.require('rattail.importing', option)
return cmd
def get_runas_for_handler(self, handler):
handler_key = handler.get_key()
runas = self.rattail_config.get('rattail.importing',
'{}.runas'.format(handler_key))
if runas:
return runas
return self.rattail_config.get('rattail', 'runas.default')
def make_runjob_cmd(self, handler, form, typ, port=None):
command, subcommand = self.get_cmd_for_handler(handler)
runas = self.get_runas_for_handler(handler)
data = form.validated
if typ == 'true':
@ -460,7 +503,10 @@ And here is the STDERR output:
cmd.append('--dry-run')
if data['warnings']:
cmd.append('--warnings')
if typ == 'true':
cmd.append('--warnings')
else:
cmd.append('-W')
return cmd
@ -479,6 +525,54 @@ cd {prefix}
self.request.session['rattail.importing.runjob.notes'] = markdown.markdown(
notes, extensions=['fenced_code', 'codehilite'])
def configure_get_context(self):
app = self.get_rattail_app()
handlers_data = []
for handler in app.get_designated_import_handlers(
with_alternates=True,
ignore_errors=True, sort=True):
data = self.normalize(handler, keep_handler=False)
data['spec_options'] = [handler.get_spec()]
for alternate in handler.alternate_handlers:
data['spec_options'].append(alternate.get_spec())
data['spec_options'].sort()
handlers_data.append(data)
return {
'handlers_data': handlers_data,
}
def configure_gather_settings(self, data):
settings = []
for handler in data['handlers']:
key = handler['key']
settings.extend([
{'name': 'rattail.importing.{}.handler'.format(key),
'value': handler['handler_spec']},
{'name': 'rattail.importing.{}.cmd'.format(key),
'value': '{} {}'.format(handler['command'],
handler['subcommand'])},
{'name': 'rattail.importing.{}.runas'.format(key),
'value': handler['default_runas']},
])
return settings
def configure_remove_settings(self):
model = self.model
self.Session.query(model.Setting)\
.filter(sa.or_(
model.Setting.name.like('rattail.importing.%.handler'),
model.Setting.name.like('rattail.importing.%.cmd'),
model.Setting.name.like('rattail.importing.%.runas')))\
.delete(synchronize_session=False)
@classmethod
def defaults(cls, config):
cls._defaults(config)
@ -493,7 +587,7 @@ cd {prefix}
# run job
config.add_tailbone_permission(permission_prefix,
'{}.runjob'.format(permission_prefix),
"Run an arbitrary import / export job")
"Run an arbitrary Import / Export Job")
config.add_route('{}.runjob'.format(route_prefix),
'{}/runjob'.format(instance_url_prefix))
config.add_view(cls, attr='runjob',

65
tailbone/views/master.py
View file

@ -114,6 +114,7 @@ class MasterView(View):
execute_progress_initial_msg = None
supports_prev_next = False
supports_import_batch_from_file = False
configurable = False
# set to True to add "View *global* Objects" permission, and
# expose / leverage the ``local_only`` object flag
@ -2032,6 +2033,16 @@ class MasterView(View):
"""
return getattr(cls, 'index_title', cls.get_model_title_plural())
@classmethod
def get_config_title(cls):
"""
Returns the view's "config title".
"""
if hasattr(cls, 'config_title'):
return cls.config_title
return cls.get_model_title_plural()
def get_action_url(self, action, instance, **kwargs):
"""
Generate a URL for the given action on the given instance
@ -2075,6 +2086,7 @@ class MasterView(View):
'permission_prefix': self.get_permission_prefix(),
'index_title': self.get_index_title(),
'index_url': self.get_index_url(),
'config_title': self.get_config_title(),
'action_url': self.get_action_url,
'grid_index': self.grid_index,
'help_url': self.get_help_url(),
@ -3982,7 +3994,46 @@ class MasterView(View):
return diffs.Diff(old_data, new_data, **kwargs)
##############################
# Config Stuff
# Configuration Views
##############################
def configure(self):
"""
Generic view for configuring some aspect of the software.
"""
if self.request.method == 'POST':
if self.request.POST.get('remove_settings'):
self.configure_remove_settings()
self.request.session.flash("Settings have been removed.")
return self.redirect(self.request.current_route_url())
else:
data = self.request.json_body
settings = self.configure_gather_settings(data)
self.configure_remove_settings()
self.configure_save_settings(settings)
self.request.session.flash("Settings have been saved.")
return self.json_response({'success': True})
context = self.configure_get_context()
return self.render_to_response('configure', context)
def configure_get_context(self):
return {}
def configure_gather_settings(self, data):
return []
def configure_remove_settings(self):
pass
def configure_save_settings(self, settings):
model = self.model
for setting in settings:
self.Session.add(model.Setting(name=setting['name'],
value=setting['value']))
##############################
# Pyramid View Config
##############################
@classmethod
@ -4025,6 +4076,7 @@ class MasterView(View):
model_key = cls.get_model_key()
model_title = cls.get_model_title()
model_title_plural = cls.get_model_title_plural()
config_title = cls.get_config_title()
if cls.has_rows:
row_model_title = cls.get_row_model_title()
@ -4087,6 +4139,17 @@ class MasterView(View):
config.add_view(cls, attr='download_results_rows', route_name='{}.download_results_rows'.format(route_prefix),
permission='{}.download_results_rows'.format(permission_prefix))
# configure
if cls.configurable:
config.add_tailbone_permission(permission_prefix,
'{}.configure'.format(permission_prefix),
label="Configure {}".format(config_title))
config.add_route('{}.configure'.format(route_prefix),
'{}/configure'.format(url_prefix))
config.add_view(cls, attr='configure',
route_name='{}.configure'.format(route_prefix),
permission='{}.configure'.format(permission_prefix))
# quickie (search)
if cls.supports_quickie_search:
config.add_tailbone_permission(permission_prefix, '{}.quickie'.format(permission_prefix),

58
tailbone/views/permissions.py Normal file
View file

@ -0,0 +1,58 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2021 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Raw Permission Views
"""
from __future__ import unicode_literals, absolute_import
from sqlalchemy import orm
from rattail.db import model
from tailbone.views import MasterView
class PermissionView(MasterView):
"""
Master view for the permissions model.
"""
model_class = model.Permission
model_title = "Raw Permission"
editable = False
bulk_deletable = True
grid_columns = [
'role',
'permission',
]
def query(self, session):
model = self.model
query = super(PermissionView, self).query(session)
query = query.options(orm.joinedload(model.Permission.role))
return query
def includeme(config):
PermissionView.defaults(config)