wutta/wuttaweb
3
0
Fork 0
Code Issues Activity

feat: add basic support for execute upgrades, download stdout/stderr

Browse source 
upgrade progress is still not being shown yet
This commit is contained in:
Lance Edgar 2024-08-25 12:20:28 -05:00
parent 1a8900c9f4
commit e5e31a7d32
17 changed files with 805 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/wuttaweb/forms/schema.py
View file

@ -456,3 +456,35 @@ class Permissions(WuttaSet):
kwargs['values'] = values
return widgets.PermissionsWidget(self.request, **kwargs)
class FileDownload(colander.String):
"""
Custom schema type for a file download field.
This field is only meant for readonly use, it does not handle file
uploads.
It expects the incoming ``appstruct`` to be the path to a file on
disk (or null).
Uses the :class:`~wuttaweb.forms.widgets.FileDownloadWidget` by
default.
:param request: Current :term:`request` object.
:param url: Optional URL for hyperlink. If not specified, file
name/size is shown with no hyperlink.
"""
def __init__(self, request, *args, **kwargs):
self.url = kwargs.pop('url', None)
super().__init__(*args, **kwargs)
self.request = request
self.config = self.request.wutta_config
self.app = self.config.get_app()
def widget_maker(self, **kwargs):
""" """
kwargs.setdefault('url', self.url)
return widgets.FileDownloadWidget(self.request, **kwargs)

66
src/wuttaweb/forms/widgets.py
View file

@ -39,7 +39,10 @@ in the namespace:
* :class:`deform:deform.widget.MoneyInputWidget`
"""
import os
import colander
import humanize
from deform.widget import (Widget, TextInputWidget, TextAreaWidget,
PasswordWidget, CheckedPasswordWidget,
CheckboxWidget, SelectWidget, CheckboxChoiceWidget,
@ -147,6 +150,63 @@ class WuttaCheckboxChoiceWidget(CheckboxChoiceWidget):
self.session = session or Session()
class FileDownloadWidget(Widget):
"""
Widget for use with :class:`~wuttaweb.forms.schema.FileDownload`
fields.
This only supports readonly, and shows a hyperlink to download the
file. Link text is the filename plus file size.
This is a subclass of :class:`deform:deform.widget.Widget` and
uses these Deform templates:
* ``readonly/filedownload``
:param request: Current :term:`request` object.
:param url: Optional URL for hyperlink. If not specified, file
name/size is shown with no hyperlink.
"""
readonly_template = 'readonly/filedownload'
def __init__(self, request, *args, **kwargs):
self.url = kwargs.pop('url', None)
super().__init__(*args, **kwargs)
self.request = request
self.config = self.request.wutta_config
self.app = self.config.get_app()
def serialize(self, field, cstruct, **kw):
""" """
# nb. readonly is the only way this rolls
kw['readonly'] = True
template = self.readonly_template
path = cstruct or None
if path:
kw.setdefault('filename', os.path.basename(path))
kw.setdefault('filesize', self.readable_size(path))
if self.url:
kw.setdefault('url', self.url)
else:
kw.setdefault('filename', None)
kw.setdefault('filesize', None)
kw.setdefault('url', None)
values = self.get_template_values(field, cstruct, kw)
return field.renderer(template, **values)
def readable_size(self, path):
""" """
try:
size = os.path.getsize(path)
except os.error:
size = 0
return humanize.naturalsize(size)
class RoleRefsWidget(WuttaCheckboxChoiceWidget):
"""
Widget for use with User
@ -184,7 +244,7 @@ class RoleRefsWidget(WuttaCheckboxChoiceWidget):
roles = []
if cstruct:
for uuid in cstruct:
role = self.session.query(model.Role).get(uuid)
role = self.session.get(model.Role, uuid)
if role:
roles.append(role)
kw['roles'] = roles
@ -228,6 +288,10 @@ class UserRefsWidget(WuttaCheckboxChoiceWidget):
users.append(dict([(key, getattr(user, key))
for key in columns + ['uuid']]))
# do not render if no data
if not users:
return HTML.tag('span')
# grid
grid = Grid(self.request, key='roles.view.users',
columns=columns, data=users)

14
src/wuttaweb/templates/deform/readonly/filedownload.pt Normal file
View file

@ -0,0 +1,14 @@
<tal:omit>
<a tal:condition="url" href="${url}">
${filename}
<tal:omit tal:condition="filesize">
(${filesize})
</tal:omit>
</a>
<span tal:condition="not url">
${filename}
<tal:omit tal:condition="filesize">
(${filesize})
</tal:omit>
</span>
</tal:omit>

20
src/wuttaweb/templates/upgrades/configure.mako Normal file
View file

@ -0,0 +1,20 @@
## -*- coding: utf-8; -*-
<%inherit file="/master/configure.mako" />
<%def name="form_content()">
<h3 class="is-size-3">Basics</h3>
<div class="block" style="padding-left: 2rem; width: 50%;">
<b-field label="Upgrade Script (for Execute)"
message="The command + args will be interpreted by the shell.">
<b-input name="${app.appname}.upgrades.command"
v-model="simpleSettings['${app.appname}.upgrades.command']"
@input="settingsNeedSaved = true"
## ref="upgradeSystemCommand"
## expanded
/>
</b-field>
</div>
</%def>

37
src/wuttaweb/templates/upgrades/view.mako Normal file
View file

@ -0,0 +1,37 @@
## -*- coding: utf-8; -*-
<%inherit file="/master/view.mako" />
<%def name="page_content()">
${parent.page_content()}
% if instance.status == app.enum.UpgradeStatus.PENDING and master.has_perm('execute'):
<div class="buttons"
style="margin: 2rem 5rem;">
${h.form(master.get_action_url('execute', instance), **{'@submit': 'executeFormSubmit'})}
${h.csrf_token(request)}
<b-button type="is-primary"
native-type="submit"
icon-pack="fas"
icon-left="arrow-circle-right"
:disabled="executeFormSubmitting">
{{ executeFormSubmitting ? "Working, please wait..." : "Execute this upgrade" }}
</b-button>
${h.end_form()}
</div>
% endif
</%def>
<%def name="modify_vue_vars()">
${parent.modify_vue_vars()}
% if instance.status == app.enum.UpgradeStatus.PENDING and master.has_perm('execute'):
<script>
ThisPageData.executeFormSubmitting = false
ThisPage.methods.executeFormSubmit = function() {
this.executeFormSubmitting = true
}
</script>
% endif
</%def>

42
src/wuttaweb/views/base.py
View file

@ -24,8 +24,11 @@
Base Logic for Views
"""
import os
from pyramid import httpexceptions
from pyramid.renderers import render_to_response
from pyramid.response import FileResponse
from wuttaweb import forms, grids
@ -119,9 +122,46 @@ class View:
"""
return httpexceptions.HTTPFound(location=url, **kwargs)
def file_response(self, path, attachment=True, filename=None):
"""
Returns a generic file response for the given path.
:param path: Path to a file on local disk; must be accessible
by the web app.
:param attachment: Whether the file should come down as an
"attachment" instead of main payload.
The attachment behavior is the default here, and will cause
the user to be prompted for where to save the file.
Set ``attachment=False`` in order to cause the browser to
render the file as if it were the page being navigated to.
:param filename: Optional filename to use for attachment
behavior. This will be the "suggested filename" when user
is prompted to save the download. If not specified, the
filename is derived from ``path``.
:returns: A :class:`~pyramid:pyramid.response.FileResponse`
object with file content.
"""
if not os.path.exists(path):
return self.notfound()
response = FileResponse(path, request=self.request)
response.content_length = os.path.getsize(path)
if attachment:
if not filename:
filename = os.path.basename(path)
response.content_disposition = f'attachment; filename="{filename}"'
return response
def json_response(self, context):
"""
Convenience method to return a JSON response.
Returns a JSON response with the given context data.
:param context: Context data to be rendered as JSON.

3
src/wuttaweb/views/common.py
View file

@ -160,6 +160,9 @@ class CommonView(View):
'upgrades.view',
'upgrades.edit',
'upgrades.delete',
'upgrades.execute',
'upgrades.download',
'upgrades.configure',
'users.list',
'users.create',
'users.view',

158
src/wuttaweb/views/master.py
View file

@ -25,6 +25,7 @@ Base Logic for Master Views
"""
import logging
import os
import threading
import sqlalchemy as sa
@ -322,6 +323,18 @@ class MasterView(View):
"autocomplete" - i.e. it should have an :meth:`autocomplete()`
view. Default is ``False``.
.. attribute:: downloadable
Boolean indicating whether the view model supports
"downloading" - i.e. it should have a :meth:`download()` view.
Default is ``False``.
.. attribute:: executable
Boolean indicating whether the view model supports "executing"
- i.e. it should have an :meth:`execute()` view. Default is
``False``.
.. attribute:: configurable
Boolean indicating whether the master view supports
@ -350,6 +363,8 @@ class MasterView(View):
deletable_bulk = False
deletable_bulk_quick = False
has_autocomplete = False
downloadable = False
executable = False
configurable = False
# current action
@ -842,6 +857,126 @@ class MasterView(View):
'label': str(obj),
}
##############################
# download methods
##############################
def download(self):
"""
View to download a file associated with a model record.
This usually corresponds to a URL like
``/widgets/XXX/download`` where ``XXX`` represents the key/ID
for the record.
By default, this view is included only if :attr:`downloadable`
is true.
This method will (try to) locate the file on disk, and return
it as a file download response to the client.
The GET request for this view may contain a ``filename`` query
string parameter, which can be used to locate one of various
files associated with the model record. This filename is
passed to :meth:`download_path()` for locating the file.
For instance: ``/widgets/XXX/download?filename=widget-specs.txt``
Subclass normally should not override this method, but rather
one of the related methods which are called (in)directly by
this one:
* :meth:`download_path()`
"""
obj = self.get_instance()
filename = self.request.GET.get('filename', None)
path = self.download_path(obj, filename)
if not path or not os.path.exists(path):
return self.notfound()
return self.file_response(path)
def download_path(self, obj, filename):
"""
Should return absolute path on disk, for the given object and
filename. Result will be used to return a file response to
client. This is called by :meth:`download()`.
Default logic always returns ``None``; subclass must override.
:param obj: Refefence to the model instance.
:param filename: Name of file for which to retrieve the path.
:returns: Path to file, or ``None`` if not found.
Note that ``filename`` may be ``None`` in which case the "default"
file path should be returned, if applicable.
If this method returns ``None`` (as it does by default) then
the :meth:`download()` view will return a 404 not found
response.
"""
##############################
# execute methods
##############################
def execute(self):
"""
View to "execute" a model record. Requires a POST request.
This usually corresponds to a URL like
``/widgets/XXX/execute`` where ``XXX`` represents the key/ID
for the record.
By default, this view is included only if :attr:`executable` is
true.
Probably this is a "rare" view to implement for a model. But
there are two notable use cases so far, namely:
* upgrades (cf. :class:`~wuttaweb.views.upgrades.UpgradeView`)
* batches (not yet implemented;
cf. :doc:`rattail-manual:data/batch/index` in Rattail
Manual)
The general idea is to take some "irrevocable" action
associated with the model record. In the case of upgrades, it
is to run the upgrade script. For batches it is to "push
live" the data held within the batch.
Subclass normally should not override this method, but rather
one of the related methods which are called (in)directly by
this one:
* :meth:`execute_instance()`
"""
model_title = self.get_model_title()
obj = self.get_instance()
try:
self.execute_instance(obj)
except Exception as error:
log.exception("failed to execute %s: %s", model_title, obj)
error = str(error) or error.__class__.__name__
self.request.session.flash(error, 'error')
else:
self.request.session.flash(f"{model_title} was executed.")
return self.redirect(self.get_action_url('view', obj))
def execute_instance(self, obj):
"""
Perform the actual "execution" logic for a model record.
Called by :meth:`execute()`.
This method does nothing by default; subclass must override.
:param obj: Reference to the model instance.
"""
##############################
# configure methods
##############################
@ -2370,6 +2505,29 @@ class MasterView(View):
renderer='json',
permission=f'{route_prefix}.list')
# download
if cls.downloadable:
config.add_route(f'{route_prefix}.download',
f'{instance_url_prefix}/download')
config.add_view(cls, attr='download',
route_name=f'{route_prefix}.download',
permission=f'{permission_prefix}.download')
config.add_wutta_permission(permission_prefix,
f'{permission_prefix}.download',
f"Download file(s) for {model_title}")
# execute
if cls.executable:
config.add_route(f'{route_prefix}.execute',
f'{instance_url_prefix}/execute',
request_method='POST')
config.add_view(cls, attr='execute',
route_name=f'{route_prefix}.execute',
permission=f'{permission_prefix}.execute')
config.add_wutta_permission(permission_prefix,
f'{permission_prefix}.execute',
f"Execute {model_title}")
# configure
if cls.configurable:
config.add_route(f'{route_prefix}.configure',

119
src/wuttaweb/views/upgrades.py
View file

@ -24,12 +24,21 @@
Upgrade Views
"""
import datetime
import logging
import os
import shutil
import subprocess
from sqlalchemy import orm
from wuttjamaican.db.model import Upgrade
from wuttaweb.views import MasterView
from wuttaweb.forms import widgets
from wuttaweb.forms.schema import UserRef, WuttaEnum
from wuttaweb.forms.schema import UserRef, WuttaEnum, FileDownload
log = logging.getLogger(__name__)
class UpgradeView(MasterView):
@ -47,6 +56,9 @@ class UpgradeView(MasterView):
* ``/upgrades/XXX/delete``
"""
model_class = Upgrade
executable = True
downloadable = True
configurable = True
grid_columns = [
'created',
@ -81,6 +93,9 @@ class UpgradeView(MasterView):
# status
g.set_renderer('status', self.grid_render_enum, enum=enum.UpgradeStatus)
# executed
g.set_renderer('executed', self.grid_render_datetime)
# executed_by
g.set_link('executed_by')
Executor = orm.aliased(model.User)
@ -138,10 +153,6 @@ class UpgradeView(MasterView):
else:
f.set_node('status', WuttaEnum(self.request, enum.UpgradeStatus))
# exit_code
if self.creating or not upgrade.executed:
f.remove('exit_code')
# executed
if self.creating or self.editing or not upgrade.executed:
f.remove('executed')
@ -152,6 +163,39 @@ class UpgradeView(MasterView):
else:
f.set_node('executed_by', UserRef(self.request))
# exit_code
if self.creating or self.editing or not upgrade.executed:
f.remove('exit_code')
# stdout / stderr
if not (self.creating or self.editing) and upgrade.status in (
enum.UpgradeStatus.SUCCESS, enum.UpgradeStatus.FAILURE):
# stdout_file
f.append('stdout_file')
f.set_label('stdout_file', "STDOUT")
url = self.get_action_url('download', upgrade, _query={'filename': 'stdout.log'})
f.set_node('stdout_file', FileDownload(self.request, url=url))
f.set_default('stdout_file', self.get_upgrade_filepath(upgrade, 'stdout.log'))
# stderr_file
f.append('stderr_file')
f.set_label('stderr_file', "STDERR")
url = self.get_action_url('download', upgrade, _query={'filename': 'stderr.log'})
f.set_node('stderr_file', FileDownload(self.request, url=url))
f.set_default('stderr_file', self.get_upgrade_filepath(upgrade, 'stderr.log'))
def delete_instance(self, upgrade):
"""
We override this method to delete any files associated with
the upgrade, in addition to deleting the upgrade proper.
"""
path = self.get_upgrade_filepath(upgrade, create=False)
if os.path.exists(path):
shutil.rmtree(path)
super().delete_instance(upgrade)
def objectify(self, form):
""" """
upgrade = super().objectify(form)
@ -164,6 +208,71 @@ class UpgradeView(MasterView):
return upgrade
def download_path(self, upgrade, filename):
""" """
if filename:
return self.get_upgrade_filepath(upgrade, filename)
def get_upgrade_filepath(self, upgrade, filename=None, create=True):
""" """
uuid = upgrade.uuid
path = self.app.get_appdir('data', 'upgrades', uuid[:2], uuid[2:],
create=create)
if filename:
path = os.path.join(path, filename)
return path
def execute_instance(self, upgrade):
"""
This method runs the actual upgrade.
Default logic will get the script command from config, and run
it via shell in a subprocess.
The ``stdout`` and ``stderr`` streams are captured to separate
log files which are then available to download.
The upgrade itself is marked as "executed" with status of
either ``SUCCESS`` or ``FAILURE``.
"""
enum = self.app.enum
script = self.config.require(f'{self.app.appname}.upgrades.command',
session=self.Session())
stdout_path = self.get_upgrade_filepath(upgrade, 'stdout.log')
stderr_path = self.get_upgrade_filepath(upgrade, 'stderr.log')
# run the command
log.debug("running upgrade command: %s", script)
with open(stdout_path, 'wb') as stdout:
with open(stderr_path, 'wb') as stderr:
upgrade.exit_code = subprocess.call(script, shell=True,
stdout=stdout, stderr=stderr)
logger = log.warning if upgrade.exit_code != 0 else log.debug
logger("upgrade command had non-zero exit code: %s", upgrade.exit_code)
# declare it complete
upgrade.executed = datetime.datetime.now()
upgrade.executed_by = self.request.user
if upgrade.exit_code == 0:
upgrade.status = enum.UpgradeStatus.SUCCESS
else:
upgrade.status = enum.UpgradeStatus.FAILURE
def configure_get_simple_settings(self):
""" """
script = self.config.get(f'{self.app.appname}.upgrades.command')
if not script:
pass
return [
# basics
{'name': f'{self.app.appname}.upgrades.command',
'default': script},
]
@classmethod
def defaults(cls, config):
""" """