rattail/tailbone
2
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

Add websockets progress, "multi-system" support for upgrades

Browse source 
and related things to better support that
This commit is contained in:
Lance Edgar 2022-08-20 17:39:33 -05:00
parent db3f215ebe
commit 18cec49a86
12 changed files with 731 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

100
tailbone/views/asgi/__init__.py
View file

@ -24,26 +24,77 @@
ASGI Views
"""
from __future__ import unicode_literals, absolute_import
from http.cookies import SimpleCookie
import http.cookies
from beaker.session import SignedCookie
from pyramid.interfaces import ISessionFactory
from beaker.cache import clsmap
from beaker.session import SessionObject, SignedCookie
class MockRequest(dict):
"""
Fake request class, needed for re-construction of the user's web
session.
"""
environ = {}
def add_response_callback(self, func):
pass
class WebsocketView(object):
def __init__(self, registry):
self.registry = registry
def __init__(self, pyramid_config):
self.pyramid_config = pyramid_config
self.registry = self.pyramid_config.registry
self.model = self.rattail_config.get_model()
@property
def rattail_config(self):
return self.registry['rattail_config']
def get_rattail_app(self):
return self.rattail_config.get_app()
async def authorize(self, scope, receive, send, permission):
# is user authorized for this socket?
authorized = await self.has_permission(scope, permission)
# wait for client to connect
message = await receive()
assert message['type'] == 'websocket.connect'
# allow or deny access, per authorization
if authorized:
await send({'type': 'websocket.accept'})
else: # forbidden
await send({'type': 'websocket.close'})
return authorized
async def get_user(self, scope, session=None):
app = self.get_rattail_app()
model = self.model
# load the user's web session
user_session = await self.get_user_session(scope)
if user_session:
# determine user uuid
user_uuid = user_session.get('auth.userid')
if user_uuid:
# use given db session, or make a new one
with app.short_session(config=self.rattail_config,
session=session):
# load user proper
return session.query(model.User).get(user_uuid)
async def get_user_session(self, scope):
settings = self.registry.settings
beaker_key = settings['beaker.session.key']
beaker_secret = settings['beaker.session.secret']
beaker_type = settings['beaker.session.type']
beaker_data_dir = settings['beaker.session.data_dir']
beaker_lock_dir = settings['beaker.session.lock_dir']
# get ahold of session identifier cookie
headers = dict(scope['headers'])
@ -51,20 +102,31 @@ class WebsocketView(object):
if not cookie:
return
cookie = cookie.decode('utf_8')
cookie = http.cookies.SimpleCookie(cookie)
cookie = SimpleCookie(cookie)
morsel = cookie[beaker_key]
# simulate pyramid_beaker logic to get at the session
# simulate pyramid_beaker logic to get at the actual session
cookieheader = morsel.output(header='')
cookie = SignedCookie(beaker_secret, input=cookieheader)
session_id = cookie[beaker_key].value
request = {'cookie': cookieheader}
session = SessionObject(
request,
id=session_id,
key=beaker_key,
namespace_class=clsmap[beaker_type],
data_dir=beaker_data_dir,
lock_dir=beaker_lock_dir)
factory = self.registry.queryUtility(ISessionFactory)
request = MockRequest()
# nb. cannot pass 'id' to our factory, but things still work
# if we assign it immediately, before load() is called
session = factory(request)
session.id = session_id
session.load()
return session
async def has_permission(self, scope, permission):
app = self.get_rattail_app()
auth_handler = app.get_auth_handler()
# figure out if user is authorized for this websocket
session = app.make_session()
user = await self.get_user(scope, session=session)
authorized = auth_handler.has_permission(session, user, permission)
session.close()
return authorized

33
tailbone/views/asgi/datasync.py
View file

@ -24,8 +24,6 @@
DataSync Views
"""
from __future__ import unicode_literals, absolute_import
import asyncio
import json
@ -35,36 +33,11 @@ from tailbone.views.asgi import WebsocketView
class DatasyncWS(WebsocketView):
async def status(self, scope, receive, send):
rattail_config = self.registry['rattail_config']
app = rattail_config.get_app()
model = app.model
auth_handler = app.get_auth_handler()
app = self.get_rattail_app()
datasync_handler = app.get_datasync_handler()
authorized = False
user_session = await self.get_user_session(scope)
if user_session:
user_uuid = user_session.get('auth.userid')
session = app.make_session()
user = None
if user_uuid:
user = session.query(model.User).get(user_uuid)
# figure out if user is authorized for this websocket
permission = 'datasync.status'
authorized = auth_handler.has_permission(session, user, permission)
session.close()
# wait for client to connect
message = await receive()
assert message['type'] == 'websocket.connect'
# allow or deny access, per authorization
if authorized:
await send({'type': 'websocket.accept'})
else: # forbidden
await send({'type': 'websocket.close'})
# is user allowed to see this?
if not await self.authorize(scope, receive, send, 'datasync.status'):
return
# this tracks when client disconnects

131
tailbone/views/asgi/upgrades.py Normal file
View file

@ -0,0 +1,131 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2022 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Upgrade Views for ASGI
"""
import asyncio
import json
import os
from urllib.parse import parse_qs
from tailbone.views.asgi import WebsocketView
from tailbone.progress import get_basic_session
class UpgradeWS(WebsocketView):
async def execution_progress(self, scope, receive, send):
rattail_config = self.registry['rattail_config']
# is user allowed to see this?
if not await self.authorize(scope, receive, send, 'upgrades.execute'):
return
# this tracks when client disconnects
state = {'disconnected': False}
async def wait_for_disconnect():
message = await receive()
if message['type'] == 'websocket.disconnect':
state['disconnected'] = True
# watch for client disconnect, while we do other things
asyncio.create_task(wait_for_disconnect())
query = scope['query_string'].decode('utf_8')
query = parse_qs(query)
uuid = query['uuid'][0]
progress_session_id = 'upgrades.{}.execution_progress'.format(uuid)
progress_session = get_basic_session(rattail_config,
id=progress_session_id)
# do the rest forever, until client disconnects
while not state['disconnected']:
# load latest progress data
progress_session.load()
# when upgrade progress is complete...
if progress_session.get('complete'):
# maybe set success flash msg
msg = progress_session.get('success_msg')
if msg:
user_session = await self.get_user_session(scope)
user_session.flash(msg)
user_session.persist()
# tell client progress is complete
await send({'type': 'websocket.send',
'subtype': 'upgrades.execute_progress',
'text': json.dumps({'complete': True})})
# this websocket is done
break
# we will send this data down to client
data = dict(progress_session)
# maybe add more lines from command output
path = rattail_config.upgrade_filepath(uuid, filename='stdout.log')
offset = progress_session.get('stdout.offset', 0)
if os.path.exists(path):
size = os.path.getsize(path) - offset
if size > 0:
with open(path, 'rb') as f:
f.seek(offset)
chunk = f.read(size)
data['stdout'] = chunk.decode('utf8').replace('\n', '<br />')
progress_session['stdout.offset'] = offset + size
progress_session.save()
# send data to client
await send({'type': 'websocket.send',
'subtype': 'upgrades.execute_progress',
'text': json.dumps(data)})
# pause for 1 second
await asyncio.sleep(1)
@classmethod
def defaults(cls, config):
cls._defaults(config)
@classmethod
def _defaults(cls, config):
# execution progress
config.add_tailbone_websocket('upgrades.execution_progress',
cls, attr='execution_progress')
def defaults(config, **kwargs):
base = globals()
UpgradeWS = kwargs.get('UpgradeWS', base['UpgradeWS'])
UpgradeWS.defaults(config)
def includeme(config):
defaults(config)

6
tailbone/views/core.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2021 Lance Edgar
# Copyright © 2010-2022 Lance Edgar
#
# This file is part of Rattail.
#
@ -134,12 +134,12 @@ class View(object):
def progress_loop(self, func, items, factory, *args, **kwargs):
return progress_loop(func, items, factory, *args, **kwargs)
def make_progress(self, key):
def make_progress(self, key, **kwargs):
"""
Create and return a :class:`tailbone.progress.SessionProgress`
instance, with the given key.
"""
return SessionProgress(self.request, key)
return SessionProgress(self.request, key, **kwargs)
# TODO: this signature seems wonky
def render_progress(self, progress, kwargs, template=None):

27
tailbone/views/master.py
View file

@ -1790,14 +1790,28 @@ class MasterView(View):
"""
obj = self.get_instance()
model_title = self.get_model_title()
progress = self.make_execute_progress(obj)
# caller must explicitly request websocket behavior; otherwise
# we will assume traditional behavior for progress
ws = self.request.is_xhr and self.request.json_body.get('ws')
# make our progress tracker
progress = self.make_execute_progress(obj, ws=ws)
# start execution in a separate thread
kwargs = {'progress': progress}
key = [self.request.matchdict[k]
for k in self.get_model_key(as_tuple=True)]
thread = Thread(target=self.execute_thread, args=(key, self.request.user.uuid), kwargs=kwargs)
thread = Thread(target=self.execute_thread,
args=(key, self.request.user.uuid),
kwargs=kwargs)
thread.start()
# we're done here if using websockets
if ws:
return self.json_response({'ok': True})
# traditional behavior sends user to dedicated progress page
return self.render_progress(progress, {
'instance': obj,
'initial_msg': self.execute_progress_initial_msg,
@ -1805,9 +1819,12 @@ class MasterView(View):
'cancel_msg': "{} execution was canceled".format(model_title),
}, template=self.execute_progress_template)
def make_execute_progress(self, obj):
key = '{}.execute'.format(self.get_grid_key())
return self.make_progress(key)
def make_execute_progress(self, obj, ws=False):
if ws:
key = '{}.{}.execution_progress'.format(self.get_route_prefix(), obj.uuid)
else:
key = '{}.execute'.format(self.get_grid_key())
return self.make_progress(key, ws=ws)
def get_instance_for_key(self, key, session):
model_key = self.get_model_key(as_tuple=True)

135
tailbone/views/upgrades.py
View file

@ -26,24 +26,27 @@ Views for app upgrades
from __future__ import unicode_literals, absolute_import
import json
import os
import re
import logging
import warnings
import six
from sqlalchemy import orm
import sqlalchemy as sa
from rattail.core import Object
from rattail.db import model, Session as RattailSession
from rattail.time import make_utc
from rattail.threads import Thread
from rattail.upgrades import get_upgrade_handler
from rattail.util import OrderedDict
from deform import widget as dfwidget
from webhelpers2.html import tags, HTML
from tailbone.views import MasterView
from tailbone.progress import get_progress_session #, SessionProgress
from tailbone.config import should_expose_websockets
log = logging.getLogger(__name__)
@ -56,6 +59,7 @@ class UpgradeView(MasterView):
model_class = model.Upgrade
downloadable = True
cloneable = True
configurable = True
executable = True
execute_progress_template = '/upgrade.mako'
execute_progress_initial_msg = "Upgrading"
@ -68,6 +72,7 @@ class UpgradeView(MasterView):
}
grid_columns = [
'system',
'created',
'description',
# 'not_until',
@ -78,6 +83,7 @@ class UpgradeView(MasterView):
]
form_fields = [
'system',
'description',
# 'not_until',
# 'requirements',
@ -97,28 +103,40 @@ class UpgradeView(MasterView):
def __init__(self, request):
super(UpgradeView, self).__init__(request)
self.handler = self.get_handler()
def get_handler(self):
"""
Returns the ``UpgradeHandler`` instance for the view. The handler
factory for this may be defined by config, e.g.:
if hasattr(self, 'get_handler'):
warnings.warn("defining get_handler() is deprecated. please "
"override AppHandler.get_upgrade_handler() instead",
DeprecationWarning, stacklevel=2)
self.upgrade_handler = self.get_handler()
.. code-block:: ini
else:
app = self.get_rattail_app()
self.upgrade_handler = app.get_upgrade_handler()
[rattail.upgrades]
handler = myapp.upgrades:CustomUpgradeHandler
"""
return get_upgrade_handler(self.rattail_config)
@property
def handler(self):
warnings.warn("handler attribute is deprecated; "
"please use upgrade_handler instead",
DeprecationWarning, stacklevel=2)
return self.upgrade_handler
def configure_grid(self, g):
super(UpgradeView, self).configure_grid(g)
# system
systems = self.upgrade_handler.get_all_systems()
systems_enum = dict([(s['key'], s['label']) for s in systems])
g.set_enum('system', systems_enum)
g.set_joiner('executed_by', lambda q: q.join(model.User, model.User.uuid == model.Upgrade.executed_by_uuid).outerjoin(model.Person))
g.set_sorter('executed_by', model.Person.display_name)
g.set_enum('status_code', self.enum.UPGRADE_STATUS)
g.set_type('created', 'datetime')
g.set_type('executed', 'datetime')
g.set_sort_defaults('created', 'desc')
g.set_link('system')
g.set_link('created')
g.set_link('description')
# g.set_link('not_until')
@ -157,6 +175,16 @@ class UpgradeView(MasterView):
super(UpgradeView, self).configure_form(f)
upgrade = f.model_instance
# system
systems = self.upgrade_handler.get_all_systems()
systems_enum = OrderedDict([(s['key'], s['label'])
for s in systems])
f.set_enum('system', systems_enum)
f.set_required('system')
if self.creating:
if len(systems) == 1:
f.set_default('system', list(systems_enum)[0])
# status_code
if self.creating:
f.remove_field('status_code')
@ -174,7 +202,15 @@ class UpgradeView(MasterView):
f.set_widget('notes', dfwidget.TextAreaWidget(cols=80, rows=8))
f.set_renderer('stdout_file', self.render_stdout_file)
f.set_renderer('stderr_file', self.render_stdout_file)
f.set_renderer('package_diff', self.render_package_diff)
# package_diff
if self.viewing and upgrade.executed and (
upgrade.system == 'rattail'
or not upgrade.system):
f.set_renderer('package_diff', self.render_package_diff)
else:
f.remove_field('package_diff')
# f.set_readonly('created')
# f.set_readonly('created_by')
f.set_readonly('executed')
@ -202,7 +238,6 @@ class UpgradeView(MasterView):
f.set_default('enabled', True)
if not self.viewing or not upgrade.executed:
f.remove_field('package_diff')
f.remove_field('exit_code')
def render_status_code(self, upgrade, field):
@ -233,10 +268,11 @@ class UpgradeView(MasterView):
return text
def configure_clone_form(self, f):
f.fields = ['description', 'notes', 'enabled']
f.fields = ['system', 'description', 'notes', 'enabled']
def clone_instance(self, original):
cloned = self.model_class()
cloned.system = original.system
cloned.created = make_utc()
cloned.created_by = self.request.user
cloned.description = original.description
@ -439,13 +475,22 @@ class UpgradeView(MasterView):
# key = '{}.execute'.format(self.get_grid_key())
# return SessionProgress(self.request, key, session_type='file')
def execute_instance(self, upgrade, user, **kwargs):
session = orm.object_session(upgrade)
self.handler.mark_executing(upgrade)
def execute_instance(self, upgrade, user, progress=None, **kwargs):
app = self.get_rattail_app()
session = app.get_session(upgrade)
# record the fact that execution has begun for this ugprade
self.upgrade_handler.mark_executing(upgrade)
session.commit()
self.handler.do_execute(upgrade, user, **kwargs)
return ("Execution has finished, for better or worse. "
"You may need to restart your web app.")
# let handler execute the upgrade
self.upgrade_handler.do_execute(upgrade, user, **kwargs)
# success msg
msg = "Execution has finished, for better or worse."
if not upgrade.system or upgrade.system == 'rattail':
msg += " You may need to restart your web app."
return msg
def execute_progress(self):
upgrade = self.get_instance()
@ -489,6 +534,50 @@ class UpgradeView(MasterView):
self.handler.delete_files(upgrade)
super(UpgradeView, self).delete_instance(upgrade)
def configure_get_context(self, **kwargs):
context = super(UpgradeView, self).configure_get_context(**kwargs)
context['upgrade_systems'] = self.upgrade_handler.get_all_systems()
return context
def configure_gather_settings(self, data):
settings = super(UpgradeView, self).configure_gather_settings(data)
keys = []
for system in json.loads(data['upgrade_systems']):
key = system['key']
if key == 'rattail':
settings.append({'name': 'rattail.upgrades.command',
'value': system['command']})
else:
keys.append(key)
settings.append({'name': 'rattail.upgrades.system.{}.label'.format(key),
'value': system['label']})
settings.append({'name': 'rattail.upgrades.system.{}.command'.format(key),
'value': system['command']})
if keys:
settings.append({'name': 'rattail.upgrades.systems',
'value': ', '.join(keys)})
return settings
def configure_remove_settings(self):
super(UpgradeView, self).configure_remove_settings()
app = self.get_rattail_app()
model = self.model
to_delete = self.Session.query(model.Setting)\
.filter(sa.or_(
model.Setting.name == 'rattail.upgrades.command',
model.Setting.name == 'rattail.upgrades.systems',
model.Setting.name.like('rattail.upgrades.system.%.label'),
model.Setting.name.like('rattail.upgrades.system.%.command')))\
.all()
for setting in to_delete:
app.delete_setting(self.Session(), setting.name)
@classmethod
def defaults(cls, config):
cls._defaults(config)
@ -520,10 +609,14 @@ class UpgradeView(MasterView):
def defaults(config, **kwargs):
base = globals()
rattail_config = config.registry['rattail_config']
UpgradeView = kwargs.get('UpgradeView', base['UpgradeView'])
UpgradeView.defaults(config)
if should_expose_websockets(rattail_config):
config.include('tailbone.views.asgi.upgrades')
def includeme(config):
defaults(config)