Add websockets progress, "multi-system" support for upgrades

and related things to better support that

tailbone/app.py

@@ -195,22 +195,20 @@ def add_websocket(config, name, view, attr=None):
             view_callable = rattail_app.load_object(view)
         else:
             view_callable = view
-        view_callable = view_callable(config.registry)
+        view_callable = view_callable(config)
         if attr:
             view_callable = getattr(view_callable, attr)
 
-        path = '/ws/{}'.format(name)
-
         # register route
-        config.add_route('ws.{}'.format(name),
+        path = '/ws/{}'.format(name)
-                         path,
+        route_name = 'ws.{}'.format(name)
-                         static=True)
+        config.add_route(route_name, path, static=True)
 
         # register view callable
         websockets = config.registry.setdefault('tailbone_websockets', {})
         websockets[path] = view_callable
 
-    config.action('tailbone-add-websocket', action,
+    config.action('tailbone-add-websocket-{}'.format(name), action,
                   # nb. since this action adds routes, it must happen
                   # sooner in the order than it normally would, hence
                   # we declare that

tailbone/progress.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2018 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -27,22 +27,33 @@ Progress Indicator
 from __future__ import unicode_literals, absolute_import
 
 import os
+import warnings
 
 from rattail.progress import ProgressBase
 
 from beaker.session import Session
 
 
+def get_basic_session(config, request={}, **kwargs):
+    """
+    Create/get a "basic" Beaker session object.
+    """
+    kwargs['use_cookies'] = False
+    session = Session(request, **kwargs)
+    return session
+
+
 def get_progress_session(request, key, **kwargs):
     """
     Create/get a Beaker session object, to be used for progress.
     """
-    id = '{}.progress.{}'.format(request.session.id, key)
+    kwargs['id'] = '{}.progress.{}'.format(request.session.id, key)
-    kwargs['use_cookies'] = False
     if kwargs.get('type') == 'file':
+        warnings.warn("Passing a 'type' kwarg to get_progress_session() "
+                      "is deprecated...i think",
+                      DeprecationWarning, stacklevel=2)
         kwargs['data_dir'] = os.path.join(request.rattail_config.appdir(), 'sessions')
-    session = Session(request, id, **kwargs)
+    return get_basic_session(request.rattail_config, request, **kwargs)
-    return session
 
 
 class SessionProgress(ProgressBase):
@@ -52,11 +63,20 @@ class SessionProgress(ProgressBase):
     This class is only responsible for keeping the progress *data* current.  It
     is the responsibility of some client-side AJAX (etc.) to consume the data
     for display to the user.
+
+    :param ws: If true, then websockets are assumed, and the progress will
+       behave accordingly.  The default is false, "traditional" behavior.
     """
 
-    def __init__(self, request, key, session_type=None):
+    def __init__(self, request, key, session_type=None, ws=False):
         self.key = key
+        self.ws = ws
+
+        if self.ws:
+            self.session = get_basic_session(request.rattail_config, id=key)
+        else:
             self.session = get_progress_session(request, key, type=session_type)
+
         self.canceled = False
         self.clear()
 

tailbone/templates/forms/deform_buefy.mako

@@ -73,6 +73,7 @@
 
   let ${form.component_studly} = {
       template: '#${form.component}-template',
+      mixins: [FormPosterMixin],
       components: {},
       props: {},
       watch: {},

tailbone/templates/themes/falafel/base.mako

@@ -682,20 +682,54 @@
   % if show_prev_next is not Undefined and show_prev_next:
       % if prev_url:
           <div class="level-item">
+            % if use_buefy:
+                <b-button tag="a" href="${prev_url}"
+                          icon-pack="fas"
+                          icon-left="arrow-left">
+                  Older
+                </b-button>
+            % else:
                 ${h.link_to(u"« Older", prev_url, class_='button autodisable')}
+            % endif
           </div>
       % else:
           <div class="level-item">
+            % if use_buefy:
+                <b-button tag="a" href="#"
+                          disabled
+                          icon-pack="fas"
+                          icon-left="arrow-left">
+                  Older
+                </b-button>
+            % else:
                 ${h.link_to(u"« Older", '#', class_='button', disabled='disabled')}
+            % endif
           </div>
       % endif
       % if next_url:
           <div class="level-item">
+            % if use_buefy:
+                <b-button tag="a" href="${next_url}"
+                          icon-pack="fas"
+                          icon-left="arrow-right">
+                  Newer
+                </b-button>
+            % else:
                 ${h.link_to(u"Newer »", next_url, class_='button autodisable')}
+            % endif
           </div>
       % else:
           <div class="level-item">
+            % if use_buefy:
+                <b-button tag="a" href="#"
+                          disabled
+                          icon-pack="fas"
+                          icon-left="arrow-right">
+                  Newer
+                </b-button>
+            % else:
                 ${h.link_to(u"Newer »", '#', class_='button', disabled='disabled')}
+            % endif
           </div>
       % endif
   % endif

tailbone/templates/upgrades/configure.mako

@@ -0,0 +1,156 @@
+## -*- coding: utf-8; -*-
+<%inherit file="/configure.mako" />
+
+<%def name="form_content()">
+  ${h.hidden('upgrade_systems', **{':value': 'JSON.stringify(upgradeSystems)'})}
+
+  <h3 class="is-size-3">Upgradable Systems</h3>
+  <div class="block" style="padding-left: 2rem; display: flex;">
+
+    <b-table :data="upgradeSystems"
+             sortable>
+      <template slot-scope="props">
+        <b-table-column field="key"
+                        label="Key"
+                        sortable>
+          {{ props.row.key }}
+        </b-table-column>
+        <b-table-column field="label"
+                        label="Label"
+                        sortable>
+          {{ props.row.label }}
+        </b-table-column>
+        <b-table-column field="command"
+                        label="Command"
+                        sortable>
+          {{ props.row.command }}
+        </b-table-column>
+        <b-table-column label="Actions">
+          <a href="#"
+             @click.prevent="upgradeSystemEdit(props.row)">
+            <i class="fas fa-edit"></i>
+            Edit
+          </a>
+          &nbsp;
+          <a href="#"
+             v-if="props.row.key != 'rattail'"
+             class="has-text-danger"
+             @click.prevent="updateSystemDelete(props.row)">
+            <i class="fas fa-trash"></i>
+            Delete
+          </a>
+        </b-table-column>
+      </template>
+    </b-table>
+
+    <div style="margin-left: 1rem;">
+      <b-button type="is-primary"
+                icon-pack="fas"
+                icon-left="plus"
+                @click="upgradeSystemCreate()">
+        New System
+      </b-button>
+
+      <b-modal has-modal-card
+               :active.sync="upgradeSystemShowDialog">
+        <div class="modal-card">
+
+          <header class="modal-card-head">
+            <p class="modal-card-title">Upgradable System</p>
+          </header>
+
+          <section class="modal-card-body">
+            <b-field label="Key"
+                     :type="upgradeSystemKey ? null : 'is-danger'">
+              <b-input v-model.trim="upgradeSystemKey"
+                       ref="upgradeSystemKey"
+                       :disabled="upgradeSystemKey == 'rattail'">
+              </b-input>
+            </b-field>
+            <b-field label="Label"
+                     :type="upgradeSystemLabel ? null : 'is-danger'">
+              <b-input v-model.trim="upgradeSystemLabel"
+                       ref="upgradeSystemLabel"
+                       :disabled="upgradeSystemKey == 'rattail'">
+              </b-input>
+            </b-field>
+            <b-field label="Command"
+                     :type="upgradeSystemCommand ? null : 'is-danger'">
+              <b-input v-model.trim="upgradeSystemCommand"
+                       ref="upgradeSystemCommand">
+              </b-input>
+            </b-field>
+          </section>
+
+          <footer class="modal-card-foot">
+            <b-button type="is-primary"
+                      icon-pack="fas"
+                      icon-left="save"
+                      @click="upgradeSystemSave()"
+                      :disabled="!upgradeSystemKey || !upgradeSystemLabel || !upgradeSystemCommand">
+              Save
+            </b-button>
+            <b-button @click="upgradeSystemShowDialog = false">
+              Cancel
+            </b-button>
+          </footer>
+        </div>
+      </b-modal>
+
+    </div>
+  </div>
+</%def>
+
+<%def name="modify_this_page_vars()">
+  ${parent.modify_this_page_vars()}
+  <script type="text/javascript">
+
+    ThisPageData.upgradeSystems = ${json.dumps(upgrade_systems)|n}
+    ThisPageData.upgradeSystemShowDialog = false
+    ThisPageData.upgradeSystem = null
+    ThisPageData.upgradeSystemKey = null
+    ThisPageData.upgradeSystemLabel = null
+    ThisPageData.upgradeSystemCommand = null
+
+    ThisPage.methods.upgradeSystemCreate = function() {
+        this.upgradeSystem = null
+        this.upgradeSystemKey = null
+        this.upgradeSystemLabel = null
+        this.upgradeSystemCommand = null
+        this.upgradeSystemShowDialog = true
+        this.$nextTick(() => {
+            this.$refs.upgradeSystemKey.focus()
+        })
+    }
+
+    ThisPage.methods.upgradeSystemEdit = function(system) {
+        this.upgradeSystem = system
+        this.upgradeSystemKey = system.key
+        this.upgradeSystemLabel = system.label
+        this.upgradeSystemCommand = system.command
+        this.upgradeSystemShowDialog = true
+        this.$nextTick(() => {
+            this.$refs.upgradeSystemCommand.focus()
+        })
+    }
+
+    ThisPage.methods.upgradeSystemSave = function() {
+        if (this.upgradeSystem) {
+            this.upgradeSystem.key = this.upgradeSystemKey
+            this.upgradeSystem.label = this.upgradeSystemLabel
+            this.upgradeSystem.command = this.upgradeSystemCommand
+        } else {
+            let system = {key: this.upgradeSystemKey,
+                          label: this.upgradeSystemLabel,
+                          command: this.upgradeSystemCommand}
+            this.upgradeSystems.push(system)
+        }
+        this.upgradeSystemShowDialog = false
+        this.settingsNeedSaved = true
+    }
+
+  </script>
+</%def>
+
+
+${parent.body()}

tailbone/templates/upgrades/view.mako

@@ -38,6 +38,18 @@
   % endif
 </%def>
 
+<%def name="extra_styles()">
+  ${parent.extra_styles()}
+  <style type="text/css">
+    .progress-with-textout {
+        border: 1px solid Black;
+        line-height: 1.2;
+        overflow: auto;
+        padding: 1rem;
+    }
+  </style>
+</%def>
+
 <%def name="render_this_page()">
   ${parent.render_this_page()}
 
@@ -60,31 +72,86 @@
 </%def>
 
 <%def name="render_form_buttons()">
-  % if not instance.executed and instance.status_code == enum.UPGRADE_STATUS_PENDING and request.has_perm('{}.execute'.format(permission_prefix)):
+  % if not instance.executed and instance.status_code == enum.UPGRADE_STATUS_PENDING and master.has_perm('execute'):
       <div class="buttons">
         % if instance.enabled and not instance.executing:
-            % if use_buefy:
+            % if use_buefy and expose_websockets:
+                <b-button type="is-primary"
+                          icon-pack="fas"
+                          icon-left="arrow-circle-right"
+                          :disabled="upgradeExecuting"
+                          @click="executeUpgrade()">
+                  {{ upgradeExecuting ? "Working, please wait..." : "Execute this upgrade" }}
+                </b-button>
+            % elif use_buefy:
                 ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), **{'@submit': 'submitForm'})}
-            % else:
-            ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), class_='autodisable')}
-            % endif
                 ${h.csrf_token(request)}
-            % if use_buefy:
                 <b-button type="is-primary"
                           native-type="submit"
+                          icon-pack="fas"
+                          icon-left="arrow-circle-right"
                           :disabled="formSubmitting">
-                  {{ formButtonText }}
+                  {{ formSubmitting ? "Working, please wait..." : "Execute this upgrade" }}
                 </b-button>
-            % else:
-                ${h.submit('execute', "Execute this upgrade", class_='button is-primary')}
-            % endif
                 ${h.end_form()}
+            % else:
+                ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), class_='autodisable')}
+                ${h.csrf_token(request)}
+                ${h.submit('execute', "Execute this upgrade", class_='button is-primary')}
+                ${h.end_form()}
+            % endif
         % elif instance.enabled:
             <button type="button" class="button is-primary" disabled="disabled" title="This upgrade is currently executing">Execute this upgrade</button>
         % else:
             <button type="button" class="button is-primary" disabled="disabled" title="This upgrade is not enabled">Execute this upgrade</button>
         % endif
       </div>
+
+      <b-modal :active.sync="upgradeExecuting"
+               full-screen
+               :can-cancel="false">
+        <div class="card">
+          <div class="card-content">
+
+            <div class="level">
+              <div class="level-item has-text-centered"
+                   style="display: flex; flex-direction: column;">
+                <p class="block">Upgrading (please wait) ...</p>
+                <b-progress size="is-large"
+                            style="width: 400px;"
+##                             :value="80"
+##                             show-value
+##                             format="percent"
+                            >
+                </b-progress>
+              </div>
+              <div class="level-right">
+                <div class="level-item">
+                  <b-button type="is-warning"
+                            icon-pack="fas"
+                            icon-left="sad-tear"
+                            @click="declareFailureClick()">
+                    Declare Failure
+                  </b-button>
+                </div>
+              </div>
+            </div>
+
+            <div class="container progress-with-textout is-family-monospace is-size-7"
+                 ref="textout">
+              <span v-for="line in progressOutput"
+                    :key="line.key"
+                    v-html="line.text">
+              </span>
+
+              ## nb. we auto-scroll down to "see" this element
+              <div ref="seeme"></div>
+            </div>
+
+          </div>
+        </div>
+      </b-modal>
+
   % endif
 </%def>
 
@@ -94,15 +161,80 @@
 
     TailboneFormData.showingPackages = 'diffs'
 
-    TailboneFormData.formButtonText = "Execute this upgrade"
+    % if master.has_perm('execute'):
+
+        % if expose_websockets:
+
+            TailboneFormData.upgradeExecuting = ${json.dumps(instance.executing)|n}
+            TailboneFormData.progressOutput = []
+            TailboneFormData.progressOutputCounter = 0
+
+            TailboneForm.methods.executeUpgrade = function() {
+                this.upgradeExecuting = true
+
+                // grow the textout area to fill most of screen
+                this.$nextTick(() => {
+                    let textout = this.$refs.textout
+                    let height = window.innerHeight - textout.offsetTop - 50
+                    textout.style.height = height + 'px'
+                })
+
+                let url = '${master.get_action_url('execute', instance)}'
+                this.submitForm(url, {ws: true}, response => {
+
+                    ## TODO: should be a cleaner way to get this url?
+                    url = '${request.route_url('ws.upgrades.execution_progress', _query={'uuid': instance.uuid})}'
+                    url = url.replace(/^https?:/, 'wss:')
+
+                    this.ws = new WebSocket(url)
+                    let that = this
+
+                    ## TODO: add support for this here?
+                    // this.ws.onclose = (event) => {
+                    //     // websocket closing means 1 of 2 things:
+                    //     // - user navigated away from page intentionally
+                    //     // - server connection was broken somehow
+                    //     // only one of those is "bad" and we only want to
+                    //     // display warning in 2nd case.  so we simply use a
+                    //     // brief delay to "rule out" the 1st scenario
+                    //     setTimeout(() => { that.websocketBroken = true },
+                    //                3000)
+                    // }
+
+                    this.ws.onmessage = (event) => {
+                        let data = JSON.parse(event.data)
+
+                        if (data.complete) {
+
+                            // upgrade has completed; reload page to view result
+                            location.reload()
+
+                        } else if (data.stdout) {
+
+                            // add lines to textout area
+                            that.progressOutput.push({
+                                key: ++that.progressOutputCounter,
+                                text: data.stdout})
+
+                            //  scroll down to end of textout area
+                            this.$nextTick(() => {
+                                this.$refs.seeme.scrollIntoView()
+                            })
+                        }
+                    }
+                })
+            }
+
+        % else:
+            ## no websockets
+
             TailboneFormData.formSubmitting = false
 
             TailboneForm.methods.submitForm = function() {
                 this.formSubmitting = true
-        this.formButtonText = "Working, please wait..."
             }
 
-    % if master.has_perm('execute'):
+        % endif
 
         TailboneFormData.declareFailureSubmitting = false
 

tailbone/views/asgi/__init__.py

@@ -24,26 +24,77 @@
 ASGI Views
 """
 
-from __future__ import unicode_literals, absolute_import
+from http.cookies import SimpleCookie
 
-import http.cookies
+from beaker.session import SignedCookie
+from pyramid.interfaces import ISessionFactory
 
-from beaker.cache import clsmap
+
-from beaker.session import SessionObject, SignedCookie
+class MockRequest(dict):
+    """
+    Fake request class, needed for re-construction of the user's web
+    session.
+    """
+    environ = {}
+
+    def add_response_callback(self, func):
+        pass
 
 
 class WebsocketView(object):
 
-    def __init__(self, registry):
+    def __init__(self, pyramid_config):
-        self.registry = registry
+        self.pyramid_config = pyramid_config
+        self.registry = self.pyramid_config.registry
+        self.model = self.rattail_config.get_model()
+
+    @property
+    def rattail_config(self):
+        return self.registry['rattail_config']
+
+    def get_rattail_app(self):
+        return self.rattail_config.get_app()
+
+    async def authorize(self, scope, receive, send, permission):
+
+        # is user authorized for this socket?
+        authorized = await self.has_permission(scope, permission)
+
+        # wait for client to connect
+        message = await receive()
+        assert message['type'] == 'websocket.connect'
+
+        # allow or deny access, per authorization
+        if authorized:
+            await send({'type': 'websocket.accept'})
+        else: # forbidden
+            await send({'type': 'websocket.close'})
+
+        return authorized
+
+    async def get_user(self, scope, session=None):
+        app = self.get_rattail_app()
+        model = self.model
+
+        # load the user's web session
+        user_session = await self.get_user_session(scope)
+        if user_session:
+
+            # determine user uuid
+            user_uuid = user_session.get('auth.userid')
+            if user_uuid:
+
+                # use given db session, or make a new one
+                with app.short_session(config=self.rattail_config,
+                                       session=session):
+
+                    # load user proper
+                    return session.query(model.User).get(user_uuid)
 
     async def get_user_session(self, scope):
         settings = self.registry.settings
         beaker_key = settings['beaker.session.key']
         beaker_secret = settings['beaker.session.secret']
-        beaker_type = settings['beaker.session.type']
-        beaker_data_dir = settings['beaker.session.data_dir']
-        beaker_lock_dir = settings['beaker.session.lock_dir']
 
         # get ahold of session identifier cookie
         headers = dict(scope['headers'])
@@ -51,20 +102,31 @@ class WebsocketView(object):
         if not cookie:
             return
         cookie = cookie.decode('utf_8')
-        cookie = http.cookies.SimpleCookie(cookie)
+        cookie = SimpleCookie(cookie)
         morsel = cookie[beaker_key]
 
-        # simulate pyramid_beaker logic to get at the session
+        # simulate pyramid_beaker logic to get at the actual session
         cookieheader = morsel.output(header='')
         cookie = SignedCookie(beaker_secret, input=cookieheader)
         session_id = cookie[beaker_key].value
-        request = {'cookie': cookieheader}
+        factory = self.registry.queryUtility(ISessionFactory)
-        session = SessionObject(
+        request = MockRequest()
-            request,
+        # nb. cannot pass 'id' to our factory, but things still work
-            id=session_id,
+        # if we assign it immediately, before load() is called
-            key=beaker_key,
+        session = factory(request)
-            namespace_class=clsmap[beaker_type],
+        session.id = session_id
-            data_dir=beaker_data_dir,
+        session.load()
-            lock_dir=beaker_lock_dir)
 
         return session
+
+    async def has_permission(self, scope, permission):
+        app = self.get_rattail_app()
+        auth_handler = app.get_auth_handler()
+
+        # figure out if user is authorized for this websocket
+        session = app.make_session()
+        user = await self.get_user(scope, session=session)
+        authorized = auth_handler.has_permission(session, user, permission)
+        session.close()
+
+        return authorized

tailbone/views/asgi/datasync.py

@@ -24,8 +24,6 @@
 DataSync Views
 """
 
-from __future__ import unicode_literals, absolute_import
-
 import asyncio
 import json
 
@@ -35,36 +33,11 @@ from tailbone.views.asgi import WebsocketView
 class DatasyncWS(WebsocketView):
 
     async def status(self, scope, receive, send):
-        rattail_config = self.registry['rattail_config']
+        app = self.get_rattail_app()
-        app = rattail_config.get_app()
-        model = app.model
-        auth_handler = app.get_auth_handler()
         datasync_handler = app.get_datasync_handler()
 
-        authorized = False
+        # is user allowed to see this?
-        user_session = await self.get_user_session(scope)
+        if not await self.authorize(scope, receive, send, 'datasync.status'):
-        if user_session:
-            user_uuid = user_session.get('auth.userid')
-            session = app.make_session()
-
-            user = None
-            if user_uuid:
-                user = session.query(model.User).get(user_uuid)
-
-            # figure out if user is authorized for this websocket
-            permission = 'datasync.status'
-            authorized = auth_handler.has_permission(session, user, permission)
-            session.close()
-
-        # wait for client to connect
-        message = await receive()
-        assert message['type'] == 'websocket.connect'
-
-        # allow or deny access, per authorization
-        if authorized:
-            await send({'type': 'websocket.accept'})
-        else: # forbidden
-            await send({'type': 'websocket.close'})
             return
 
         # this tracks when client disconnects

tailbone/views/asgi/upgrades.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2022 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Upgrade Views for ASGI
+"""
+
+import asyncio
+import json
+import os
+from urllib.parse import parse_qs
+
+from tailbone.views.asgi import WebsocketView
+from tailbone.progress import get_basic_session
+
+
+class UpgradeWS(WebsocketView):
+
+    async def execution_progress(self, scope, receive, send):
+        rattail_config = self.registry['rattail_config']
+
+        # is user allowed to see this?
+        if not await self.authorize(scope, receive, send, 'upgrades.execute'):
+            return
+
+        # this tracks when client disconnects
+        state = {'disconnected': False}
+
+        async def wait_for_disconnect():
+            message = await receive()
+            if message['type'] == 'websocket.disconnect':
+                state['disconnected'] = True
+
+        # watch for client disconnect, while we do other things
+        asyncio.create_task(wait_for_disconnect())
+
+        query = scope['query_string'].decode('utf_8')
+        query = parse_qs(query)
+        uuid = query['uuid'][0]
+        progress_session_id = 'upgrades.{}.execution_progress'.format(uuid)
+        progress_session = get_basic_session(rattail_config,
+                                             id=progress_session_id)
+
+        # do the rest forever, until client disconnects
+        while not state['disconnected']:
+
+            # load latest progress data
+            progress_session.load()
+
+            # when upgrade progress is complete...
+            if progress_session.get('complete'):
+
+                # maybe set success flash msg
+                msg = progress_session.get('success_msg')
+                if msg:
+                    user_session = await self.get_user_session(scope)
+                    user_session.flash(msg)
+                    user_session.persist()
+
+                # tell client progress is complete
+                await send({'type': 'websocket.send',
+                            'subtype': 'upgrades.execute_progress',
+                            'text': json.dumps({'complete': True})})
+
+                # this websocket is done
+                break
+
+            # we will send this data down to client
+            data = dict(progress_session)
+
+            # maybe add more lines from command output
+            path = rattail_config.upgrade_filepath(uuid, filename='stdout.log')
+            offset = progress_session.get('stdout.offset', 0)
+            if os.path.exists(path):
+                size = os.path.getsize(path) - offset
+                if size > 0:
+                    with open(path, 'rb') as f:
+                        f.seek(offset)
+                        chunk = f.read(size)
+                        data['stdout'] = chunk.decode('utf8').replace('\n', '<br />')
+                    progress_session['stdout.offset'] = offset + size
+                    progress_session.save()
+
+            # send data to client
+            await send({'type': 'websocket.send',
+                        'subtype': 'upgrades.execute_progress',
+                        'text': json.dumps(data)})
+
+            # pause for 1 second
+            await asyncio.sleep(1)
+
+    @classmethod
+    def defaults(cls, config):
+        cls._defaults(config)
+
+    @classmethod
+    def _defaults(cls, config):
+
+        # execution progress
+        config.add_tailbone_websocket('upgrades.execution_progress',
+                                      cls, attr='execution_progress')
+
+
+def defaults(config, **kwargs):
+    base = globals()
+
+    UpgradeWS = kwargs.get('UpgradeWS', base['UpgradeWS'])
+    UpgradeWS.defaults(config)
+
+
+def includeme(config):
+    defaults(config)

tailbone/views/core.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2021 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -134,12 +134,12 @@ class View(object):
     def progress_loop(self, func, items, factory, *args, **kwargs):
         return progress_loop(func, items, factory, *args, **kwargs)
 
-    def make_progress(self, key):
+    def make_progress(self, key, **kwargs):
         """
         Create and return a :class:`tailbone.progress.SessionProgress`
         instance, with the given key.
         """
-        return SessionProgress(self.request, key)
+        return SessionProgress(self.request, key, **kwargs)
 
     # TODO: this signature seems wonky
     def render_progress(self, progress, kwargs, template=None):

tailbone/views/master.py

@@ -1790,14 +1790,28 @@ class MasterView(View):
         """
         obj = self.get_instance()
         model_title = self.get_model_title()
-        progress = self.make_execute_progress(obj)
 
+        # caller must explicitly request websocket behavior; otherwise
+        # we will assume traditional behavior for progress
+        ws = self.request.is_xhr and self.request.json_body.get('ws')
+
+        # make our progress tracker
+        progress = self.make_execute_progress(obj, ws=ws)
+
+        # start execution in a separate thread
         kwargs = {'progress': progress}
         key = [self.request.matchdict[k]
                for k in self.get_model_key(as_tuple=True)]
-        thread = Thread(target=self.execute_thread, args=(key, self.request.user.uuid), kwargs=kwargs)
+        thread = Thread(target=self.execute_thread,
+                        args=(key, self.request.user.uuid),
+                        kwargs=kwargs)
         thread.start()
 
+        # we're done here if using websockets
+        if ws:
+            return self.json_response({'ok': True})
+
+        # traditional behavior sends user to dedicated progress page
         return self.render_progress(progress, {
             'instance': obj,
             'initial_msg': self.execute_progress_initial_msg,
@@ -1805,9 +1819,12 @@ class MasterView(View):
             'cancel_msg': "{} execution was canceled".format(model_title),
         }, template=self.execute_progress_template)
 
-    def make_execute_progress(self, obj):
+    def make_execute_progress(self, obj, ws=False):
+        if ws:
+            key = '{}.{}.execution_progress'.format(self.get_route_prefix(), obj.uuid)
+        else:
             key = '{}.execute'.format(self.get_grid_key())
-        return self.make_progress(key)
+        return self.make_progress(key, ws=ws)
 
     def get_instance_for_key(self, key, session):
         model_key = self.get_model_key(as_tuple=True)

tailbone/views/upgrades.py

@@ -26,24 +26,27 @@ Views for app upgrades
 
 from __future__ import unicode_literals, absolute_import
 
+import json
 import os
 import re
 import logging
+import warnings
 
 import six
-from sqlalchemy import orm
+import sqlalchemy as sa
 
 from rattail.core import Object
 from rattail.db import model, Session as RattailSession
 from rattail.time import make_utc
 from rattail.threads import Thread
-from rattail.upgrades import get_upgrade_handler
+from rattail.util import OrderedDict
 
 from deform import widget as dfwidget
 from webhelpers2.html import tags, HTML
 
 from tailbone.views import MasterView
 from tailbone.progress import get_progress_session #, SessionProgress
+from tailbone.config import should_expose_websockets
 
 
 log = logging.getLogger(__name__)
@@ -56,6 +59,7 @@ class UpgradeView(MasterView):
     model_class = model.Upgrade
     downloadable = True
     cloneable = True
+    configurable = True
     executable = True
     execute_progress_template = '/upgrade.mako'
     execute_progress_initial_msg = "Upgrading"
@@ -68,6 +72,7 @@ class UpgradeView(MasterView):
     }
 
     grid_columns = [
+        'system',
         'created',
         'description',
         # 'not_until',
@@ -78,6 +83,7 @@ class UpgradeView(MasterView):
     ]
 
     form_fields = [
+        'system',
         'description',
         # 'not_until',
         # 'requirements',
@@ -97,28 +103,40 @@ class UpgradeView(MasterView):
 
     def __init__(self, request):
         super(UpgradeView, self).__init__(request)
-        self.handler = self.get_handler()
 
-    def get_handler(self):
+        if hasattr(self, 'get_handler'):
-        """
+            warnings.warn("defining get_handler() is deprecated.  please "
-        Returns the ``UpgradeHandler`` instance for the view.  The handler
+                          "override AppHandler.get_upgrade_handler() instead",
-        factory for this may be defined by config, e.g.:
+                          DeprecationWarning, stacklevel=2)
+            self.upgrade_handler = self.get_handler()
 
-        .. code-block:: ini
+        else:
+            app = self.get_rattail_app()
+            self.upgrade_handler = app.get_upgrade_handler()
 
-           [rattail.upgrades]
+    @property
-           handler = myapp.upgrades:CustomUpgradeHandler
+    def handler(self):
-        """
+        warnings.warn("handler attribute is deprecated; "
-        return get_upgrade_handler(self.rattail_config)
+                      "please use upgrade_handler instead",
+                      DeprecationWarning, stacklevel=2)
+        return self.upgrade_handler
 
     def configure_grid(self, g):
         super(UpgradeView, self).configure_grid(g)
+
+        # system
+        systems = self.upgrade_handler.get_all_systems()
+        systems_enum = dict([(s['key'], s['label']) for s in systems])
+        g.set_enum('system', systems_enum)
+
         g.set_joiner('executed_by', lambda q: q.join(model.User, model.User.uuid == model.Upgrade.executed_by_uuid).outerjoin(model.Person))
         g.set_sorter('executed_by', model.Person.display_name)
         g.set_enum('status_code', self.enum.UPGRADE_STATUS)
         g.set_type('created', 'datetime')
         g.set_type('executed', 'datetime')
         g.set_sort_defaults('created', 'desc')
+
+        g.set_link('system')
         g.set_link('created')
         g.set_link('description')
         # g.set_link('not_until')
@@ -157,6 +175,16 @@ class UpgradeView(MasterView):
         super(UpgradeView, self).configure_form(f)
         upgrade = f.model_instance
 
+        # system
+        systems = self.upgrade_handler.get_all_systems()
+        systems_enum = OrderedDict([(s['key'], s['label'])
+                                    for s in systems])
+        f.set_enum('system', systems_enum)
+        f.set_required('system')
+        if self.creating:
+            if len(systems) == 1:
+                f.set_default('system', list(systems_enum)[0])
+
         # status_code
         if self.creating:
             f.remove_field('status_code')
@@ -174,7 +202,15 @@ class UpgradeView(MasterView):
         f.set_widget('notes', dfwidget.TextAreaWidget(cols=80, rows=8))
         f.set_renderer('stdout_file', self.render_stdout_file)
         f.set_renderer('stderr_file', self.render_stdout_file)
+
+        # package_diff
+        if self.viewing and upgrade.executed and (
+                upgrade.system == 'rattail'
+                or not upgrade.system):
             f.set_renderer('package_diff', self.render_package_diff)
+        else:
+            f.remove_field('package_diff')
+
         # f.set_readonly('created')
         # f.set_readonly('created_by')
         f.set_readonly('executed')
@@ -202,7 +238,6 @@ class UpgradeView(MasterView):
             f.set_default('enabled', True)
 
         if not self.viewing or not upgrade.executed:
-            f.remove_field('package_diff')
             f.remove_field('exit_code')
 
     def render_status_code(self, upgrade, field):
@@ -233,10 +268,11 @@ class UpgradeView(MasterView):
         return text
 
     def configure_clone_form(self, f):
-        f.fields = ['description', 'notes', 'enabled']
+        f.fields = ['system', 'description', 'notes', 'enabled']
 
     def clone_instance(self, original):
         cloned = self.model_class()
+        cloned.system = original.system
         cloned.created = make_utc()
         cloned.created_by = self.request.user
         cloned.description = original.description
@@ -439,13 +475,22 @@ class UpgradeView(MasterView):
     #     key = '{}.execute'.format(self.get_grid_key())
     #     return SessionProgress(self.request, key, session_type='file')
 
-    def execute_instance(self, upgrade, user, **kwargs):
+    def execute_instance(self, upgrade, user, progress=None, **kwargs):
-        session = orm.object_session(upgrade)
+        app = self.get_rattail_app()
-        self.handler.mark_executing(upgrade)
+        session = app.get_session(upgrade)
+
+        # record the fact that execution has begun for this ugprade
+        self.upgrade_handler.mark_executing(upgrade)
         session.commit()
-        self.handler.do_execute(upgrade, user, **kwargs)
+
-        return ("Execution has finished, for better or worse.  "
+        # let handler execute the upgrade
-                "You may need to restart your web app.")
+        self.upgrade_handler.do_execute(upgrade, user, **kwargs)
+
+        # success msg
+        msg = "Execution has finished, for better or worse."
+        if not upgrade.system or upgrade.system == 'rattail':
+            msg += "  You may need to restart your web app."
+        return msg
 
     def execute_progress(self):
         upgrade = self.get_instance()
@@ -489,6 +534,50 @@ class UpgradeView(MasterView):
         self.handler.delete_files(upgrade)
         super(UpgradeView, self).delete_instance(upgrade)
 
+    def configure_get_context(self, **kwargs):
+        context = super(UpgradeView, self).configure_get_context(**kwargs)
+
+        context['upgrade_systems'] = self.upgrade_handler.get_all_systems()
+
+        return context
+
+    def configure_gather_settings(self, data):
+        settings = super(UpgradeView, self).configure_gather_settings(data)
+
+        keys = []
+        for system in json.loads(data['upgrade_systems']):
+            key = system['key']
+            if key == 'rattail':
+                settings.append({'name': 'rattail.upgrades.command',
+                                 'value': system['command']})
+            else:
+                keys.append(key)
+                settings.append({'name': 'rattail.upgrades.system.{}.label'.format(key),
+                                 'value': system['label']})
+                settings.append({'name': 'rattail.upgrades.system.{}.command'.format(key),
+                                 'value': system['command']})
+        if keys:
+            settings.append({'name': 'rattail.upgrades.systems',
+                             'value': ', '.join(keys)})
+
+        return settings
+
+    def configure_remove_settings(self):
+        super(UpgradeView, self).configure_remove_settings()
+        app = self.get_rattail_app()
+        model = self.model
+
+        to_delete = self.Session.query(model.Setting)\
+                           .filter(sa.or_(
+                               model.Setting.name == 'rattail.upgrades.command',
+                               model.Setting.name == 'rattail.upgrades.systems',
+                               model.Setting.name.like('rattail.upgrades.system.%.label'),
+                               model.Setting.name.like('rattail.upgrades.system.%.command')))\
+                           .all()
+
+        for setting in to_delete:
+            app.delete_setting(self.Session(), setting.name)
+
     @classmethod
     def defaults(cls, config):
         cls._defaults(config)
@@ -520,10 +609,14 @@ class UpgradeView(MasterView):
 
 def defaults(config, **kwargs):
     base = globals()
+    rattail_config = config.registry['rattail_config']
 
     UpgradeView = kwargs.get('UpgradeView', base['UpgradeView'])
     UpgradeView.defaults(config)
 
+    if should_expose_websockets(rattail_config):
+        config.include('tailbone.views.asgi.upgrades')
+
 
 def includeme(config):
     defaults(config)