Add first experiment with websockets, for datasync status page

setup.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2021 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -84,6 +84,7 @@ requires = [
     # TODO: cornice<5 requires pyramid<2 (see above)
     'pyramid<2',                        # 1.3b2                 1.10.8
 
+    'asgiref',                          # 3.2.3
     'colander',                         # 1.7.0
     'ColanderAlchemy',                  # 0.3.3
     'humanize',                         # 0.5.1

tailbone/app.py

@@ -129,6 +129,9 @@ def make_pyramid_config(settings, configure_csrf=True):
         settings.setdefault('pyramid_deform.template_search_path', 'tailbone:templates/deform')
         config = Configurator(settings=settings, root_factory=Root)
 
+    # add rattail config directly to registry
+    config.registry['rattail_config'] = rattail_config
+
     # configure user authorization / authentication
     config.set_authorization_policy(TailboneAuthorizationPolicy())
     config.set_authentication_policy(SessionAuthenticationPolicy())
@@ -175,9 +178,45 @@ def make_pyramid_config(settings, configure_csrf=True):
     config.add_directive('add_tailbone_index_page', 'tailbone.app.add_index_page')
     config.add_directive('add_tailbone_config_page', 'tailbone.app.add_config_page')
 
+    config.add_directive('add_tailbone_websocket', 'tailbone.app.add_websocket')
+
     return config
 
 
+def add_websocket(config, name, view, attr=None):
+    """
+    Register a websocket entry point for the app.
+    """
+    def action():
+        rattail_config = config.registry.settings['rattail_config']
+        rattail_app = rattail_config.get_app()
+
+        if isinstance(view, six.string_types):
+            view_callable = rattail_app.load_object(view)
+        else:
+            view_callable = view
+        view_callable = view_callable(config.registry)
+        if attr:
+            view_callable = getattr(view_callable, attr)
+
+        path = '/ws/{}'.format(name)
+
+        # register route
+        config.add_route('ws.{}'.format(name),
+                         path,
+                         static=True)
+
+        # register view callable
+        websockets = config.registry.setdefault('tailbone_websockets', {})
+        websockets[path] = view_callable
+
+    config.action('tailbone-add-websocket', action,
+                  # nb. since this action adds routes, it must happen
+                  # sooner in the order than it normally would, hence
+                  # we declare that
+                  order=-20)
+
+
 def add_index_page(config, route_name, label, permission):
     """
     Register a config page for the app.

tailbone/asgi.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2022 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+ASGI App Utilities
+"""
+
+from __future__ import unicode_literals, absolute_import
+
+import os
+import logging
+
+import six
+from six.moves import configparser
+
+from rattail.util import load_object
+
+from asgiref.wsgi import WsgiToAsgi
+
+
+log = logging.getLogger(__name__)
+
+
+class TailboneWsgiToAsgi(WsgiToAsgi):
+    """
+    Custom WSGI -> ASGI wrapper, to add routing for websockets.
+    """
+
+    async def __call__(self, scope, *args, **kwargs):
+        protocol = scope['type']
+        path = scope['path']
+
+        if protocol == 'websocket':
+            websockets = self.wsgi_application.registry.get(
+                'tailbone_websockets', {})
+            if path in websockets:
+                await websockets[path](scope, *args, **kwargs)
+
+        try:
+            await super().__call__(scope, *args, **kwargs)
+        except ValueError as e:
+            # The developer may wish to improve handling of this exception.
+            # See https://github.com/Pylons/pyramid_cookbook/issues/225 and
+            # https://asgi.readthedocs.io/en/latest/specs/www.html#websocket
+            pass
+        except Exception as e:
+            raise e
+
+
+def make_asgi_app(main_app=None):
+    """
+    This function returns an ASGI application.
+    """
+    path = os.environ.get('TAILBONE_ASGI_CONFIG')
+    if not path:
+        raise RuntimeError("You must define TAILBONE_ASGI_CONFIG env variable.")
+
+    # make a config parser good enough to load pyramid settings
+    configdir = os.path.dirname(path)
+    parser = configparser.ConfigParser(defaults={'__file__': path,
+                                                 'here': configdir})
+
+    # read the config file
+    parser.read(path)
+
+    # parse the settings needed for pyramid app
+    settings = dict(parser.items('app:main'))
+
+    if isinstance(main_app, six.string_types):
+        make_wsgi_app = load_object(main_app)
+    elif callable(main_app):
+        make_wsgi_app = main_app
+    else:
+        if main_app:
+            log.warning("specified main app of unknown type: %s", main_app)
+        make_wsgi_app = load_object('tailbone.app:main')
+
+    # construct a pyramid app "per usual"
+    app = make_wsgi_app({}, **settings)
+
+    # then wrap it with ASGI
+    return TailboneWsgiToAsgi(app)
+
+
+def asgi_main():
+    """
+    This function returns an ASGI application.
+    """
+    return make_asgi_app()

tailbone/config.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2021 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -67,3 +67,8 @@ def global_help_url(config):
 
 def protected_usernames(config):
     return config.getlist('tailbone', 'protected_usernames')
+
+
+def should_expose_websockets(config):
+    return config.getbool('tailbone', 'expose_websockets',
+                          usedb=False, default=False)

tailbone/subscribers.py

@@ -31,7 +31,6 @@ import json
 import datetime
 
 import rattail
-from rattail.db import model
 
 import colander
 import deform
@@ -41,7 +40,7 @@ from webhelpers2.html import tags
 import tailbone
 from tailbone import helpers
 from tailbone.db import Session
-from tailbone.config import csrf_header_name
+from tailbone.config import csrf_header_name, should_expose_websockets
 from tailbone.menus import make_simple_menus
 from tailbone.util import should_use_buefy
 
@@ -72,13 +71,17 @@ def new_request(event):
     if rattail_config:
         request.rattail_config = rattail_config
 
-    request.user = None
+    def user(request):
+        user = None
         uuid = request.authenticated_userid
         if uuid:
-        request.user = Session.query(model.User).get(uuid)
+            model = request.rattail_config.get_model()
-        if request.user:
+            user = Session.query(model.User).get(uuid)
-            # assign user to the session, for sake of versioning
+            if user:
-            Session().set_continuum_user(request.user)
+                Session().set_continuum_user(user)
+        return user
+
+    request.set_property(user, reify=True)
 
     # assign client IP address to the session, for sake of versioning
     Session().continuum_remote_addr = request.client_addr
@@ -99,6 +102,7 @@ def before_render(event):
     """
 
     request = event.get('request') or threadlocal.get_current_request()
+    rattail_config = request.rattail_config
 
     renderer_globals = event
     renderer_globals['rattail_app'] = request.rattail_config.get_app()
@@ -183,6 +187,9 @@ def before_render(event):
         renderer_globals['filter_fieldname_width'] = widths[0]
         renderer_globals['filter_verb_width'] = widths[1]
 
+        # declare global support for websockets, or lack thereof
+        renderer_globals['expose_websockets'] = should_expose_websockets(rattail_config)
+
 
 def add_inbox_count(event):
     """
@@ -196,6 +203,7 @@ def add_inbox_count(event):
     if request.user:
         renderer_globals = event
         enum = request.rattail_config.get_enum()
+        model = request.rattail_config.get_model()
         renderer_globals['inbox_count'] = Session.query(model.Message)\
                                                  .outerjoin(model.MessageRecipient)\
                                                  .filter(model.MessageRecipient.recipient == Session.merge(request.user))\

tailbone/templates/datasync/status.mako

@@ -11,14 +11,17 @@
 </%def>
 
 <%def name="page_content()">
+  % if expose_websockets:
+      <b-notification type="is-warning"
+                      :active="websocketClosed"
+                      :closable="false">
+        Server connection was broken - please refresh page to see accurate status!
+      </b-notification>
+  % endif
   <b-field label="Supervisor Status">
     <div style="display: flex;">
 
-      % if process_info:
+      <pre :class="processInfo.statename == 'RUNNING' ? 'has-background-success' : 'has-background-warning'">{{ processDescription }}</pre>
-          <pre class="has-background-${'success' if process_info['statename'] == 'RUNNING' else 'danger'}">${process_info['group']}:${process_info['name']}    ${process_info['statename']}    ${process_info['description']}</pre>
-      % else:
-          <pre class="has-background-warning">${supervisor_error}</pre>
-      % endif
 
       <div style="margin-left: 1rem;">
         % if request.has_perm('datasync.restart'):
@@ -106,6 +109,17 @@
 <%def name="modify_this_page_vars()">
   <script type="text/javascript">
 
+    ThisPageData.processInfo = ${json.dumps(process_info)|n}
+
+    ThisPage.computed.processDescription = function() {
+        let info = this.processInfo
+        if (info) {
+            return `${'$'}{info.group}:${'$'}{info.name}    ${'$'}{info.statename}    ${'$'}{info.description}`
+        } else {
+            return "NO PROCESS INFO AVAILABLE"
+        }
+    }
+
     ThisPageData.restartingProcess = false
     ThisPageData.watchers = ${json.dumps(watcher_data)|n}
     ThisPageData.consumers = ${json.dumps(consumer_data)|n}
@@ -114,6 +128,31 @@
         this.restartingProcess = true
     }
 
+    % if expose_websockets:
+
+        ThisPageData.ws = null
+        ThisPageData.websocketClosed = false
+
+        ThisPage.mounted = function() {
+
+            ## TODO: should be a cleaner way to get this url?
+            let url = '${request.route_url('ws.datasync.status')}'
+            url = url.replace(/^https?:/, 'wss:')
+
+            this.ws = new WebSocket(url)
+            let that = this
+
+            this.ws.onclose = (event) => {
+                that.websocketClosed = true
+            }
+
+            this.ws.onmessage = (event) => {
+                that.processInfo = JSON.parse(event.data)
+            }
+        }
+
+    % endif
+
   </script>
 </%def>
 

tailbone/views/asgi/__init__.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2022 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+ASGI Views
+"""
+
+from __future__ import unicode_literals, absolute_import
+
+import http.cookies
+
+from beaker.cache import clsmap
+from beaker.session import SessionObject, SignedCookie
+
+
+class WebsocketView(object):
+
+    def __init__(self, registry):
+        self.registry = registry
+
+    async def get_user_session(self, scope):
+        settings = self.registry.settings
+        beaker_key = settings['beaker.session.key']
+        beaker_secret = settings['beaker.session.secret']
+        beaker_type = settings['beaker.session.type']
+        beaker_data_dir = settings['beaker.session.data_dir']
+        beaker_lock_dir = settings['beaker.session.lock_dir']
+
+        # get ahold of session identifier cookie
+        headers = dict(scope['headers'])
+        cookie = headers.get(b'cookie')
+        if not cookie:
+            return
+        cookie = cookie.decode('utf_8')
+        cookie = http.cookies.SimpleCookie(cookie)
+        morsel = cookie[beaker_key]
+
+        # simulate pyramid_beaker logic to get at the session
+        cookieheader = morsel.output(header='')
+        cookie = SignedCookie(beaker_secret, input=cookieheader)
+        session_id = cookie[beaker_key].value
+        request = {'cookie': cookieheader}
+        session = SessionObject(
+            request,
+            id=session_id,
+            key=beaker_key,
+            namespace_class=clsmap[beaker_type],
+            data_dir=beaker_data_dir,
+            lock_dir=beaker_lock_dir)
+
+        return session

tailbone/views/asgi/datasync.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2022 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+DataSync Views
+"""
+
+from __future__ import unicode_literals, absolute_import
+
+import asyncio
+import json
+
+from tailbone.views.asgi import WebsocketView
+
+
+class DatasyncWS(WebsocketView):
+
+    async def status(self, scope, receive, send):
+        rattail_config = self.registry['rattail_config']
+        app = rattail_config.get_app()
+        model = app.model
+        auth_handler = app.get_auth_handler()
+        datasync_handler = app.get_datasync_handler()
+
+        authorized = False
+        user_session = await self.get_user_session(scope)
+        if user_session:
+            user_uuid = user_session.get('auth.userid')
+            session = app.make_session()
+
+            user = None
+            if user_uuid:
+                user = session.query(model.User).get(user_uuid)
+
+            # figure out if user is authorized for this websocket
+            permission = 'datasync.status'
+            authorized = auth_handler.has_permission(session, user, permission)
+            session.close()
+
+        # wait for client to connect
+        message = await receive()
+        assert message['type'] == 'websocket.connect'
+
+        # allow or deny access, per authorization
+        if authorized:
+            await send({'type': 'websocket.accept'})
+        else: # forbidden
+            await send({'type': 'websocket.close'})
+            return
+
+        # this tracks when client disconnects
+        state = {'disconnected': False}
+
+        async def wait_for_disconnect():
+            message = await receive()
+            if message['type'] == 'websocket.disconnect':
+                state['disconnected'] = True
+
+        # watch for client disconnect, while we do other things
+        asyncio.create_task(wait_for_disconnect())
+
+        # do the rest forever, until client disconnects
+        while not state['disconnected']:
+
+            # give client latest supervisor process info
+            info = datasync_handler.get_supervisor_process_info()
+            await send({'type': 'websocket.send',
+                        'subtype': 'datasync.supervisor_process_info',
+                        'text': json.dumps(info)})
+
+            # pause for 1 second
+            await asyncio.sleep(1)
+
+    @classmethod
+    def defaults(cls, config):
+        cls._defaults(config)
+
+    @classmethod
+    def _defaults(cls, config):
+
+        # status
+        config.add_tailbone_websocket('datasync.status',
+                                      cls, attr='status')
+
+
+def defaults(config, **kwargs):
+    base = globals()
+
+    DatasyncWS = kwargs.get('DatasyncWS', base['DatasyncWS'])
+    DatasyncWS.defaults(config)
+
+
+def includeme(config):
+    defaults(config)

tailbone/views/datasync.py

@@ -40,6 +40,7 @@ from rattail.util import simple_error
 
 from tailbone.views import MasterView
 from tailbone.util import raw_datetime
+from tailbone.config import should_expose_websockets
 
 
 log = logging.getLogger(__name__)
@@ -400,6 +401,19 @@ class DataSyncChangeView(MasterView):
 DataSyncChangesView = DataSyncChangeView
 
 
-def includeme(config):
+def defaults(config, **kwargs):
+    base = globals()
+    rattail_config = config.registry['rattail_config']
+
+    DataSyncThreadView = kwargs.get('DataSyncThreadView', base['DataSyncThreadView'])
     DataSyncThreadView.defaults(config)
+
+    DataSyncChangeView = kwargs.get('DataSyncChangeView', base['DataSyncChangeView'])
     DataSyncChangeView.defaults(config)
+
+    if should_expose_websockets(rattail_config):
+        config.include('tailbone.views.asgi.datasync')
+
+
+def includeme(config):
+    defaults(config)