From 9de35a6e8b0dbf555cc336d3f44f2d261917ce56 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 17 Aug 2022 22:59:50 -0500 Subject: [PATCH 001/879] Add brief delay before declaring websocket broken --- tailbone/templates/datasync/status.mako | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tailbone/templates/datasync/status.mako b/tailbone/templates/datasync/status.mako index c80615ce..29ca00cf 100644 --- a/tailbone/templates/datasync/status.mako +++ b/tailbone/templates/datasync/status.mako @@ -13,7 +13,7 @@ <%def name="page_content()"> % if expose_websockets and not supervisor_error: Server connection was broken - please refresh page to see accurate status! @@ -135,7 +135,7 @@ % if expose_websockets and not supervisor_error: ThisPageData.ws = null - ThisPageData.websocketClosed = false + ThisPageData.websocketBroken = false ThisPage.mounted = function() { @@ -147,7 +147,14 @@ let that = this this.ws.onclose = (event) => { - that.websocketClosed = true + // websocket closing means 1 of 2 things: + // - user navigated away from page intentionally + // - server connection was broken somehow + // only one of those is "bad" and we only want to + // display warning in 2nd case. so we simply use a + // brief delay to "rule out" the 1st scenario + setTimeout(() => { that.websocketBroken = true }, + 3000) } this.ws.onmessage = (event) => { From d23e5d169adeb8aa1ccac649f9f6e7c79d9ea8f5 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Aug 2022 15:11:09 -0500 Subject: [PATCH 002/879] Add basic views for Luigi / overnight tasks --- tailbone/templates/luigi/configure.mako | 129 +++++++++++++++++++ tailbone/templates/luigi/index.mako | 126 ++++++++++++++++++ tailbone/views/datasync.py | 2 - tailbone/views/luigi.py | 164 ++++++++++++++++++++++++ tailbone/views/master.py | 5 + 5 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 tailbone/templates/luigi/configure.mako create mode 100644 tailbone/templates/luigi/index.mako create mode 100644 tailbone/views/luigi.py diff --git a/tailbone/templates/luigi/configure.mako b/tailbone/templates/luigi/configure.mako new file mode 100644 index 00000000..b8fba490 --- /dev/null +++ b/tailbone/templates/luigi/configure.mako @@ -0,0 +1,129 @@ +## -*- coding: utf-8; -*- +<%inherit file="/configure.mako" /> + +<%def name="form_content()"> + ${h.hidden('overnight_tasks', **{':value': 'JSON.stringify(overnightTasks)'})} + +

Overnight Tasks

+
+ + + + + +
+ + New Task + + + +
+ +
+

Overnight Task

+
+ +
+ + + + +
+ +
+ + Save + + + Cancel + +
+
+ + +
+
+ +

Luigi Proper

+
+ + + + + + + + + + + + + + + + +
+ + + +<%def name="modify_this_page_vars()"> + ${parent.modify_this_page_vars()} + + + + +${parent.body()} diff --git a/tailbone/templates/luigi/index.mako b/tailbone/templates/luigi/index.mako new file mode 100644 index 00000000..16ea3489 --- /dev/null +++ b/tailbone/templates/luigi/index.mako @@ -0,0 +1,126 @@ +## -*- coding: utf-8; -*- +<%inherit file="/page.mako" /> + +<%def name="title()">Luigi Jobs + +<%def name="page_content()"> +
+
+ +
+ + + Luigi Task Visualiser + + + + Luigi Task History + + + % if master.has_perm('restart_scheduler'): + ${h.form(url('{}.restart_scheduler'.format(route_prefix)), **{'@submit': 'submitRestartSchedulerForm'})} + ${h.csrf_token(request)} + + {{ restartSchedulerFormSubmitting ? "Working, please wait..." : "Restart Luigi Scheduler" }} + + ${h.end_form()} + % endif +
+ + % if master.has_perm('launch'): +

Overnight Tasks

+ % for task in overnight_tasks: + + + % endfor + % endif + +
+ + +<%def name="modify_this_page_vars()"> + ${parent.modify_this_page_vars()} + % if master.has_perm('restart_scheduler'): + + % endif + + +<%def name="finalize_this_page_vars()"> + ${parent.finalize_this_page_vars()} + % if master.has_perm('launch'): + + % endif + + +<%def name="render_this_page_template()"> + ${parent.render_this_page_template()} + % if master.has_perm('launch'): + + % endif + + + +${parent.body()} diff --git a/tailbone/views/datasync.py b/tailbone/views/datasync.py index 20f970e4..0f198795 100644 --- a/tailbone/views/datasync.py +++ b/tailbone/views/datasync.py @@ -26,7 +26,6 @@ DataSync Views from __future__ import unicode_literals, absolute_import -import getpass import json import subprocess import logging @@ -234,7 +233,6 @@ class DataSyncThreadView(MasterView): 'rattail.datasync', 'supervisor_process_name'), 'restart_command': self.rattail_config.get( 'tailbone', 'datasync.restart'), - 'system_user': getpass.getuser(), } def configure_gather_settings(self, data): diff --git a/tailbone/views/luigi.py b/tailbone/views/luigi.py new file mode 100644 index 00000000..6b0b60e3 --- /dev/null +++ b/tailbone/views/luigi.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2022 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Views for Luigi +""" + +from __future__ import unicode_literals, absolute_import + +import json + +from rattail.util import simple_error + +from tailbone.views import MasterView + + +class LuigiJobView(MasterView): + """ + Simple views for Luigi jobs. + """ + normalized_model_name = 'luigijobs' + model_key = 'jobname' + model_title = "Luigi Job" + route_prefix = 'luigi' + url_prefix = '/luigi' + + viewable = False + creatable = False + editable = False + deletable = False + configurable = True + + def __init__(self, request, context=None): + super(LuigiJobView, self).__init__(request, context=context) + app = self.get_rattail_app() + self.luigi_handler = app.get_luigi_handler() + + def index(self): + luigi_url = self.rattail_config.get('luigi', 'url') + history_url = '{}/history'.format(luigi_url.rstrip('/')) if luigi_url else None + return self.render_to_response('index', { + 'use_buefy': self.get_use_buefy(), + 'index_url': None, + 'luigi_url': luigi_url, + 'luigi_history_url': history_url, + 'overnight_tasks': self.luigi_handler.get_all_overnight_tasks(), + }) + + def launch(self): + key = self.request.POST['job'] + assert key + self.luigi_handler.restart_overnight_task(key) + self.request.session.flash("Scheduled overnight task for immediate launch: {}".format(key)) + return self.redirect(self.get_index_url()) + + def restart_scheduler(self): + try: + self.luigi_handler.restart_supervisor_process() + self.request.session.flash("Luigi scheduler has been restarted.") + + except Exception as error: + self.request.session.flash(simple_error(error), 'error') + + return self.redirect(self.request.get_referrer( + default=self.get_index_url())) + + def configure_get_simple_settings(self): + return [ + + # luigi proper + {'section': 'luigi', + 'option': 'url'}, + {'section': 'luigi', + 'option': 'scheduler.supervisor_process_name'}, + {'section': 'luigi', + 'option': 'scheduler.restart_command'}, + + ] + + def configure_get_context(self, **kwargs): + context = super(LuigiJobView, self).configure_get_context(**kwargs) + context['overnight_tasks'] = self.luigi_handler.get_all_overnight_tasks() + return context + + def configure_gather_settings(self, data): + settings = super(LuigiJobView, self).configure_gather_settings(data) + + keys = [] + for task in json.loads(data['overnight_tasks']): + keys.append(task['key']) + + if keys: + settings.append({'name': 'luigi.overnight_tasks', + 'value': ', '.join(keys)}) + + return settings + + def configure_remove_settings(self): + super(LuigiJobView, self).configure_remove_settings() + self.luigi_handler.purge_luigi_settings(self.Session()) + + @classmethod + def defaults(cls, config): + cls._defaults(config) + cls._luigi_defaults(config) + + @classmethod + def _luigi_defaults(cls, config): + route_prefix = cls.get_route_prefix() + permission_prefix = cls.get_permission_prefix() + url_prefix = cls.get_url_prefix() + model_title_plural = cls.get_model_title_plural() + + # launch job + config.add_tailbone_permission(permission_prefix, + '{}.launch'.format(permission_prefix), + label="Launch any Luigi job") + config.add_route('{}.launch'.format(route_prefix), + '{}/launch'.format(url_prefix), + request_method='POST') + config.add_view(cls, attr='launch', + route_name='{}.launch'.format(route_prefix), + permission='{}.launch'.format(permission_prefix)) + + # restart luigid scheduler + config.add_tailbone_permission(permission_prefix, + '{}.restart_scheduler'.format(permission_prefix), + label="Restart the Luigi Scheduler daemon") + config.add_route('{}.restart_scheduler'.format(route_prefix), + '{}/restart-scheduler'.format(url_prefix), + request_method='POST') + config.add_view(cls, attr='restart_scheduler', + route_name='{}.restart_scheduler'.format(route_prefix), + permission='{}.restart_scheduler'.format(permission_prefix)) + + +def defaults(config, **kwargs): + base = globals() + + LuigiJobView = kwargs.get('LuigiJobView', base['LuigiJobView']) + LuigiJobView.defaults(config) + + +def includeme(config): + defaults(config) diff --git a/tailbone/views/master.py b/tailbone/views/master.py index 1915ac83..1906d620 100644 --- a/tailbone/views/master.py +++ b/tailbone/views/master.py @@ -29,6 +29,7 @@ from __future__ import unicode_literals, absolute_import import os import csv import datetime +import getpass import shutil import tempfile import logging @@ -4324,6 +4325,10 @@ class MasterView(View): context = self.configure_get_context() return self.render_to_response('configure', context) + def template_kwargs_configure(self, **kwargs): + kwargs['system_user'] = getpass.getuser() + return kwargs + def configure_flash_settings_saved(self): self.request.session.flash("Settings have been saved.") From 89da6ae5011672ec84a0238e0f40a08ceaf5075b Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Aug 2022 17:27:30 -0500 Subject: [PATCH 003/879] Expose setting for auto-correct when receiving from invoice --- tailbone/templates/receiving/configure.mako | 7 +++++++ tailbone/views/purchasing/receiving.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/tailbone/templates/receiving/configure.mako b/tailbone/templates/receiving/configure.mako index 36ff5c39..f4a697f4 100644 --- a/tailbone/templates/receiving/configure.mako +++ b/tailbone/templates/receiving/configure.mako @@ -115,6 +115,13 @@ + + Try to auto-correct "case vs. unit" mistakes from invoice parser + +

Mobile Interface

diff --git a/tailbone/views/purchasing/receiving.py b/tailbone/views/purchasing/receiving.py index a7286b07..af96448f 100644 --- a/tailbone/views/purchasing/receiving.py +++ b/tailbone/views/purchasing/receiving.py @@ -1928,6 +1928,9 @@ class ReceivingBatchView(PurchasingBatchView): {'section': 'rattail.batch', 'option': 'purchase.allow_expired_credits', 'type': bool}, + {'section': 'rattail.batch', + 'option': 'purchase.receiving.should_autofix_invoice_case_vs_unit', + 'type': bool}, # mobile interface {'section': 'rattail.batch', From 8afc3766365b503b7b4a9627dced0db8470fbc3b Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Aug 2022 17:29:13 -0500 Subject: [PATCH 004/879] Update changelog --- CHANGES.rst | 10 ++++++++++ tailbone/_version.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a7c0c344..b3631727 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,16 @@ CHANGELOG ========= +0.8.249 (2022-08-18) +-------------------- + +* Add brief delay before declaring websocket broken. + +* Add basic views for Luigi / overnight tasks. + +* Expose setting for auto-correct when receiving from invoice. + + 0.8.248 (2022-08-17) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index c45030ec..5e741492 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.248' +__version__ = '0.8.249' From 7d72a43ecd68123486564a27176cfd3a43b495bf Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Aug 2022 18:19:54 -0500 Subject: [PATCH 005/879] Use pytest instead of nosetests, for tox runs --- setup.py | 2 ++ tox.ini | 19 +++++++------------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 44a5910a..1f65ca97 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,8 @@ extras = { 'fixture', # 1.5 'mock', # 1.0.1 'nose', # 1.3.0 + 'pytest', # 4.6.11 + 'pytest-cov', # 2.12.1 ], } diff --git a/tox.ini b/tox.ini index 6dd5ada3..9cda1c76 100644 --- a/tox.ini +++ b/tox.ini @@ -1,35 +1,30 @@ + [tox] -envlist = py27, py35 +envlist = py27, py35, py37 [testenv] -deps = - coverage - fixture - mock - nose commands = pip install --upgrade pip + pip install --upgrade setuptools wheel pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon - nosetests {posargs} + pytest {posargs} [testenv:py27] commands = pip install --upgrade pip + pip install --upgrade setuptools wheel pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon SQLAlchemy-Utils<0.36.7 SQLAlchemy-Continuum<1.3.12 - nosetests {posargs} + pytest {posargs} [testenv:coverage] basepython = python3 commands = pip install --upgrade pip pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon - nosetests {posargs:--with-coverage --cover-html-dir={envtmpdir}/coverage} + pytest --cov=tailbone --cov-report=html [testenv:docs] basepython = python3 -deps = - Sphinx - sphinx-rtd-theme changedir = docs commands = pip install --upgrade pip From 9566a882b58549c81a011ea54e7dff2b1ff92bd6 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Aug 2022 18:23:30 -0500 Subject: [PATCH 006/879] Install dependencies when running tests etc. via tox --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 9cda1c76..401b5e62 100644 --- a/tox.ini +++ b/tox.ini @@ -6,21 +6,21 @@ envlist = py27, py35, py37 commands = pip install --upgrade pip pip install --upgrade setuptools wheel - pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon + pip install --upgrade --upgrade-strategy eager Tailbone[tests] rattail[auth,bouncer,db] rattail-tempmon pytest {posargs} [testenv:py27] commands = pip install --upgrade pip pip install --upgrade setuptools wheel - pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon SQLAlchemy-Utils<0.36.7 SQLAlchemy-Continuum<1.3.12 + pip install --upgrade --upgrade-strategy eager Tailbone[tests] rattail[auth,bouncer,db] rattail-tempmon SQLAlchemy-Utils<0.36.7 SQLAlchemy-Continuum<1.3.12 pytest {posargs} [testenv:coverage] basepython = python3 commands = pip install --upgrade pip - pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon + pip install --upgrade --upgrade-strategy eager Tailbone[tests] rattail[auth,bouncer,db] rattail-tempmon pytest --cov=tailbone --cov-report=html [testenv:docs] @@ -28,5 +28,5 @@ basepython = python3 changedir = docs commands = pip install --upgrade pip - pip install --upgrade --upgrade-strategy eager Tailbone rattail[auth,bouncer,db] rattail-tempmon + pip install --upgrade --upgrade-strategy eager Tailbone[docs] rattail[auth,bouncer,db] rattail-tempmon sphinx-build -b html -d {envtmpdir}/doctrees -W -T . {envtmpdir}/docs From 8470126918903f98deea2d1a4f3d951c84031ad2 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Aug 2022 19:22:04 -0500 Subject: [PATCH 007/879] Add `render_person_profile()` method to MasterView --- tailbone/views/master.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tailbone/views/master.py b/tailbone/views/master.py index 1906d620..62502035 100644 --- a/tailbone/views/master.py +++ b/tailbone/views/master.py @@ -897,6 +897,14 @@ class MasterView(View): url = self.request.route_url('people.view', uuid=person.uuid) return tags.link_to(text, url) + def render_person_profile(self, obj, field): + person = getattr(obj, field) + if not person: + return "" + text = six.text_type(person) + url = self.request.route_url('people.view_profile', uuid=person.uuid) + return tags.link_to(text, url) + def render_user(self, obj, field): user = getattr(obj, field) if not user: From db3f215ebeb0ef8ddf483e373372a16049db824b Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 19 Aug 2022 17:20:01 -0500 Subject: [PATCH 008/879] Add way to declare failure for an upgrade doesn't really cancel it, since Tailbone isn't actually tracking the subprocess etc. but saves a step when something goes off the rails --- tailbone/templates/upgrades/view.mako | 38 ++++++++++++++++ tailbone/views/upgrades.py | 65 ++++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/tailbone/templates/upgrades/view.mako b/tailbone/templates/upgrades/view.mako index 03fd9b6b..6a027921 100644 --- a/tailbone/templates/upgrades/view.mako +++ b/tailbone/templates/upgrades/view.mako @@ -38,6 +38,27 @@ % endif +<%def name="render_this_page()"> + ${parent.render_this_page()} + + % if master.has_perm('execute'): + ${h.form(master.get_action_url('declare_failure', instance), ref='declareFailureForm')} + ${h.csrf_token(request)} + ${h.end_form()} + % endif + + +<%def name="render_buefy_form()"> +
+ <${form.component} + % if master.has_perm('execute'): + @declare-failure="declareFailure" + % endif + > + +
+ + <%def name="render_form_buttons()"> % if not instance.executed and instance.status_code == enum.UPGRADE_STATUS_PENDING and request.has_perm('{}.execute'.format(permission_prefix)):
@@ -81,6 +102,23 @@ this.formButtonText = "Working, please wait..." } + % if master.has_perm('execute'): + + TailboneFormData.declareFailureSubmitting = false + + TailboneForm.methods.declareFailureClick = function() { + if (confirm("Really declare this upgrade a failure?")) { + this.declareFailureSubmitting = true + this.$emit('declare-failure') + } + } + + ThisPage.methods.declareFailure = function() { + this.$refs.declareFailureForm.submit() + } + + % endif + diff --git a/tailbone/views/upgrades.py b/tailbone/views/upgrades.py index ff4de768..2e7c2fc4 100644 --- a/tailbone/views/upgrades.py +++ b/tailbone/views/upgrades.py @@ -162,7 +162,7 @@ class UpgradeView(MasterView): f.remove_field('status_code') else: f.set_enum('status_code', self.enum.UPGRADE_STATUS) - # f.set_readonly('status_code') + f.set_renderer('status_code', self.render_status_code) # executing if not self.editing: @@ -205,6 +205,33 @@ class UpgradeView(MasterView): f.remove_field('package_diff') f.remove_field('exit_code') + def render_status_code(self, upgrade, field): + code = getattr(upgrade, field) + text = self.enum.UPGRADE_STATUS[code] + + if self.get_use_buefy(): + if code == self.enum.UPGRADE_STATUS_EXECUTING: + + text = HTML.tag('span', c=[text]) + + button = HTML.tag('b-button', + type='is-warning', + icon_pack='fas', + icon_left='sad-tear', + c=['{{ declareFailureSubmitting ? "Working, please wait..." : "Declare Failure" }}'], + **{':disabled': 'declareFailureSubmitting', + '@click': 'declareFailureClick'}) + + return HTML.tag('div', class_='level', c=[ + HTML.tag('div', class_='level-left', c=[ + HTML.tag('div', class_='level-item', c=[text]), + HTML.tag('div', class_='level-item', c=[button]), + ]), + ]) + + # just show status per normal + return text + def configure_clone_form(self, f): f.fields = ['description', 'notes', 'enabled'] @@ -446,23 +473,49 @@ class UpgradeView(MasterView): return data + def declare_failure(self): + upgrade = self.get_instance() + if upgrade.executing and upgrade.status_code == self.enum.UPGRADE_STATUS_EXECUTING: + upgrade.executing = False + upgrade.status_code = self.enum.UPGRADE_STATUS_FAILED + self.request.session.flash("Upgrade was declared a failure.", 'warning') + else: + self.request.session.flash("Upgrade was not currently executing! " + "So it was not declared a failure.", + 'error') + return self.redirect(self.get_action_url('view', upgrade)) + def delete_instance(self, upgrade): self.handler.delete_files(upgrade) super(UpgradeView, self).delete_instance(upgrade) @classmethod def defaults(cls, config): + cls._defaults(config) + cls._upgrade_defaults(config) + + @classmethod + def _upgrade_defaults(cls, config): route_prefix = cls.get_route_prefix() - url_prefix = cls.get_url_prefix() permission_prefix = cls.get_permission_prefix() + instance_url_prefix = cls.get_instance_url_prefix() model_key = cls.get_model_key() # execution progress - config.add_route('{}.execute_progress'.format(route_prefix), '{}/{{{}}}/execute/progress'.format(url_prefix, model_key)) - config.add_view(cls, attr='execute_progress', route_name='{}.execute_progress'.format(route_prefix), - permission='{}.execute'.format(permission_prefix), renderer='json') + config.add_route('{}.execute_progress'.format(route_prefix), + '{}/execute/progress'.format(instance_url_prefix)) + config.add_view(cls, attr='execute_progress', + route_name='{}.execute_progress'.format(route_prefix), + permission='{}.execute'.format(permission_prefix), + renderer='json') - cls._defaults(config) + # declare failure + config.add_route('{}.declare_failure'.format(route_prefix), + '{}/declare-failure'.format(instance_url_prefix), + request_method='POST') + config.add_view(cls, attr='declare_failure', + route_name='{}.declare_failure'.format(route_prefix), + permission='{}.execute'.format(permission_prefix)) def defaults(config, **kwargs): From 18cec49a86a97481700f46254ba7c55e4373b5e2 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Aug 2022 17:39:33 -0500 Subject: [PATCH 009/879] Add websockets progress, "multi-system" support for upgrades and related things to better support that --- tailbone/app.py | 12 +- tailbone/progress.py | 34 +++- tailbone/templates/forms/deform_buefy.mako | 1 + tailbone/templates/themes/falafel/base.mako | 42 ++++- tailbone/templates/upgrades/configure.mako | 156 ++++++++++++++++++ tailbone/templates/upgrades/view.mako | 168 +++++++++++++++++--- tailbone/views/asgi/__init__.py | 100 +++++++++--- tailbone/views/asgi/datasync.py | 33 +--- tailbone/views/asgi/upgrades.py | 131 +++++++++++++++ tailbone/views/core.py | 6 +- tailbone/views/master.py | 27 +++- tailbone/views/upgrades.py | 135 +++++++++++++--- 12 files changed, 731 insertions(+), 114 deletions(-) create mode 100644 tailbone/templates/upgrades/configure.mako create mode 100644 tailbone/views/asgi/upgrades.py diff --git a/tailbone/app.py b/tailbone/app.py index 5eb0911e..d7155829 100644 --- a/tailbone/app.py +++ b/tailbone/app.py @@ -195,22 +195,20 @@ def add_websocket(config, name, view, attr=None): view_callable = rattail_app.load_object(view) else: view_callable = view - view_callable = view_callable(config.registry) + view_callable = view_callable(config) if attr: view_callable = getattr(view_callable, attr) - path = '/ws/{}'.format(name) - # register route - config.add_route('ws.{}'.format(name), - path, - static=True) + path = '/ws/{}'.format(name) + route_name = 'ws.{}'.format(name) + config.add_route(route_name, path, static=True) # register view callable websockets = config.registry.setdefault('tailbone_websockets', {}) websockets[path] = view_callable - config.action('tailbone-add-websocket', action, + config.action('tailbone-add-websocket-{}'.format(name), action, # nb. since this action adds routes, it must happen # sooner in the order than it normally would, hence # we declare that diff --git a/tailbone/progress.py b/tailbone/progress.py index 90fa21be..5c45f390 100644 --- a/tailbone/progress.py +++ b/tailbone/progress.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2018 Lance Edgar +# Copyright © 2010-2022 Lance Edgar # # This file is part of Rattail. # @@ -27,22 +27,33 @@ Progress Indicator from __future__ import unicode_literals, absolute_import import os +import warnings from rattail.progress import ProgressBase from beaker.session import Session +def get_basic_session(config, request={}, **kwargs): + """ + Create/get a "basic" Beaker session object. + """ + kwargs['use_cookies'] = False + session = Session(request, **kwargs) + return session + + def get_progress_session(request, key, **kwargs): """ Create/get a Beaker session object, to be used for progress. """ - id = '{}.progress.{}'.format(request.session.id, key) - kwargs['use_cookies'] = False + kwargs['id'] = '{}.progress.{}'.format(request.session.id, key) if kwargs.get('type') == 'file': + warnings.warn("Passing a 'type' kwarg to get_progress_session() " + "is deprecated...i think", + DeprecationWarning, stacklevel=2) kwargs['data_dir'] = os.path.join(request.rattail_config.appdir(), 'sessions') - session = Session(request, id, **kwargs) - return session + return get_basic_session(request.rattail_config, request, **kwargs) class SessionProgress(ProgressBase): @@ -52,11 +63,20 @@ class SessionProgress(ProgressBase): This class is only responsible for keeping the progress *data* current. It is the responsibility of some client-side AJAX (etc.) to consume the data for display to the user. + + :param ws: If true, then websockets are assumed, and the progress will + behave accordingly. The default is false, "traditional" behavior. """ - def __init__(self, request, key, session_type=None): + def __init__(self, request, key, session_type=None, ws=False): self.key = key - self.session = get_progress_session(request, key, type=session_type) + self.ws = ws + + if self.ws: + self.session = get_basic_session(request.rattail_config, id=key) + else: + self.session = get_progress_session(request, key, type=session_type) + self.canceled = False self.clear() diff --git a/tailbone/templates/forms/deform_buefy.mako b/tailbone/templates/forms/deform_buefy.mako index 860449fb..c387d965 100644 --- a/tailbone/templates/forms/deform_buefy.mako +++ b/tailbone/templates/forms/deform_buefy.mako @@ -73,6 +73,7 @@ let ${form.component_studly} = { template: '#${form.component}-template', + mixins: [FormPosterMixin], components: {}, props: {}, watch: {}, diff --git a/tailbone/templates/themes/falafel/base.mako b/tailbone/templates/themes/falafel/base.mako index 9b9236fe..fe3ef429 100644 --- a/tailbone/templates/themes/falafel/base.mako +++ b/tailbone/templates/themes/falafel/base.mako @@ -682,20 +682,54 @@ % if show_prev_next is not Undefined and show_prev_next: % if prev_url:
- ${h.link_to(u"« Older", prev_url, class_='button autodisable')} + % if use_buefy: + + Older + + % else: + ${h.link_to(u"« Older", prev_url, class_='button autodisable')} + % endif
% else:
- ${h.link_to(u"« Older", '#', class_='button', disabled='disabled')} + % if use_buefy: + + Older + + % else: + ${h.link_to(u"« Older", '#', class_='button', disabled='disabled')} + % endif
% endif % if next_url:
- ${h.link_to(u"Newer »", next_url, class_='button autodisable')} + % if use_buefy: + + Newer + + % else: + ${h.link_to(u"Newer »", next_url, class_='button autodisable')} + % endif
% else:
- ${h.link_to(u"Newer »", '#', class_='button', disabled='disabled')} + % if use_buefy: + + Newer + + % else: + ${h.link_to(u"Newer »", '#', class_='button', disabled='disabled')} + % endif
% endif % endif diff --git a/tailbone/templates/upgrades/configure.mako b/tailbone/templates/upgrades/configure.mako new file mode 100644 index 00000000..cde81b9e --- /dev/null +++ b/tailbone/templates/upgrades/configure.mako @@ -0,0 +1,156 @@ +## -*- coding: utf-8; -*- +<%inherit file="/configure.mako" /> + +<%def name="form_content()"> + ${h.hidden('upgrade_systems', **{':value': 'JSON.stringify(upgradeSystems)'})} + +

Upgradable Systems

+
+ + + + + +
+ + New System + + + +
+ +
+

Upgradable System

+
+ +
+ + + + + + + + + + + + +
+ +
+ + Save + + + Cancel + +
+
+ + +
+
+ + +<%def name="modify_this_page_vars()"> + ${parent.modify_this_page_vars()} + + + + +${parent.body()} diff --git a/tailbone/templates/upgrades/view.mako b/tailbone/templates/upgrades/view.mako index 6a027921..ed23c83a 100644 --- a/tailbone/templates/upgrades/view.mako +++ b/tailbone/templates/upgrades/view.mako @@ -38,6 +38,18 @@ % endif +<%def name="extra_styles()"> + ${parent.extra_styles()} + + + <%def name="render_this_page()"> ${parent.render_this_page()} @@ -60,31 +72,86 @@ <%def name="render_form_buttons()"> - % if not instance.executed and instance.status_code == enum.UPGRADE_STATUS_PENDING and request.has_perm('{}.execute'.format(permission_prefix)): + % if not instance.executed and instance.status_code == enum.UPGRADE_STATUS_PENDING and master.has_perm('execute'):
% if instance.enabled and not instance.executing: - % if use_buefy: - ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), **{'@submit': 'submitForm'})} - % else: - ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), class_='autodisable')} - % endif - ${h.csrf_token(request)} - % if use_buefy: + % if use_buefy and expose_websockets: + + {{ upgradeExecuting ? "Working, please wait..." : "Execute this upgrade" }} + + % elif use_buefy: + ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), **{'@submit': 'submitForm'})} + ${h.csrf_token(request)} - {{ formButtonText }} + {{ formSubmitting ? "Working, please wait..." : "Execute this upgrade" }} + ${h.end_form()} % else: + ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), class_='autodisable')} + ${h.csrf_token(request)} ${h.submit('execute', "Execute this upgrade", class_='button is-primary')} + ${h.end_form()} % endif - ${h.end_form()} % elif instance.enabled: % else: % endif
+ + +
+
+ +
+
+

Upgrading (please wait) ...

+ + +
+
+
+ + Declare Failure + +
+
+
+ +
+ + + + ## nb. we auto-scroll down to "see" this element +
+
+ +
+
+ + % endif @@ -94,16 +161,81 @@ TailboneFormData.showingPackages = 'diffs' - TailboneFormData.formButtonText = "Execute this upgrade" - TailboneFormData.formSubmitting = false - - TailboneForm.methods.submitForm = function() { - this.formSubmitting = true - this.formButtonText = "Working, please wait..." - } - % if master.has_perm('execute'): + % if expose_websockets: + + TailboneFormData.upgradeExecuting = ${json.dumps(instance.executing)|n} + TailboneFormData.progressOutput = [] + TailboneFormData.progressOutputCounter = 0 + + TailboneForm.methods.executeUpgrade = function() { + this.upgradeExecuting = true + + // grow the textout area to fill most of screen + this.$nextTick(() => { + let textout = this.$refs.textout + let height = window.innerHeight - textout.offsetTop - 50 + textout.style.height = height + 'px' + }) + + let url = '${master.get_action_url('execute', instance)}' + this.submitForm(url, {ws: true}, response => { + + ## TODO: should be a cleaner way to get this url? + url = '${request.route_url('ws.upgrades.execution_progress', _query={'uuid': instance.uuid})}' + url = url.replace(/^https?:/, 'wss:') + + this.ws = new WebSocket(url) + let that = this + + ## TODO: add support for this here? + // this.ws.onclose = (event) => { + // // websocket closing means 1 of 2 things: + // // - user navigated away from page intentionally + // // - server connection was broken somehow + // // only one of those is "bad" and we only want to + // // display warning in 2nd case. so we simply use a + // // brief delay to "rule out" the 1st scenario + // setTimeout(() => { that.websocketBroken = true }, + // 3000) + // } + + this.ws.onmessage = (event) => { + let data = JSON.parse(event.data) + + if (data.complete) { + + // upgrade has completed; reload page to view result + location.reload() + + } else if (data.stdout) { + + // add lines to textout area + that.progressOutput.push({ + key: ++that.progressOutputCounter, + text: data.stdout}) + + // scroll down to end of textout area + this.$nextTick(() => { + this.$refs.seeme.scrollIntoView() + }) + } + } + }) + } + + % else: + ## no websockets + + TailboneFormData.formSubmitting = false + + TailboneForm.methods.submitForm = function() { + this.formSubmitting = true + } + + % endif + TailboneFormData.declareFailureSubmitting = false TailboneForm.methods.declareFailureClick = function() { diff --git a/tailbone/views/asgi/__init__.py b/tailbone/views/asgi/__init__.py index a3450c11..01649f97 100644 --- a/tailbone/views/asgi/__init__.py +++ b/tailbone/views/asgi/__init__.py @@ -24,26 +24,77 @@ ASGI Views """ -from __future__ import unicode_literals, absolute_import +from http.cookies import SimpleCookie -import http.cookies +from beaker.session import SignedCookie +from pyramid.interfaces import ISessionFactory -from beaker.cache import clsmap -from beaker.session import SessionObject, SignedCookie + +class MockRequest(dict): + """ + Fake request class, needed for re-construction of the user's web + session. + """ + environ = {} + + def add_response_callback(self, func): + pass class WebsocketView(object): - def __init__(self, registry): - self.registry = registry + def __init__(self, pyramid_config): + self.pyramid_config = pyramid_config + self.registry = self.pyramid_config.registry + self.model = self.rattail_config.get_model() + + @property + def rattail_config(self): + return self.registry['rattail_config'] + + def get_rattail_app(self): + return self.rattail_config.get_app() + + async def authorize(self, scope, receive, send, permission): + + # is user authorized for this socket? + authorized = await self.has_permission(scope, permission) + + # wait for client to connect + message = await receive() + assert message['type'] == 'websocket.connect' + + # allow or deny access, per authorization + if authorized: + await send({'type': 'websocket.accept'}) + else: # forbidden + await send({'type': 'websocket.close'}) + + return authorized + + async def get_user(self, scope, session=None): + app = self.get_rattail_app() + model = self.model + + # load the user's web session + user_session = await self.get_user_session(scope) + if user_session: + + # determine user uuid + user_uuid = user_session.get('auth.userid') + if user_uuid: + + # use given db session, or make a new one + with app.short_session(config=self.rattail_config, + session=session): + + # load user proper + return session.query(model.User).get(user_uuid) async def get_user_session(self, scope): settings = self.registry.settings beaker_key = settings['beaker.session.key'] beaker_secret = settings['beaker.session.secret'] - beaker_type = settings['beaker.session.type'] - beaker_data_dir = settings['beaker.session.data_dir'] - beaker_lock_dir = settings['beaker.session.lock_dir'] # get ahold of session identifier cookie headers = dict(scope['headers']) @@ -51,20 +102,31 @@ class WebsocketView(object): if not cookie: return cookie = cookie.decode('utf_8') - cookie = http.cookies.SimpleCookie(cookie) + cookie = SimpleCookie(cookie) morsel = cookie[beaker_key] - # simulate pyramid_beaker logic to get at the session + # simulate pyramid_beaker logic to get at the actual session cookieheader = morsel.output(header='') cookie = SignedCookie(beaker_secret, input=cookieheader) session_id = cookie[beaker_key].value - request = {'cookie': cookieheader} - session = SessionObject( - request, - id=session_id, - key=beaker_key, - namespace_class=clsmap[beaker_type], - data_dir=beaker_data_dir, - lock_dir=beaker_lock_dir) + factory = self.registry.queryUtility(ISessionFactory) + request = MockRequest() + # nb. cannot pass 'id' to our factory, but things still work + # if we assign it immediately, before load() is called + session = factory(request) + session.id = session_id + session.load() return session + + async def has_permission(self, scope, permission): + app = self.get_rattail_app() + auth_handler = app.get_auth_handler() + + # figure out if user is authorized for this websocket + session = app.make_session() + user = await self.get_user(scope, session=session) + authorized = auth_handler.has_permission(session, user, permission) + session.close() + + return authorized diff --git a/tailbone/views/asgi/datasync.py b/tailbone/views/asgi/datasync.py index ffb63174..2dec06ea 100644 --- a/tailbone/views/asgi/datasync.py +++ b/tailbone/views/asgi/datasync.py @@ -24,8 +24,6 @@ DataSync Views """ -from __future__ import unicode_literals, absolute_import - import asyncio import json @@ -35,36 +33,11 @@ from tailbone.views.asgi import WebsocketView class DatasyncWS(WebsocketView): async def status(self, scope, receive, send): - rattail_config = self.registry['rattail_config'] - app = rattail_config.get_app() - model = app.model - auth_handler = app.get_auth_handler() + app = self.get_rattail_app() datasync_handler = app.get_datasync_handler() - authorized = False - user_session = await self.get_user_session(scope) - if user_session: - user_uuid = user_session.get('auth.userid') - session = app.make_session() - - user = None - if user_uuid: - user = session.query(model.User).get(user_uuid) - - # figure out if user is authorized for this websocket - permission = 'datasync.status' - authorized = auth_handler.has_permission(session, user, permission) - session.close() - - # wait for client to connect - message = await receive() - assert message['type'] == 'websocket.connect' - - # allow or deny access, per authorization - if authorized: - await send({'type': 'websocket.accept'}) - else: # forbidden - await send({'type': 'websocket.close'}) + # is user allowed to see this? + if not await self.authorize(scope, receive, send, 'datasync.status'): return # this tracks when client disconnects diff --git a/tailbone/views/asgi/upgrades.py b/tailbone/views/asgi/upgrades.py new file mode 100644 index 00000000..fc066326 --- /dev/null +++ b/tailbone/views/asgi/upgrades.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2022 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Upgrade Views for ASGI +""" + +import asyncio +import json +import os +from urllib.parse import parse_qs + +from tailbone.views.asgi import WebsocketView +from tailbone.progress import get_basic_session + + +class UpgradeWS(WebsocketView): + + async def execution_progress(self, scope, receive, send): + rattail_config = self.registry['rattail_config'] + + # is user allowed to see this? + if not await self.authorize(scope, receive, send, 'upgrades.execute'): + return + + # this tracks when client disconnects + state = {'disconnected': False} + + async def wait_for_disconnect(): + message = await receive() + if message['type'] == 'websocket.disconnect': + state['disconnected'] = True + + # watch for client disconnect, while we do other things + asyncio.create_task(wait_for_disconnect()) + + query = scope['query_string'].decode('utf_8') + query = parse_qs(query) + uuid = query['uuid'][0] + progress_session_id = 'upgrades.{}.execution_progress'.format(uuid) + progress_session = get_basic_session(rattail_config, + id=progress_session_id) + + # do the rest forever, until client disconnects + while not state['disconnected']: + + # load latest progress data + progress_session.load() + + # when upgrade progress is complete... + if progress_session.get('complete'): + + # maybe set success flash msg + msg = progress_session.get('success_msg') + if msg: + user_session = await self.get_user_session(scope) + user_session.flash(msg) + user_session.persist() + + # tell client progress is complete + await send({'type': 'websocket.send', + 'subtype': 'upgrades.execute_progress', + 'text': json.dumps({'complete': True})}) + + # this websocket is done + break + + # we will send this data down to client + data = dict(progress_session) + + # maybe add more lines from command output + path = rattail_config.upgrade_filepath(uuid, filename='stdout.log') + offset = progress_session.get('stdout.offset', 0) + if os.path.exists(path): + size = os.path.getsize(path) - offset + if size > 0: + with open(path, 'rb') as f: + f.seek(offset) + chunk = f.read(size) + data['stdout'] = chunk.decode('utf8').replace('\n', '
') + progress_session['stdout.offset'] = offset + size + progress_session.save() + + # send data to client + await send({'type': 'websocket.send', + 'subtype': 'upgrades.execute_progress', + 'text': json.dumps(data)}) + + # pause for 1 second + await asyncio.sleep(1) + + @classmethod + def defaults(cls, config): + cls._defaults(config) + + @classmethod + def _defaults(cls, config): + + # execution progress + config.add_tailbone_websocket('upgrades.execution_progress', + cls, attr='execution_progress') + + +def defaults(config, **kwargs): + base = globals() + + UpgradeWS = kwargs.get('UpgradeWS', base['UpgradeWS']) + UpgradeWS.defaults(config) + + +def includeme(config): + defaults(config) diff --git a/tailbone/views/core.py b/tailbone/views/core.py index bcb5b01b..c0f03e19 100644 --- a/tailbone/views/core.py +++ b/tailbone/views/core.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2021 Lance Edgar +# Copyright © 2010-2022 Lance Edgar # # This file is part of Rattail. # @@ -134,12 +134,12 @@ class View(object): def progress_loop(self, func, items, factory, *args, **kwargs): return progress_loop(func, items, factory, *args, **kwargs) - def make_progress(self, key): + def make_progress(self, key, **kwargs): """ Create and return a :class:`tailbone.progress.SessionProgress` instance, with the given key. """ - return SessionProgress(self.request, key) + return SessionProgress(self.request, key, **kwargs) # TODO: this signature seems wonky def render_progress(self, progress, kwargs, template=None): diff --git a/tailbone/views/master.py b/tailbone/views/master.py index 62502035..05c05ffd 100644 --- a/tailbone/views/master.py +++ b/tailbone/views/master.py @@ -1790,14 +1790,28 @@ class MasterView(View): """ obj = self.get_instance() model_title = self.get_model_title() - progress = self.make_execute_progress(obj) + # caller must explicitly request websocket behavior; otherwise + # we will assume traditional behavior for progress + ws = self.request.is_xhr and self.request.json_body.get('ws') + + # make our progress tracker + progress = self.make_execute_progress(obj, ws=ws) + + # start execution in a separate thread kwargs = {'progress': progress} key = [self.request.matchdict[k] for k in self.get_model_key(as_tuple=True)] - thread = Thread(target=self.execute_thread, args=(key, self.request.user.uuid), kwargs=kwargs) + thread = Thread(target=self.execute_thread, + args=(key, self.request.user.uuid), + kwargs=kwargs) thread.start() + # we're done here if using websockets + if ws: + return self.json_response({'ok': True}) + + # traditional behavior sends user to dedicated progress page return self.render_progress(progress, { 'instance': obj, 'initial_msg': self.execute_progress_initial_msg, @@ -1805,9 +1819,12 @@ class MasterView(View): 'cancel_msg': "{} execution was canceled".format(model_title), }, template=self.execute_progress_template) - def make_execute_progress(self, obj): - key = '{}.execute'.format(self.get_grid_key()) - return self.make_progress(key) + def make_execute_progress(self, obj, ws=False): + if ws: + key = '{}.{}.execution_progress'.format(self.get_route_prefix(), obj.uuid) + else: + key = '{}.execute'.format(self.get_grid_key()) + return self.make_progress(key, ws=ws) def get_instance_for_key(self, key, session): model_key = self.get_model_key(as_tuple=True) diff --git a/tailbone/views/upgrades.py b/tailbone/views/upgrades.py index 2e7c2fc4..dcab7980 100644 --- a/tailbone/views/upgrades.py +++ b/tailbone/views/upgrades.py @@ -26,24 +26,27 @@ Views for app upgrades from __future__ import unicode_literals, absolute_import +import json import os import re import logging +import warnings import six -from sqlalchemy import orm +import sqlalchemy as sa from rattail.core import Object from rattail.db import model, Session as RattailSession from rattail.time import make_utc from rattail.threads import Thread -from rattail.upgrades import get_upgrade_handler +from rattail.util import OrderedDict from deform import widget as dfwidget from webhelpers2.html import tags, HTML from tailbone.views import MasterView from tailbone.progress import get_progress_session #, SessionProgress +from tailbone.config import should_expose_websockets log = logging.getLogger(__name__) @@ -56,6 +59,7 @@ class UpgradeView(MasterView): model_class = model.Upgrade downloadable = True cloneable = True + configurable = True executable = True execute_progress_template = '/upgrade.mako' execute_progress_initial_msg = "Upgrading" @@ -68,6 +72,7 @@ class UpgradeView(MasterView): } grid_columns = [ + 'system', 'created', 'description', # 'not_until', @@ -78,6 +83,7 @@ class UpgradeView(MasterView): ] form_fields = [ + 'system', 'description', # 'not_until', # 'requirements', @@ -97,28 +103,40 @@ class UpgradeView(MasterView): def __init__(self, request): super(UpgradeView, self).__init__(request) - self.handler = self.get_handler() - def get_handler(self): - """ - Returns the ``UpgradeHandler`` instance for the view. The handler - factory for this may be defined by config, e.g.: + if hasattr(self, 'get_handler'): + warnings.warn("defining get_handler() is deprecated. please " + "override AppHandler.get_upgrade_handler() instead", + DeprecationWarning, stacklevel=2) + self.upgrade_handler = self.get_handler() - .. code-block:: ini + else: + app = self.get_rattail_app() + self.upgrade_handler = app.get_upgrade_handler() - [rattail.upgrades] - handler = myapp.upgrades:CustomUpgradeHandler - """ - return get_upgrade_handler(self.rattail_config) + @property + def handler(self): + warnings.warn("handler attribute is deprecated; " + "please use upgrade_handler instead", + DeprecationWarning, stacklevel=2) + return self.upgrade_handler def configure_grid(self, g): super(UpgradeView, self).configure_grid(g) + + # system + systems = self.upgrade_handler.get_all_systems() + systems_enum = dict([(s['key'], s['label']) for s in systems]) + g.set_enum('system', systems_enum) + g.set_joiner('executed_by', lambda q: q.join(model.User, model.User.uuid == model.Upgrade.executed_by_uuid).outerjoin(model.Person)) g.set_sorter('executed_by', model.Person.display_name) g.set_enum('status_code', self.enum.UPGRADE_STATUS) g.set_type('created', 'datetime') g.set_type('executed', 'datetime') g.set_sort_defaults('created', 'desc') + + g.set_link('system') g.set_link('created') g.set_link('description') # g.set_link('not_until') @@ -157,6 +175,16 @@ class UpgradeView(MasterView): super(UpgradeView, self).configure_form(f) upgrade = f.model_instance + # system + systems = self.upgrade_handler.get_all_systems() + systems_enum = OrderedDict([(s['key'], s['label']) + for s in systems]) + f.set_enum('system', systems_enum) + f.set_required('system') + if self.creating: + if len(systems) == 1: + f.set_default('system', list(systems_enum)[0]) + # status_code if self.creating: f.remove_field('status_code') @@ -174,7 +202,15 @@ class UpgradeView(MasterView): f.set_widget('notes', dfwidget.TextAreaWidget(cols=80, rows=8)) f.set_renderer('stdout_file', self.render_stdout_file) f.set_renderer('stderr_file', self.render_stdout_file) - f.set_renderer('package_diff', self.render_package_diff) + + # package_diff + if self.viewing and upgrade.executed and ( + upgrade.system == 'rattail' + or not upgrade.system): + f.set_renderer('package_diff', self.render_package_diff) + else: + f.remove_field('package_diff') + # f.set_readonly('created') # f.set_readonly('created_by') f.set_readonly('executed') @@ -202,7 +238,6 @@ class UpgradeView(MasterView): f.set_default('enabled', True) if not self.viewing or not upgrade.executed: - f.remove_field('package_diff') f.remove_field('exit_code') def render_status_code(self, upgrade, field): @@ -233,10 +268,11 @@ class UpgradeView(MasterView): return text def configure_clone_form(self, f): - f.fields = ['description', 'notes', 'enabled'] + f.fields = ['system', 'description', 'notes', 'enabled'] def clone_instance(self, original): cloned = self.model_class() + cloned.system = original.system cloned.created = make_utc() cloned.created_by = self.request.user cloned.description = original.description @@ -439,13 +475,22 @@ class UpgradeView(MasterView): # key = '{}.execute'.format(self.get_grid_key()) # return SessionProgress(self.request, key, session_type='file') - def execute_instance(self, upgrade, user, **kwargs): - session = orm.object_session(upgrade) - self.handler.mark_executing(upgrade) + def execute_instance(self, upgrade, user, progress=None, **kwargs): + app = self.get_rattail_app() + session = app.get_session(upgrade) + + # record the fact that execution has begun for this ugprade + self.upgrade_handler.mark_executing(upgrade) session.commit() - self.handler.do_execute(upgrade, user, **kwargs) - return ("Execution has finished, for better or worse. " - "You may need to restart your web app.") + + # let handler execute the upgrade + self.upgrade_handler.do_execute(upgrade, user, **kwargs) + + # success msg + msg = "Execution has finished, for better or worse." + if not upgrade.system or upgrade.system == 'rattail': + msg += " You may need to restart your web app." + return msg def execute_progress(self): upgrade = self.get_instance() @@ -489,6 +534,50 @@ class UpgradeView(MasterView): self.handler.delete_files(upgrade) super(UpgradeView, self).delete_instance(upgrade) + def configure_get_context(self, **kwargs): + context = super(UpgradeView, self).configure_get_context(**kwargs) + + context['upgrade_systems'] = self.upgrade_handler.get_all_systems() + + return context + + def configure_gather_settings(self, data): + settings = super(UpgradeView, self).configure_gather_settings(data) + + keys = [] + for system in json.loads(data['upgrade_systems']): + key = system['key'] + if key == 'rattail': + settings.append({'name': 'rattail.upgrades.command', + 'value': system['command']}) + else: + keys.append(key) + settings.append({'name': 'rattail.upgrades.system.{}.label'.format(key), + 'value': system['label']}) + settings.append({'name': 'rattail.upgrades.system.{}.command'.format(key), + 'value': system['command']}) + if keys: + settings.append({'name': 'rattail.upgrades.systems', + 'value': ', '.join(keys)}) + + return settings + + def configure_remove_settings(self): + super(UpgradeView, self).configure_remove_settings() + app = self.get_rattail_app() + model = self.model + + to_delete = self.Session.query(model.Setting)\ + .filter(sa.or_( + model.Setting.name == 'rattail.upgrades.command', + model.Setting.name == 'rattail.upgrades.systems', + model.Setting.name.like('rattail.upgrades.system.%.label'), + model.Setting.name.like('rattail.upgrades.system.%.command')))\ + .all() + + for setting in to_delete: + app.delete_setting(self.Session(), setting.name) + @classmethod def defaults(cls, config): cls._defaults(config) @@ -520,10 +609,14 @@ class UpgradeView(MasterView): def defaults(config, **kwargs): base = globals() + rattail_config = config.registry['rattail_config'] UpgradeView = kwargs.get('UpgradeView', base['UpgradeView']) UpgradeView.defaults(config) + if should_expose_websockets(rattail_config): + config.include('tailbone.views.asgi.upgrades') + def includeme(config): defaults(config) From e93063a3440288757b268bca2e89e8393c92ea05 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Aug 2022 18:55:33 -0500 Subject: [PATCH 010/879] Refactor upgrade websocket progress, so "anyone" can join in to see now while an upgrade is executing, anyone with permission can "view" the upgrade and see the same progress the executor is seeing --- tailbone/templates/upgrades/view.mako | 314 +++++++++++++++----------- tailbone/views/master.py | 10 + tailbone/views/upgrades.py | 7 + 3 files changed, 204 insertions(+), 127 deletions(-) diff --git a/tailbone/templates/upgrades/view.mako b/tailbone/templates/upgrades/view.mako index ed23c83a..f3884340 100644 --- a/tailbone/templates/upgrades/view.mako +++ b/tailbone/templates/upgrades/view.mako @@ -40,73 +40,22 @@ <%def name="extra_styles()"> ${parent.extra_styles()} - + % if master.has_perm('execute'): + + % endif <%def name="render_this_page()"> ${parent.render_this_page()} - % if master.has_perm('execute'): - ${h.form(master.get_action_url('declare_failure', instance), ref='declareFailureForm')} - ${h.csrf_token(request)} - ${h.end_form()} - % endif - - -<%def name="render_buefy_form()"> -
- <${form.component} - % if master.has_perm('execute'): - @declare-failure="declareFailure" - % endif - > - -
- - -<%def name="render_form_buttons()"> - % if not instance.executed and instance.status_code == enum.UPGRADE_STATUS_PENDING and master.has_perm('execute'): -
- % if instance.enabled and not instance.executing: - % if use_buefy and expose_websockets: - - {{ upgradeExecuting ? "Working, please wait..." : "Execute this upgrade" }} - - % elif use_buefy: - ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), **{'@submit': 'submitForm'})} - ${h.csrf_token(request)} - - {{ formSubmitting ? "Working, please wait..." : "Execute this upgrade" }} - - ${h.end_form()} - % else: - ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), class_='autodisable')} - ${h.csrf_token(request)} - ${h.submit('execute', "Execute this upgrade", class_='button is-primary')} - ${h.end_form()} - % endif - % elif instance.enabled: - - % else: - - % endif -
- + % if expose_websockets and master.has_perm('execute'): @@ -116,12 +65,15 @@
-

Upgrading (please wait) ...

+

+ Upgrading (please wait) ... + {{ executeUpgradeComplete ? "DONE!" : "" }} +

@@ -151,7 +103,64 @@
+ % endif + % if master.has_perm('execute'): + ${h.form(master.get_action_url('declare_failure', instance), ref='declareFailureForm')} + ${h.csrf_token(request)} + ${h.end_form()} + % endif + + +<%def name="render_buefy_form()"> +
+ <${form.component} + % if expose_websockets and master.has_perm('execute'): + @execute-upgrade-click="executeUpgrade" + :upgrade-executing="upgradeExecuting" + @declare-failure-click="declareFailureClick" + :declare-failure-submitting="declareFailureSubmitting" + % endif + > + +
+ + +<%def name="render_form_buttons()"> + % if instance_executable and master.has_perm('execute'): +
+ % if instance.enabled and not instance.executing: + % if use_buefy and expose_websockets: + + {{ upgradeExecuting ? "Working, please wait..." : "Execute this upgrade" }} + + % elif use_buefy: + ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), **{'@submit': 'submitForm'})} + ${h.csrf_token(request)} + + {{ formSubmitting ? "Working, please wait..." : "Execute this upgrade" }} + + ${h.end_form()} + % else: + ${h.form(url('{}.execute'.format(route_prefix), uuid=instance.uuid), class_='autodisable')} + ${h.csrf_token(request)} + ${h.submit('execute', "Execute this upgrade", class_='button is-primary')} + ${h.end_form()} + % endif + % elif instance.enabled: + + % else: + + % endif +
% endif @@ -165,69 +174,111 @@ % if expose_websockets: - TailboneFormData.upgradeExecuting = ${json.dumps(instance.executing)|n} - TailboneFormData.progressOutput = [] - TailboneFormData.progressOutputCounter = 0 + ThisPageData.ws = null - TailboneForm.methods.executeUpgrade = function() { - this.upgradeExecuting = true + ////////////////////////////// + // execute upgrade + ////////////////////////////// + + TailboneForm.props.upgradeExecuting = { + type: Boolean, + default: false, + } + + ThisPageData.upgradeExecuting = false + ThisPageData.progressOutput = [] + ThisPageData.progressOutputCounter = 0 + ThisPageData.executeUpgradeComplete = false + + ThisPage.methods.adjustTextoutHeight = function() { // grow the textout area to fill most of screen + let textout = this.$refs.textout + let height = window.innerHeight - textout.offsetTop - 50 + textout.style.height = height + 'px' + } + + ThisPage.methods.showExecuteDialog = function() { + this.upgradeExecuting = true this.$nextTick(() => { - let textout = this.$refs.textout - let height = window.innerHeight - textout.offsetTop - 50 - textout.style.height = height + 'px' - }) - - let url = '${master.get_action_url('execute', instance)}' - this.submitForm(url, {ws: true}, response => { - - ## TODO: should be a cleaner way to get this url? - url = '${request.route_url('ws.upgrades.execution_progress', _query={'uuid': instance.uuid})}' - url = url.replace(/^https?:/, 'wss:') - - this.ws = new WebSocket(url) - let that = this - - ## TODO: add support for this here? - // this.ws.onclose = (event) => { - // // websocket closing means 1 of 2 things: - // // - user navigated away from page intentionally - // // - server connection was broken somehow - // // only one of those is "bad" and we only want to - // // display warning in 2nd case. so we simply use a - // // brief delay to "rule out" the 1st scenario - // setTimeout(() => { that.websocketBroken = true }, - // 3000) - // } - - this.ws.onmessage = (event) => { - let data = JSON.parse(event.data) - - if (data.complete) { - - // upgrade has completed; reload page to view result - location.reload() - - } else if (data.stdout) { - - // add lines to textout area - that.progressOutput.push({ - key: ++that.progressOutputCounter, - text: data.stdout}) - - // scroll down to end of textout area - this.$nextTick(() => { - this.$refs.seeme.scrollIntoView() - }) - } - } + this.adjustTextoutHeight() }) } + ThisPage.methods.establishWebsocket = function() { + + ## TODO: should be a cleaner way to get this url? + url = '${request.route_url('ws.upgrades.execution_progress', _query={'uuid': instance.uuid})}' + url = url.replace(/^https?:/, 'wss:') + + this.ws = new WebSocket(url) + + ## TODO: add support for this here? + // this.ws.onclose = (event) => { + // // websocket closing means 1 of 2 things: + // // - user navigated away from page intentionally + // // - server connection was broken somehow + // // only one of those is "bad" and we only want to + // // display warning in 2nd case. so we simply use a + // // brief delay to "rule out" the 1st scenario + // setTimeout(() => { that.websocketBroken = true }, + // 3000) + // } + + this.ws.onmessage = (event) => { + let data = JSON.parse(event.data) + + if (data.complete) { + + // upgrade has completed; reload page to view result + this.executeUpgradeComplete = true + this.$nextTick(() => { + location.reload() + }) + + } else if (data.stdout) { + + // add lines to textout area + this.progressOutput.push({ + key: ++this.progressOutputCounter, + text: data.stdout}) + + // scroll down to end of textout area + this.$nextTick(() => { + this.$refs.seeme.scrollIntoView() + }) + } + } + } + + % if instance.executing: + ThisPage.mounted = function() { + this.showExecuteDialog() + this.establishWebsocket() + } + % endif + + % if instance_executable: + + ThisPage.methods.executeUpgrade = function() { + this.showExecuteDialog() + + let url = '${master.get_action_url('execute', instance)}' + this.submitForm(url, {ws: true}, response => { + + this.establishWebsocket() + }) + } + + % endif + % else: ## no websockets + ////////////////////////////// + // execute upgrade + ////////////////////////////// + TailboneFormData.formSubmitting = false TailboneForm.methods.submitForm = function() { @@ -236,17 +287,26 @@ % endif - TailboneFormData.declareFailureSubmitting = false + ////////////////////////////// + // declare failure + ////////////////////////////// - TailboneForm.methods.declareFailureClick = function() { - if (confirm("Really declare this upgrade a failure?")) { - this.declareFailureSubmitting = true - this.$emit('declare-failure') - } + TailboneForm.props.declareFailureSubmitting = { + type: Boolean, + default: false, } - ThisPage.methods.declareFailure = function() { - this.$refs.declareFailureForm.submit() + TailboneForm.methods.declareFailureClick = function() { + this.$emit('declare-failure-click') + } + + ThisPageData.declareFailureSubmitting = false + + ThisPage.methods.declareFailureClick = function() { + if (confirm("Really declare this upgrade a failure?")) { + this.declareFailureSubmitting = true + this.$refs.declareFailureForm.submit() + } } % endif diff --git a/tailbone/views/master.py b/tailbone/views/master.py index 05c05ffd..ad1d088d 100644 --- a/tailbone/views/master.py +++ b/tailbone/views/master.py @@ -1063,6 +1063,8 @@ class MasterView(View): 'instance_deletable': self.deletable_instance(instance), 'form': form, } + if self.executable: + context['instance_executable'] = self.executable_instance(instance) if hasattr(form, 'make_deform_form'): context['dform'] = form.make_deform_form() @@ -1784,6 +1786,14 @@ class MasterView(View): elif importer.allow_create: return importer.create_object(key, host_data) + def executable_instance(self, instance): + """ + Returns boolean indicating whether or not the given instance + can be considered "executable". Returns ``True`` by default; + override as necessary. + """ + return True + def execute(self): """ Execute an object. diff --git a/tailbone/views/upgrades.py b/tailbone/views/upgrades.py index dcab7980..0b5e4b87 100644 --- a/tailbone/views/upgrades.py +++ b/tailbone/views/upgrades.py @@ -475,6 +475,13 @@ class UpgradeView(MasterView): # key = '{}.execute'.format(self.get_grid_key()) # return SessionProgress(self.request, key, session_type='file') + def executable_instance(self, upgrade): + if upgrade.executed: + return False + if upgrade.status_code != self.enum.UPGRADE_STATUS_PENDING: + return False + return True + def execute_instance(self, upgrade, user, progress=None, **kwargs): app = self.get_rattail_app() session = app.get_session(upgrade) From 0a113611e865659890509e73c42efd4d8456508c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Aug 2022 21:19:20 -0500 Subject: [PATCH 011/879] Let just one "task" handle collect/transmit of progress for websocket first client to connect, will cause task to start; subsequent clients are just added to running set, for broadcast messaging --- tailbone/views/asgi/__init__.py | 4 +- tailbone/views/asgi/upgrades.py | 131 +++++++++++++++++++++++--------- 2 files changed, 99 insertions(+), 36 deletions(-) diff --git a/tailbone/views/asgi/__init__.py b/tailbone/views/asgi/__init__.py index 01649f97..68300a44 100644 --- a/tailbone/views/asgi/__init__.py +++ b/tailbone/views/asgi/__init__.py @@ -77,7 +77,7 @@ class WebsocketView(object): model = self.model # load the user's web session - user_session = await self.get_user_session(scope) + user_session = self.get_user_session(scope) if user_session: # determine user uuid @@ -91,7 +91,7 @@ class WebsocketView(object): # load user proper return session.query(model.User).get(user_uuid) - async def get_user_session(self, scope): + def get_user_session(self, scope): settings = self.registry.settings beaker_key = settings['beaker.session.key'] beaker_secret = settings['beaker.session.secret'] diff --git a/tailbone/views/asgi/upgrades.py b/tailbone/views/asgi/upgrades.py index fc066326..f06fc7d3 100644 --- a/tailbone/views/asgi/upgrades.py +++ b/tailbone/views/asgi/upgrades.py @@ -33,35 +33,92 @@ from tailbone.views.asgi import WebsocketView from tailbone.progress import get_basic_session -class UpgradeWS(WebsocketView): +class UpgradeExecutionProgressWS(WebsocketView): - async def execution_progress(self, scope, receive, send): - rattail_config = self.registry['rattail_config'] + # keep track of all "global" state for this socket + global_state = { + 'upgrades': {}, + } + + async def __call__(self, scope, receive, send): + app = self.get_rattail_app() # is user allowed to see this? if not await self.authorize(scope, receive, send, 'upgrades.execute'): return - # this tracks when client disconnects - state = {'disconnected': False} + # keep track of client state + client_state = { + 'uuid': app.make_uuid(), + 'disconnected': False, + 'scope': scope, + 'receive': receive, + 'send': send, + } + + # parse upgrade uuid from query string + query = scope['query_string'].decode('utf_8') + query = parse_qs(query) + uuid = query['uuid'][0] + + # first client to request progress for this upgrade, must + # start a task to manage the collect/transmit logic for + # progress data, on behalf of this and/or any future clients + started_task = None + if uuid not in self.global_state['upgrades']: + + # this upgrade is new to us; establish state and add first client + upgrade_state = self.global_state['upgrades'][uuid] = { + 'clients': {client_state['uuid']: client_state}, + } + + # start task for transmit of progress data to all clients + started_task = asyncio.create_task(self.manage_progress(uuid)) + + else: + + # progress task is already running, just add new client + upgrade_state = self.global_state['upgrades'][uuid] + upgrade_state['clients'][client_state['uuid']] = client_state async def wait_for_disconnect(): message = await receive() if message['type'] == 'websocket.disconnect': - state['disconnected'] = True + client_state['disconnected'] = True - # watch for client disconnect, while we do other things + # wait forever, until client disconnects asyncio.create_task(wait_for_disconnect()) + while not client_state['disconnected']: - query = scope['query_string'].decode('utf_8') - query = parse_qs(query) - uuid = query['uuid'][0] + # can stop if upgrade has completed + if uuid not in self.global_state['upgrades']: + break + + await asyncio.sleep(0.1) + + # remove client from global set, if upgrade still running + if client_state['disconnected']: + upgrade_state = self.global_state['upgrades'].get(uuid) + if upgrade_state: + del upgrade_state['clients'][client_state['uuid']] + + # must continue to wait for other clients, if this client was + # the first to request progress + if started_task: + await started_task + + async def manage_progress(self, uuid): + """ + Task which handles collect / transmit of progress data, for + sake of all attached clients. + """ progress_session_id = 'upgrades.{}.execution_progress'.format(uuid) - progress_session = get_basic_session(rattail_config, + progress_session = get_basic_session(self.rattail_config, id=progress_session_id) - # do the rest forever, until client disconnects - while not state['disconnected']: + upgrade_state = self.global_state['upgrades'][uuid] + clients = upgrade_state['clients'] + while clients: # load latest progress data progress_session.load() @@ -69,26 +126,30 @@ class UpgradeWS(WebsocketView): # when upgrade progress is complete... if progress_session.get('complete'): - # maybe set success flash msg + # maybe set success flash msg (for all clients) msg = progress_session.get('success_msg') if msg: - user_session = await self.get_user_session(scope) - user_session.flash(msg) - user_session.persist() + for client in clients.values(): + user_session = self.get_user_session(client['scope']) + user_session.flash(msg) + user_session.persist() - # tell client progress is complete - await send({'type': 'websocket.send', - 'subtype': 'upgrades.execute_progress', - 'text': json.dumps({'complete': True})}) + # tell clients progress is complete + for client in clients.values(): + await client['send']({ + 'type': 'websocket.send', + 'subtype': 'upgrades.execute_progress', + 'text': json.dumps({'complete': True})}) - # this websocket is done + # this websocket is done, so remove all clients + clients.clear() break # we will send this data down to client - data = dict(progress_session) + data = {} # maybe add more lines from command output - path = rattail_config.upgrade_filepath(uuid, filename='stdout.log') + path = self.rattail_config.upgrade_filepath(uuid, filename='stdout.log') offset = progress_session.get('stdout.offset', 0) if os.path.exists(path): size = os.path.getsize(path) - offset @@ -100,31 +161,33 @@ class UpgradeWS(WebsocketView): progress_session['stdout.offset'] = offset + size progress_session.save() - # send data to client - await send({'type': 'websocket.send', - 'subtype': 'upgrades.execute_progress', - 'text': json.dumps(data)}) + # send data to clients + for client in clients.values(): + await client['send']({ + 'type': 'websocket.send', + 'subtype': 'upgrades.execute_progress', + 'text': json.dumps(data)}) # pause for 1 second await asyncio.sleep(1) + # no more clients, no more reason to track this upgrade + del self.global_state['upgrades'][uuid] + @classmethod def defaults(cls, config): cls._defaults(config) @classmethod def _defaults(cls, config): - - # execution progress - config.add_tailbone_websocket('upgrades.execution_progress', - cls, attr='execution_progress') + config.add_tailbone_websocket('upgrades.execution_progress', cls) def defaults(config, **kwargs): base = globals() - UpgradeWS = kwargs.get('UpgradeWS', base['UpgradeWS']) - UpgradeWS.defaults(config) + UpgradeExecutionProgressWS = kwargs.get('UpgradeExecutionProgressWS', base['UpgradeExecutionProgressWS']) + UpgradeExecutionProgressWS.defaults(config) def includeme(config): From 2ca93a07e9f6475f437a031d32a8fa37966e93f4 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Aug 2022 22:40:16 -0500 Subject: [PATCH 012/879] Make separate tasks for collect vs. transmit of upgrade progress data --- tailbone/templates/upgrades/view.mako | 2 + tailbone/views/asgi/upgrades.py | 101 ++++++++++++++++++-------- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/tailbone/templates/upgrades/view.mako b/tailbone/templates/upgrades/view.mako index f3884340..90450c94 100644 --- a/tailbone/templates/upgrades/view.mako +++ b/tailbone/templates/upgrades/view.mako @@ -116,7 +116,9 @@
<${form.component} % if expose_websockets and master.has_perm('execute'): + % if instance_executable: @execute-upgrade-click="executeUpgrade" + % endif :upgrade-executing="upgradeExecuting" @declare-failure-click="declareFailureClick" :declare-failure-submitting="declareFailureSubmitting" diff --git a/tailbone/views/asgi/upgrades.py b/tailbone/views/asgi/upgrades.py index f06fc7d3..13458f23 100644 --- a/tailbone/views/asgi/upgrades.py +++ b/tailbone/views/asgi/upgrades.py @@ -40,6 +40,8 @@ class UpgradeExecutionProgressWS(WebsocketView): 'upgrades': {}, } + new_messages = asyncio.Queue() + async def __call__(self, scope, receive, send): app = self.get_rattail_app() @@ -116,10 +118,34 @@ class UpgradeExecutionProgressWS(WebsocketView): progress_session = get_basic_session(self.rattail_config, id=progress_session_id) + # start collecting status, textout messages + asyncio.create_task(self.collect_status(uuid, progress_session)) + asyncio.create_task(self.collect_textout(uuid)) + upgrade_state = self.global_state['upgrades'][uuid] clients = upgrade_state['clients'] while clients: + msg = await self.new_messages.get() + + # send message to all clients + for client in clients.values(): + await client['send']({ + 'type': 'websocket.send', + 'subtype': 'upgrades.execute_progress', + 'text': json.dumps(msg)}) + + await asyncio.sleep(0.1) + + # no more clients, no more reason to track this upgrade + del self.global_state['upgrades'][uuid] + + async def collect_status(self, uuid, progress_session): + + upgrade_state = self.global_state['upgrades'][uuid] + clients = upgrade_state['clients'] + while True: + # load latest progress data progress_session.load() @@ -134,45 +160,58 @@ class UpgradeExecutionProgressWS(WebsocketView): user_session.flash(msg) user_session.persist() - # tell clients progress is complete - for client in clients.values(): - await client['send']({ - 'type': 'websocket.send', - 'subtype': 'upgrades.execute_progress', - 'text': json.dumps({'complete': True})}) + # push "complete" message to queue + await self.new_messages.put({'complete': True}) - # this websocket is done, so remove all clients - clients.clear() + # there will be no more status coming break - # we will send this data down to client - data = {} + await asyncio.sleep(0.1) - # maybe add more lines from command output - path = self.rattail_config.upgrade_filepath(uuid, filename='stdout.log') - offset = progress_session.get('stdout.offset', 0) - if os.path.exists(path): + async def collect_textout(self, uuid): + path = self.rattail_config.upgrade_filepath(uuid, filename='stdout.log') + + # wait until stdout file exists + while not os.path.exists(path): + + # bail if upgrade is complete + if uuid not in self.global_state['upgrades']: + return + + await asyncio.sleep(0.1) + + offset = 0 + while True: + + # wait until we have something new to read + size = os.path.getsize(path) - offset + while not size: + + # bail if upgrade is complete + if uuid not in self.global_state['upgrades']: + return + + # wait a whole second, then look again + # (the less frequent we look, the bigger the chunk) + await asyncio.sleep(1) size = os.path.getsize(path) - offset - if size > 0: - with open(path, 'rb') as f: - f.seek(offset) - chunk = f.read(size) - data['stdout'] = chunk.decode('utf8').replace('\n', '
') - progress_session['stdout.offset'] = offset + size - progress_session.save() - # send data to clients - for client in clients.values(): - await client['send']({ - 'type': 'websocket.send', - 'subtype': 'upgrades.execute_progress', - 'text': json.dumps(data)}) + # bail if upgrade is complete + if uuid not in self.global_state['upgrades']: + return - # pause for 1 second - await asyncio.sleep(1) + # read the latest chunk and bookmark new offset + with open(path, 'rb') as f: + f.seek(offset) + chunk = f.read(size) + textout = chunk.decode('utf_8') + offset += size - # no more clients, no more reason to track this upgrade - del self.global_state['upgrades'][uuid] + # push new chunk onto message queue + textout = textout.replace('\n', '
') + await self.new_messages.put({'stdout': textout}) + + await asyncio.sleep(0.1) @classmethod def defaults(cls, config): From bdbbe990ddab24c0cee651d3aabd5e1141a026c8 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Aug 2022 23:07:19 -0500 Subject: [PATCH 013/879] Add global context from handler, for email previews --- tailbone/views/email.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tailbone/views/email.py b/tailbone/views/email.py index d381907d..536bf6ed 100644 --- a/tailbone/views/email.py +++ b/tailbone/views/email.py @@ -381,7 +381,15 @@ class EmailPreview(View): def __init__(self, request): super(EmailPreview, self).__init__(request) - self.email_handler = self.get_handler() + + if hasattr(self, 'get_handler'): + warnings.warn("defining a get_handler() method is deprecated; " + "please use AppHandler.get_email_handler() instead", + DeprecationWarning, stacklevel=2) + self.email_handler = get_handler() + else: + app = self.get_rattail_app() + self.email_handler = app.get_email_handler() @property def handler(self): @@ -390,10 +398,6 @@ class EmailPreview(View): DeprecationWarning, stacklevel=2) return self.email_handler - def get_handler(self): - app = self.get_rattail_app() - return app.get_email_handler() - def __call__(self): # Forms submitted via POST are only used for sending emails. @@ -416,10 +420,12 @@ class EmailPreview(View): key = self.request.POST.get('email_key') if key: email = self.email_handler.get_email(key) - data = email.obtain_sample_data(self.request) + + context = self.email_handler.make_context() + context.update(email.obtain_sample_data(self.request)) try: - self.email_handler.send_message(email, data, + self.email_handler.send_message(email, context, subject_prefix="[PREVIEW] ", to=[recipient], cc=None, bcc=None) @@ -433,8 +439,11 @@ class EmailPreview(View): def preview_template(self, key, type_): email = self.email_handler.get_email(key) template = email.get_template(type_) - data = email.obtain_sample_data(self.request) - self.request.response.text = template.render(**data) + + context = self.email_handler.make_context() + context.update(email.obtain_sample_data(self.request)) + + self.request.response.text = template.render(**context) if type_ == 'txt': self.request.response.content_type = str('text/plain') return self.request.response From 2ce242ba427bacaedeb1076507991ed2251e4a40 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Aug 2022 23:33:46 -0500 Subject: [PATCH 014/879] Make textout scrolling "smooth" for upgrade progress --- tailbone/templates/upgrades/view.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tailbone/templates/upgrades/view.mako b/tailbone/templates/upgrades/view.mako index 90450c94..c6ae11f2 100644 --- a/tailbone/templates/upgrades/view.mako +++ b/tailbone/templates/upgrades/view.mako @@ -247,7 +247,7 @@ // scroll down to end of textout area this.$nextTick(() => { - this.$refs.seeme.scrollIntoView() + this.$refs.seeme.scrollIntoView({behavior: 'smooth'}) }) } } From 87cced1637a88fa08dc022586048094e1782c228 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 21 Aug 2022 11:32:39 -0500 Subject: [PATCH 015/879] Fix perm check --- tailbone/templates/datasync/changes/index.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tailbone/templates/datasync/changes/index.mako b/tailbone/templates/datasync/changes/index.mako index 632f50ee..e92c3c3c 100644 --- a/tailbone/templates/datasync/changes/index.mako +++ b/tailbone/templates/datasync/changes/index.mako @@ -3,7 +3,7 @@ <%def name="context_menu_items()"> ${parent.context_menu_items()} - % if request.has_perm('datasync.list'): + % if request.has_perm('datasync.status'):
  • ${h.link_to("View DataSync Status", url('datasync.status'))}
    • % endif From 7b2fef5f093a615c812b473bcf460ec011ada6c4 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 21 Aug 2022 15:22:29 -0500 Subject: [PATCH 016/879] Allow configuring datasync watcher kwargs --- tailbone/templates/datasync/configure.mako | 197 ++++++++++++++++++++- tailbone/views/asgi/__init__.py | 4 +- tailbone/views/datasync.py | 23 ++- 3 files changed, 209 insertions(+), 15 deletions(-) diff --git a/tailbone/templates/datasync/configure.mako b/tailbone/templates/datasync/configure.mako index 2d6d6435..014668be 100644 --- a/tailbone/templates/datasync/configure.mako +++ b/tailbone/templates/datasync/configure.mako @@ -218,9 +218,111 @@ + + + {{ editingWatcherKwargs ? "Stop Editing" : "Edit Kwargs" }} + + + -
    +
    + + + New Watcher Kwarg + + +
    + + + + + + + + + + + + + + + + + + + Cancel + + + + Update Kwarg + + + + +
    + + + + + + + +
    + +
    @@ -512,6 +614,7 @@ ThisPage.methods.newProfile = function() { this.editingProfile = {} this.editingConsumer = null + this.editingWatcherKwargs = false this.editingProfileKey = null this.editingProfileWatcherSpec = null @@ -523,6 +626,7 @@ this.editingProfileWatcherConsumesSelf = false this.editingProfileEnabled = true this.editingProfilePendingConsumers = [] + this.editingProfilePendingWatcherKwargs = [] this.editProfileShowDialog = true this.$nextTick(() => { @@ -533,6 +637,7 @@ ThisPage.methods.editProfile = function(row) { this.editingProfile = row this.editingConsumer = null + this.editingWatcherKwargs = false this.editingProfileKey = row.key this.editingProfileWatcherSpec = row.watcher_spec @@ -544,6 +649,16 @@ this.editingProfileWatcherConsumesSelf = row.watcher_consumes_self this.editingProfileEnabled = row.enabled + this.editingProfilePendingWatcherKwargs = [] + for (let kwarg of row.watcher_kwargs_data) { + let pending = { + original_key: kwarg.key, + key: kwarg.key, + value: kwarg.value, + } + this.editingProfilePendingWatcherKwargs.push(pending) + } + this.editingProfilePendingConsumers = [] for (let consumer of row.consumers_data) { let pending = { @@ -563,6 +678,46 @@ this.editProfileShowDialog = true } + ThisPageData.editingWatcherKwargs = false + ThisPageData.editingProfilePendingWatcherKwargs = [] + ThisPageData.editingWatcherKwarg = null + ThisPageData.editingWatcherKwargKey = null + ThisPageData.editingWatcherKwargValue = null + + ThisPage.methods.newWatcherKwarg = function() { + this.editingWatcherKwargKey = null + this.editingWatcherKwargValue = null + this.editingWatcherKwarg = {key: null, value: null} + this.$nextTick(() => { + this.$refs.watcherKwargKey.focus() + }) + } + + ThisPage.methods.editProfileWatcherKwarg = function(row) { + this.editingWatcherKwargKey = row.key + this.editingWatcherKwargValue = row.value + this.editingWatcherKwarg = row + } + + ThisPage.methods.updateWatcherKwarg = function() { + let pending = this.editingWatcherKwarg + let isNew = !pending.key + + pending.key = this.editingWatcherKwargKey + pending.value = this.editingWatcherKwargValue + + if (isNew) { + this.editingProfilePendingWatcherKwargs.push(pending) + } + + this.editingWatcherKwarg = null + } + + ThisPage.methods.deleteProfileWatcherKwarg = function(row) { + let i = this.editingProfilePendingWatcherKwargs.indexOf(row) + this.editingProfilePendingWatcherKwargs.splice(i, 1) + } + ThisPage.methods.findOriginalConsumer = function(key) { for (let consumer of this.editingProfile.consumers_data) { if (consumer.key == key) { @@ -590,11 +745,39 @@ row.enabled = this.editingProfileEnabled // track which keys still belong (persistent) - let persistent = [] + let persistentWatcherKwargs = [] + + // transfer pending data to profile watcher kwargs + for (let pending of this.editingProfilePendingWatcherKwargs) { + persistentWatcherKwargs.push(pending.key) + if (pending.original_key) { + let kwarg = this.findOriginalWatcherKwarg(pending.original_key) + kwarg.key = pending.key + kwarg.value = pending.value + } else { + row.watcher_kwargs_data.push(pending) + } + } + + // remove any kwargs not being persisted + let removeWatcherKwargs = [] + for (let kwarg of row.watcher_kwargs_data) { + let i = persistentWatcherKwargs.indexOf(kwarg.key) + if (i < 0) { + removeWatcherKwargs.push(kwarg) + } + } + for (let kwarg of removeWatcherKwargs) { + let i = row.watcher_kwargs_data.indexOf(kwarg) + row.watcher_kwargs_data.splice(i, 1) + } + + // track which keys still belong (persistent) + let persistentConsumers = [] // transfer pending data to profile consumers for (let pending of this.editingProfilePendingConsumers) { - persistent.push(pending.key) + persistentConsumers.push(pending.key) if (pending.original_key) { let consumer = this.findOriginalConsumer(pending.original_key) consumer.key = pending.key @@ -611,14 +794,14 @@ } // remove any consumers not being persisted - let remove = [] + let removeConsumers = [] for (let consumer of row.consumers_data) { - let i = persistent.indexOf(consumer.key) + let i = persistentConsumers.indexOf(consumer.key) if (i < 0) { - remove.push(consumer) + removeConsumers.push(consumer) } } - for (let consumer of remove) { + for (let consumer of removeConsumers) { let i = row.consumers_data.indexOf(consumer) row.consumers_data.splice(i, 1) } diff --git a/tailbone/views/asgi/__init__.py b/tailbone/views/asgi/__init__.py index 68300a44..d0c12d9c 100644 --- a/tailbone/views/asgi/__init__.py +++ b/tailbone/views/asgi/__init__.py @@ -86,10 +86,10 @@ class WebsocketView(object): # use given db session, or make a new one with app.short_session(config=self.rattail_config, - session=session): + session=session) as s: # load user proper - return session.query(model.User).get(user_uuid) + return s.query(model.User).get(user_uuid) def get_user_session(self, scope): settings = self.registry.settings diff --git a/tailbone/views/datasync.py b/tailbone/views/datasync.py index 0f198795..c40d6aa2 100644 --- a/tailbone/views/datasync.py +++ b/tailbone/views/datasync.py @@ -202,6 +202,9 @@ class DataSyncThreadView(MasterView): 'watcher_retry_delay': profile.watcher.retry_delay, 'watcher_default_runas': profile.watcher.default_runas, 'watcher_consumes_self': profile.watcher.consumes_self, + 'watcher_kwargs_data': [{'key': key, + 'value': profile.watcher_kwargs[key]} + for key in sorted(profile.watcher_kwargs)], # 'notes': None, # TODO 'enabled': profile.enabled, } @@ -227,8 +230,7 @@ class DataSyncThreadView(MasterView): return { 'profiles': profiles, 'profiles_data': profiles_data, - 'use_profile_settings': self.rattail_config.getbool( - 'rattail.datasync', 'use_profile_settings'), + 'use_profile_settings': self.datasync_handler.should_use_profile_settings(), 'supervisor_process_name': self.rattail_config.get( 'rattail.datasync', 'supervisor_process_name'), 'restart_command': self.rattail_config.get( @@ -265,6 +267,13 @@ class DataSyncThreadView(MasterView): 'value': profile['watcher_default_runas']}, ]) + for kwarg in profile['watcher_kwargs_data']: + settings.append({ + 'name': 'rattail.datasync.{}.watcher.kwarg.{}'.format( + pkey, kwarg['key']), + 'value': kwarg['value'], + }) + consumers = [] if profile['watcher_consumes_self']: consumers = ['self'] @@ -298,11 +307,13 @@ class DataSyncThreadView(MasterView): settings.append({'name': 'rattail.datasync.watch', 'value': ', '.join(watch)}) - settings.append({'name': 'rattail.datasync.supervisor_process_name', - 'value': data['supervisor_process_name']}) + if data['supervisor_process_name']: + settings.append({'name': 'rattail.datasync.supervisor_process_name', + 'value': data['supervisor_process_name']}) - settings.append({'name': 'tailbone.datasync.restart', - 'value': data['restart_command']}) + if data['restart_command']: + settings.append({'name': 'tailbone.datasync.restart', + 'value': data['restart_command']}) return settings From e50356d276f75fbafac586ca7474c98a2d67ead4 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 21 Aug 2022 19:36:48 -0500 Subject: [PATCH 017/879] Expose, honor "admin-ish" flag for roles prevent user (un)assignment etc. unless admin is doing it --- tailbone/views/roles.py | 16 +++++++++++++++- tailbone/views/users.py | 17 +++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/tailbone/views/roles.py b/tailbone/views/roles.py index 78389d5d..61de606a 100644 --- a/tailbone/views/roles.py +++ b/tailbone/views/roles.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2021 Lance Edgar +# Copyright © 2010-2022 Lance Edgar # # This file is part of Rattail. # @@ -54,6 +54,7 @@ class RoleView(PrincipalMasterView): touchable = True labels = { + 'adminish': "Admin-ish", 'sync_me': "Sync Attrs & Perms", } @@ -68,6 +69,7 @@ class RoleView(PrincipalMasterView): form_fields = [ 'name', + 'adminish', 'session_timeout', 'notes', 'sync_me', @@ -112,6 +114,10 @@ class RoleView(PrincipalMasterView): if role is administrator_role(self.Session()): return self.request.is_root + # only "admin" can edit "admin-ish" roles + if role.adminish: + return self.request.is_admin + # can edit Authenticated only if user has permission if role is authenticated_role(self.Session()): return self.has_perm('edit_authenticated') @@ -143,6 +149,10 @@ class RoleView(PrincipalMasterView): if role is guest_role(self.Session()): return False + # only "admin" can delete "admin-ish" roles + if role.adminish: + return self.request.is_admin + # current user can delete their own roles, only if they have permission user = self.request.user if user and role in user.roles: @@ -169,6 +179,10 @@ class RoleView(PrincipalMasterView): # name f.set_validator('name', self.unique_name) + # adminish + if not self.request.is_admin: + f.remove('adminish') + # session_timeout f.set_renderer('session_timeout', self.render_session_timeout) if self.editing and role is guest_role(self.Session()): diff --git a/tailbone/views/users.py b/tailbone/views/users.py index 1fb1250d..0c5821b5 100644 --- a/tailbone/views/users.py +++ b/tailbone/views/users.py @@ -27,6 +27,7 @@ User Views from __future__ import unicode_literals, absolute_import import six +import sqlalchemy as sa from sqlalchemy import orm from rattail.db.model import User, UserEvent @@ -276,13 +277,21 @@ class UserView(PrincipalMasterView): authenticated_role(self.Session()).uuid, ] - # only allow "root" user to change admin role membership + # only allow "root" user to change true admin role membership if not self.request.is_root: excluded.append(administrator_role(self.Session()).uuid) - return self.Session.query(model.Role)\ - .filter(~model.Role.uuid.in_(excluded))\ - .order_by(model.Role.name) + # basic list, minus exclusions so far + roles = self.Session.query(model.Role)\ + .filter(~model.Role.uuid.in_(excluded)) + + # only allow "admin" user to change admin-ish role memberships + if not self.request.is_admin: + roles = roles.filter(sa.or_( + model.Role.adminish == False, + model.Role.adminish == None)) + + return roles.order_by(model.Role.name) def objectify(self, form, data=None): model = self.model From 6dfda201169e7eb1efd7c82d54a76c8f0b50d123 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 21 Aug 2022 20:41:55 -0500 Subject: [PATCH 018/879] Update changelog --- CHANGES.rst | 16 ++++++++++++++++ tailbone/_version.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b3631727..886c5399 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,22 @@ CHANGELOG ========= +0.8.250 (2022-08-21) +-------------------- + +* Add ``render_person_profile()`` method to MasterView. + +* Add way to declare failure for an upgrade. + +* Add websockets progress, "multi-system" support for upgrades. + +* Add global context from handler, for email previews. + +* Allow configuring datasync watcher kwargs. + +* Expose, honor "admin-ish" flag for roles. + + 0.8.249 (2022-08-18) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 5e741492..1063c3d3 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.249' +__version__ = '0.8.250' From 488696cb39717e61c53abe114db9083b3e3696a4 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 22 Aug 2022 01:07:58 -0500 Subject: [PATCH 019/879] Fix index title for datasync configure page --- tailbone/views/datasync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tailbone/views/datasync.py b/tailbone/views/datasync.py index c40d6aa2..316e17fe 100644 --- a/tailbone/views/datasync.py +++ b/tailbone/views/datasync.py @@ -54,7 +54,7 @@ class DataSyncThreadView(MasterView): For now it only serves the config view. """ model_title = "DataSync Thread" - model_title_plural = "DataSync Daemon" + model_title_plural = "DataSync Status" model_key = 'key' route_prefix = 'datasync' url_prefix = '/datasync' From 78500770d9e1c3089785f9925f7d759986d7774d Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 23 Aug 2022 23:27:47 -0500 Subject: [PATCH 020/879] Add basic support for backfill Luigi tasks idea being, sometimes you must import many days worth of data into Trainwreck or what-not, and it must be split up b/c e.g. it would take too long to import all at once (i.e. might interfere with overnight tasks) --- tailbone/templates/luigi/configure.mako | 341 ++++++++++++++++++++---- tailbone/templates/luigi/index.mako | 279 +++++++++++++++---- tailbone/views/luigi.py | 205 +++++++++++--- 3 files changed, 688 insertions(+), 137 deletions(-) diff --git a/tailbone/templates/luigi/configure.mako b/tailbone/templates/luigi/configure.mako index b8fba490..cf590adb 100644 --- a/tailbone/templates/luigi/configure.mako +++ b/tailbone/templates/luigi/configure.mako @@ -3,61 +3,213 @@ <%def name="form_content()"> ${h.hidden('overnight_tasks', **{':value': 'JSON.stringify(overnightTasks)'})} + ${h.hidden('backfill_tasks', **{':value': 'JSON.stringify(backfillTasks)'})} -

    Overnight Tasks

    +
    +
    +
    +

    Overnight Tasks

    +
    +
    + + New Task + +
    +
    +
    -
    - - New Task - + +
    - -
    +
    +

    Overnight Task

    +
    -
    -

    Overnight Task

    -
    +
    + + + + + + + + + + + + + + + + + +
    -
    - - - - -
    +
    + + Save + + + Cancel + +
    +
    +     -
    - - Save - - - Cancel - -
    -
    -     +
    +
    +
    +
    +

    Backfill Tasks

    +
    +
    + + New Task + +
    +
    + + + + + + +
    + +
    +

    Backfill Task

    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + + Save + + + Cancel + +
    +
    +     + +

    Luigi Proper

    @@ -65,8 +217,8 @@ - @@ -74,8 +226,8 @@ - @@ -83,8 +235,8 @@ - @@ -100,28 +252,113 @@ ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n} ThisPageData.overnightTaskShowDialog = false ThisPageData.overnightTask = null + ThisPageData.overnightTaskCounter = 0 ThisPageData.overnightTaskKey = null + ThisPageData.overnightTaskDescription = null + ThisPageData.overnightTaskScript = null + ThisPageData.overnightTaskNotes = null ThisPage.methods.overnightTaskCreate = function() { - this.overnightTask = null + this.overnightTask = {key: null} this.overnightTaskKey = null + this.overnightTaskDescription = null + this.overnightTaskScript = null + this.overnightTaskNotes = null this.overnightTaskShowDialog = true this.$nextTick(() => { - this.$refs.overnightTaskKey.focus() + this.$refs.overnightTaskDescription.focus() }) } + ThisPage.methods.overnightTaskEdit = function(task) { + this.overnightTask = task + this.overnightTaskKey = task.key + this.overnightTaskDescription = task.description + this.overnightTaskScript = task.script + this.overnightTaskNotes = task.notes + this.overnightTaskShowDialog = true + } + ThisPage.methods.overnightTaskSave = function() { - if (this.overnightTask) { - this.overnightTask.key = this.overnightTaskKey - } else { - let task = {key: this.overnightTaskKey} - this.overnightTasks.push(task) + this.overnightTask.description = this.overnightTaskDescription + this.overnightTask.script = this.overnightTaskScript + this.overnightTask.notes = this.overnightTaskNotes + + if (!this.overnightTask.key) { + this.overnightTask.key = `_new_${'$'}{++this.overnightTaskCounter}` + this.overnightTasks.push(this.overnightTask) } + this.overnightTaskShowDialog = false this.settingsNeedSaved = true } + ThisPage.methods.overnightTaskDelete = function(task) { + if (confirm("Really delete this task?")) { + let i = this.overnightTasks.indexOf(task) + this.overnightTasks.splice(i, 1) + this.settingsNeedSaved = true + } + } + + ThisPageData.backfillTasks = ${json.dumps(backfill_tasks)|n} + ThisPageData.backfillTaskShowDialog = false + ThisPageData.backfillTask = null + ThisPageData.backfillTaskCounter = 0 + ThisPageData.backfillTaskKey = null + ThisPageData.backfillTaskDescription = null + ThisPageData.backfillTaskScript = null + ThisPageData.backfillTaskForward = false + ThisPageData.backfillTaskTargetDate = null + ThisPageData.backfillTaskNotes = null + + ThisPage.methods.backfillTaskCreate = function() { + this.backfillTask = {key: null} + this.backfillTaskDescription = null + this.backfillTaskScript = null + this.backfillTaskForward = false + this.backfillTaskTargetDate = null + this.backfillTaskNotes = null + this.backfillTaskShowDialog = true + this.$nextTick(() => { + this.$refs.backfillTaskDescription.focus() + }) + } + + ThisPage.methods.backfillTaskEdit = function(task) { + this.backfillTask = task + this.backfillTaskDescription = task.description + this.backfillTaskScript = task.script + this.backfillTaskForward = task.forward + this.backfillTaskTargetDate = task.target_date + this.backfillTaskNotes = task.notes + this.backfillTaskShowDialog = true + } + + ThisPage.methods.backfillTaskDelete = function(task) { + if (confirm("Really delete this task?")) { + let i = this.backfillTasks.indexOf(task) + this.backfillTasks.splice(i, 1) + this.settingsNeedSaved = true + } + } + + ThisPage.methods.backfillTaskSave = function() { + this.backfillTask.description = this.backfillTaskDescription + this.backfillTask.script = this.backfillTaskScript + this.backfillTask.forward = this.backfillTaskForward + this.backfillTask.target_date = this.backfillTaskTargetDate + this.backfillTask.notes = this.backfillTaskNotes + + if (!this.backfillTask.key) { + this.backfillTask.key = `_new_${'$'}{++this.backfillTaskCounter}` + this.backfillTasks.push(this.backfillTask) + } + + this.backfillTaskShowDialog = false + this.settingsNeedSaved = true + } + diff --git a/tailbone/templates/luigi/index.mako b/tailbone/templates/luigi/index.mako index 16ea3489..c4407ff1 100644 --- a/tailbone/templates/luigi/index.mako +++ b/tailbone/templates/luigi/index.mako @@ -1,7 +1,7 @@ ## -*- coding: utf-8; -*- <%inherit file="/page.mako" /> -<%def name="title()">Luigi Jobs +<%def name="title()">View / Launch Tasks <%def name="page_content()">
    @@ -49,13 +49,141 @@ % endif
    - % if master.has_perm('launch'): + % if master.has_perm('launch_overnight'): +

    Overnight Tasks

    - % for task in overnight_tasks: - - - % endfor + + + + + + + % endif + + % if master.has_perm('launch_backfill'): + +

    Backfill Tasks

    + + + + + + + +
    + +
    +

    Launch Backfill Task

    +
    + +
    + +

    + {{ backfillTask.description }} + (goes {{ backfillTask.forward ? "FORWARD" : "BACKWARD" }}) +

    + + + + {{ backfillTask.last_date || "n/a" }} + + + {{ backfillTask.target_date || "n/a" }} + + + + + + + + + + + + + + + + + +
    + +
    + + Cancel + + + {{ backfillTaskLaunching ? "Working, please wait..." : "Launch" }} + +
    +
    +     + % endif
    @@ -63,8 +191,9 @@ <%def name="modify_this_page_vars()"> ${parent.modify_this_page_vars()} - % if master.has_perm('restart_scheduler'): - - % endif - + % endif -<%def name="finalize_this_page_vars()"> - ${parent.finalize_this_page_vars()} - % if master.has_perm('launch'): - - % endif - + let url = '${url('{}.launch_overnight'.format(route_prefix))}' + let params = {key: task.key} -<%def name="render_this_page_template()"> - ${parent.render_this_page_template()} - % if master.has_perm('launch'): - - % endif + this.submitForm(url, params, response => { + this.$buefy.toast.open({ + message: "Task has been scheduled for immediate launch!", + type: 'is-success', + duration: 5000, // 5 seconds + }) + this.overnightTaskLaunching = false + }) + } + + % endif + + % if master.has_perm('launch_backfill'): + + ThisPageData.backfillTasks = ${json.dumps(backfill_tasks)|n} + ThisPageData.backfillTask = null + ThisPageData.backfillTaskStartDate = null + ThisPageData.backfillTaskEndDate = null + ThisPageData.backfillTaskShowLaunchDialog = false + ThisPageData.backfillTaskLaunching = false + + ThisPage.methods.backfillTextClass = function(task) { + if (task.target_date) { + if (task.last_date) { + if (task.forward) { + if (task.last_date >= task.target_date) { + return 'has-text-success' + } else { + return 'has-text-warning' + } + } else { + if (task.last_date <= task.target_date) { + return 'has-text-success' + } else { + return 'has-text-warning' + } + } + } + } + } + + ThisPage.methods.backfillTaskLaunch = function(task) { + this.backfillTask = task + this.backfillTaskStartDate = null + this.backfillTaskEndDate = null + this.backfillTaskShowLaunchDialog = true + } + + ThisPage.methods.backfillTaskLaunchSubmit = function() { + this.backfillTaskLaunching = true + + let url = '${url('{}.launch_backfill'.format(route_prefix))}' + let params = { + key: this.backfillTask.key, + start_date: this.backfillTaskStartDate, + end_date: this.backfillTaskEndDate, + } + + this.submitForm(url, params, response => { + this.$buefy.toast.open({ + message: "Task has been scheduled for immediate launch!", + type: 'is-success', + duration: 5000, // 5 seconds + }) + this.backfillTaskLaunching = false + this.backfillTaskShowLaunchDialog = false + }) + } + + % endif + + diff --git a/tailbone/views/luigi.py b/tailbone/views/luigi.py index 6b0b60e3..dfc68d2f 100644 --- a/tailbone/views/luigi.py +++ b/tailbone/views/luigi.py @@ -27,19 +27,29 @@ Views for Luigi from __future__ import unicode_literals, absolute_import import json +import logging +import os +import re +import shlex + +import six +import sqlalchemy as sa from rattail.util import simple_error from tailbone.views import MasterView -class LuigiJobView(MasterView): +log = logging.getLogger(__name__) + + +class LuigiTaskView(MasterView): """ - Simple views for Luigi jobs. + Simple views for Luigi tasks. """ - normalized_model_name = 'luigijobs' - model_key = 'jobname' - model_title = "Luigi Job" + normalized_model_name = 'luigitasks' + model_key = 'key' + model_title = "Luigi Task" route_prefix = 'luigi' url_prefix = '/luigi' @@ -50,27 +60,57 @@ class LuigiJobView(MasterView): configurable = True def __init__(self, request, context=None): - super(LuigiJobView, self).__init__(request, context=context) + super(LuigiTaskView, self).__init__(request, context=context) app = self.get_rattail_app() self.luigi_handler = app.get_luigi_handler() def index(self): - luigi_url = self.rattail_config.get('luigi', 'url') + luigi_url = self.rattail_config.get('rattail.luigi', 'url') history_url = '{}/history'.format(luigi_url.rstrip('/')) if luigi_url else None return self.render_to_response('index', { 'use_buefy': self.get_use_buefy(), 'index_url': None, 'luigi_url': luigi_url, 'luigi_history_url': history_url, - 'overnight_tasks': self.luigi_handler.get_all_overnight_tasks(), + 'overnight_tasks': self.get_overnight_tasks(), + 'backfill_tasks': self.get_backfill_tasks(), }) - def launch(self): - key = self.request.POST['job'] - assert key - self.luigi_handler.restart_overnight_task(key) - self.request.session.flash("Scheduled overnight task for immediate launch: {}".format(key)) - return self.redirect(self.get_index_url()) + def launch_overnight(self): + app = self.get_rattail_app() + data = self.request.json_body + + key = data.get('key') + task = self.luigi_handler.get_overnight_task(key) if key else None + if not task: + return self.json_response({'error': "Task not found"}) + + try: + self.luigi_handler.launch_overnight_task(task, app.yesterday()) + except Exception as error: + log.warning("failed to launch overnight task: %s", task, + exc_info=True) + return self.json_response({'error': simple_error(error)}) + return self.json_response({'ok': True}) + + def launch_backfill(self): + app = self.get_rattail_app() + data = self.request.json_body + + key = data.get('key') + task = self.luigi_handler.get_backfill_task(key) if key else None + if not task: + return self.json_response({'error': "Task not found"}) + + start_date = app.parse_date(data['start_date']) + end_date = app.parse_date(data['end_date']) + try: + self.luigi_handler.launch_backfill_task(task, start_date, end_date) + except Exception as error: + log.warning("failed to launch backfill task: %s", task, + exc_info=True) + return self.json_response({'error': simple_error(error)}) + return self.json_response({'ok': True}) def restart_scheduler(self): try: @@ -87,36 +127,120 @@ class LuigiJobView(MasterView): return [ # luigi proper - {'section': 'luigi', + {'section': 'rattail.luigi', 'option': 'url'}, - {'section': 'luigi', + {'section': 'rattail.luigi', 'option': 'scheduler.supervisor_process_name'}, - {'section': 'luigi', + {'section': 'rattail.luigi', 'option': 'scheduler.restart_command'}, ] def configure_get_context(self, **kwargs): - context = super(LuigiJobView, self).configure_get_context(**kwargs) - context['overnight_tasks'] = self.luigi_handler.get_all_overnight_tasks() + context = super(LuigiTaskView, self).configure_get_context(**kwargs) + context['overnight_tasks'] = self.get_overnight_tasks() + context['backfill_tasks'] = self.get_backfill_tasks() return context - def configure_gather_settings(self, data): - settings = super(LuigiJobView, self).configure_gather_settings(data) + def get_overnight_tasks(self): + tasks = self.luigi_handler.get_all_overnight_tasks() + for task in tasks: + if task['last_date']: + task['last_date'] = six.text_type(task['last_date']) + return tasks + def get_backfill_tasks(self): + tasks = self.luigi_handler.get_all_backfill_tasks() + for task in tasks: + if task['last_date']: + task['last_date'] = six.text_type(task['last_date']) + if task['target_date']: + task['target_date'] = six.text_type(task['target_date']) + return tasks + + def configure_gather_settings(self, data): + settings = super(LuigiTaskView, self).configure_gather_settings(data) + app = self.get_rattail_app() + + # overnight tasks keys = [] for task in json.loads(data['overnight_tasks']): - keys.append(task['key']) + key = task['key'] + if key.startswith('_new_'): + key = app.make_uuid() + + key = task['key'] + if key.startswith('_new_'): + cmd = shlex.split(task['script']) + script = os.path.basename(cmd[0]) + root, ext = os.path.splitext(script) + key = re.sub(r'\s+', '-', root) + + keys.append(key) + settings.extend([ + {'name': 'rattail.luigi.overnight.{}.description'.format(key), + 'value': task['description']}, + {'name': 'rattail.luigi.overnight.{}.script'.format(key), + 'value': task['script']}, + {'name': 'rattail.luigi.overnight.{}.notes'.format(key), + 'value': task['notes']}, + ]) if keys: - settings.append({'name': 'luigi.overnight_tasks', + settings.append({'name': 'rattail.luigi.overnight_tasks', + 'value': ', '.join(keys)}) + + # backfill tasks + keys = [] + for task in json.loads(data['backfill_tasks']): + + key = task['key'] + if key.startswith('_new_'): + script = os.path.basename(task['script']) + root, ext = os.path.splitext(script) + key = re.sub(r'\s+', '-', root) + + keys.append(key) + settings.extend([ + {'name': 'rattail.luigi.backfill.{}.description'.format(key), + 'value': task['description']}, + {'name': 'rattail.luigi.backfill.{}.script'.format(key), + 'value': task['script']}, + {'name': 'rattail.luigi.backfill.{}.forward'.format(key), + 'value': 'true' if task['forward'] else 'false'}, + {'name': 'rattail.luigi.backfill.{}.notes'.format(key), + 'value': task['notes']}, + {'name': 'rattail.luigi.backfill.{}.target_date'.format(key), + 'value': six.text_type(task['target_date'])}, + ]) + if keys: + settings.append({'name': 'rattail.luigi.backfill_tasks', 'value': ', '.join(keys)}) return settings def configure_remove_settings(self): - super(LuigiJobView, self).configure_remove_settings() - self.luigi_handler.purge_luigi_settings(self.Session()) + super(LuigiTaskView, self).configure_remove_settings() + app = self.get_rattail_app() + model = self.model + session = self.Session() + + to_delete = session.query(model.Setting)\ + .filter(sa.or_( + model.Setting.name == 'rattail.luigi.backfill_tasks', + model.Setting.name.like('rattail.luigi.backfill.%.description'), + model.Setting.name.like('rattail.luigi.backfill.%.forward'), + model.Setting.name.like('rattail.luigi.backfill.%.notes'), + model.Setting.name.like('rattail.luigi.backfill.%.script'), + model.Setting.name.like('rattail.luigi.backfill.%.target_date'), + model.Setting.name == 'rattail.luigi.overnight_tasks', + model.Setting.name.like('rattail.luigi.overnight.%.description'), + model.Setting.name.like('rattail.luigi.overnight.%.notes'), + model.Setting.name.like('rattail.luigi.overnight.%.script')))\ + .all() + + for setting in to_delete: + app.delete_setting(session, setting.name) @classmethod def defaults(cls, config): @@ -130,16 +254,27 @@ class LuigiJobView(MasterView): url_prefix = cls.get_url_prefix() model_title_plural = cls.get_model_title_plural() - # launch job + # launch overnight config.add_tailbone_permission(permission_prefix, - '{}.launch'.format(permission_prefix), - label="Launch any Luigi job") - config.add_route('{}.launch'.format(route_prefix), - '{}/launch'.format(url_prefix), + '{}.launch_overnight'.format(permission_prefix), + label="Launch any Overnight Task") + config.add_route('{}.launch_overnight'.format(route_prefix), + '{}/launch-overnight'.format(url_prefix), request_method='POST') - config.add_view(cls, attr='launch', - route_name='{}.launch'.format(route_prefix), - permission='{}.launch'.format(permission_prefix)) + config.add_view(cls, attr='launch_overnight', + route_name='{}.launch_overnight'.format(route_prefix), + permission='{}.launch_overnight'.format(permission_prefix)) + + # launch backfill + config.add_tailbone_permission(permission_prefix, + '{}.launch_backfill'.format(permission_prefix), + label="Launch any Backfill Task") + config.add_route('{}.launch_backfill'.format(route_prefix), + '{}/launch-backfill'.format(url_prefix), + request_method='POST') + config.add_view(cls, attr='launch_backfill', + route_name='{}.launch_backfill'.format(route_prefix), + permission='{}.launch_backfill'.format(permission_prefix)) # restart luigid scheduler config.add_tailbone_permission(permission_prefix, @@ -156,8 +291,8 @@ class LuigiJobView(MasterView): def defaults(config, **kwargs): base = globals() - LuigiJobView = kwargs.get('LuigiJobView', base['LuigiJobView']) - LuigiJobView.defaults(config) + LuigiTaskView = kwargs.get('LuigiTaskView', base['LuigiTaskView']) + LuigiTaskView.defaults(config) def includeme(config): From bcedc58d9f958944ba24b3931c28062b62be853d Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 24 Aug 2022 18:24:42 -0500 Subject: [PATCH 021/879] Update changelog --- CHANGES.rst | 8 ++++++++ tailbone/_version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 886c5399..e691cc2f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,14 @@ CHANGELOG ========= +0.8.251 (2022-08-24) +-------------------- + +* Fix index title for datasync configure page. + +* Add basic support for backfill Luigi tasks. + + 0.8.250 (2022-08-21) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 1063c3d3..5cff828f 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.250' +__version__ = '0.8.251' From 2dbba970b9905f96676a734d56da9aa828e80009 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 24 Aug 2022 18:29:46 -0500 Subject: [PATCH 022/879] Only run tests if requested, for release task --- tasks.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tasks.py b/tasks.py index ed19d68f..48b51b39 100644 --- a/tasks.py +++ b/tasks.py @@ -37,13 +37,14 @@ exec(open(os.path.join(here, 'tailbone', '_version.py')).read()) @task -def release(ctx, skip_tests=False): +def release(c, tests=False): """ Release a new version of 'Tailbone'. """ - if not skip_tests: - ctx.run('tox') + if tests: + c.run('tox') - shutil.rmtree('Tailbone.egg-info') - ctx.run('python -m build --sdist') - ctx.run('twine upload dist/Tailbone-{}.tar.gz'.format(__version__)) + if os.path.exists('Tailbone.egg-info'): + shutil.rmtree('Tailbone.egg-info') + c.run('python -m build --sdist') + c.run('twine upload dist/Tailbone-{}.tar.gz'.format(__version__)) From 6a0a4627b4a127c40665dd93c810ddeef6b6f88f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 24 Aug 2022 20:06:38 -0500 Subject: [PATCH 023/879] Avoid error when no datasync profiles configured at least, according to the web app none are configured..but they may be in another config file --- tailbone/views/datasync.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tailbone/views/datasync.py b/tailbone/views/datasync.py index 316e17fe..e6c31721 100644 --- a/tailbone/views/datasync.py +++ b/tailbone/views/datasync.py @@ -97,7 +97,12 @@ class DataSyncThreadView(MasterView): process_info = None supervisor_error = simple_error(error) - profiles = self.datasync_handler.get_configured_profiles() + try: + profiles = self.datasync_handler.get_configured_profiles() + except Exception as error: + log.warning("could not load profiles!", exc_info=True) + self.request.session.flash(simple_error(error), 'error') + profiles = {} sql = """ select source, consumer, count(*) as changes From f005ef4d523b5c026a55eb252724a3c702f86a0f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 25 Aug 2022 22:15:56 -0500 Subject: [PATCH 024/879] Add max lengths when editing person name via profile view --- tailbone/templates/people/view_profile_buefy.mako | 12 +++++++++--- tailbone/views/people.py | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tailbone/templates/people/view_profile_buefy.mako b/tailbone/templates/people/view_profile_buefy.mako index cf665da9..51ecaed0 100644 --- a/tailbone/templates/people/view_profile_buefy.mako +++ b/tailbone/templates/people/view_profile_buefy.mako @@ -69,13 +69,19 @@
    - + + - + + - + +
    diff --git a/tailbone/views/people.py b/tailbone/views/people.py index 5dc76b73..1993c2e3 100644 --- a/tailbone/views/people.py +++ b/tailbone/views/people.py @@ -447,6 +447,9 @@ class PersonView(MasterView): def get_max_lengths(self): model = self.model return { + 'person_first_name': maxlen(model.Person.first_name), + 'person_middle_name': maxlen(model.Person.middle_name), + 'person_last_name': maxlen(model.Person.last_name), 'address_street': maxlen(model.PersonMailingAddress.street), 'address_street2': maxlen(model.PersonMailingAddress.street2), 'address_city': maxlen(model.PersonMailingAddress.city), From 36ba6f146341503f54c635218c162f9d67ce4757 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 25 Aug 2022 22:18:33 -0500 Subject: [PATCH 025/879] Update changelog --- CHANGES.rst | 8 ++++++++ tailbone/_version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e691cc2f..1bdff255 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,14 @@ CHANGELOG ========= +0.8.252 (2022-08-25) +-------------------- + +* Avoid error when no datasync profiles configured. + +* Add max lengths when editing person name via profile view. + + 0.8.251 (2022-08-24) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 5cff828f..c2efe75a 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.251' +__version__ = '0.8.252' From 187fea6d1b4deee67e39358915025e09643a7287 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 27 Aug 2022 22:45:52 -0500 Subject: [PATCH 026/879] Convert value for date filter; only add condition if valid --- tailbone/grids/filters.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tailbone/grids/filters.py b/tailbone/grids/filters.py index 06c4e7db..00f73e9b 100644 --- a/tailbone/grids/filters.py +++ b/tailbone/grids/filters.py @@ -682,6 +682,23 @@ class AlchemyDateFilter(AlchemyGridFilter): else: return dt.date() + def filter_equal(self, query, value): + date = self.make_date(value) + if not date: + return query + + return query.filter(self.column == self.encode_value(date)) + + def filter_not_equal(self, query, value): + date = self.make_date(value) + if not date: + return query + + return query.filter(sa.or_( + self.column == None, + self.column != self.encode_value(date), + )) + def filter_between(self, query, value): """ Filter data with a "between" query. Really this uses ">=" and "<=" From 6ea8a02b57b8a9020b621b06cf8882f6b3a9bd45 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 27 Aug 2022 23:36:09 -0500 Subject: [PATCH 027/879] Add 'warning' flash messages to old jquery base template --- tailbone/templates/base.mako | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tailbone/templates/base.mako b/tailbone/templates/base.mako index daa60e2d..43f3a1dd 100644 --- a/tailbone/templates/base.mako +++ b/tailbone/templates/base.mako @@ -138,6 +138,17 @@
    % endif + % if request.session.peek_flash('warning'): +
    + % for msg in request.session.pop_flash('warning'): +
    + + ${msg} +
    + % endfor +
    + % endif + % if request.session.peek_flash():
    % for msg in request.session.pop_flash(): From bb4e98af8d3d1eccd911cbecc00a0036daf7435d Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 10:58:13 -0500 Subject: [PATCH 028/879] Add uom fields, configurable template for newproduct batch --- .../static/files/newproduct_template.xlsx | Bin 0 -> 5041 bytes .../templates/batch/newproduct/configure.mako | 9 +++++ tailbone/views/batch/newproduct.py | 38 +++++++++++++++--- 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 tailbone/static/files/newproduct_template.xlsx create mode 100644 tailbone/templates/batch/newproduct/configure.mako diff --git a/tailbone/static/files/newproduct_template.xlsx b/tailbone/static/files/newproduct_template.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..82ce5ff1e5fb5f5f29db60c98f2a020aef725a17 GIT binary patch literal 5041 zcmaJ_by$>p)24UnB?P6@rCUl;x|ee4E=lPUL|UX31f--Cq*FQs$pr$zus_uMlOHB2mWG$0U&7U4>#k9N(dkoSJhyyhOZ&Yrx! zZXQlnZf;IIelE^Ds9(yGG@L&05k&4@+)zC3Q2|)TinhkeR$QE*f7D+7)GeGmQFj&0 zLv4&_yY=b$r^$mh0OSkUB^5z#ESQ9GgN#C`Q zO@&aZ@EX+|h#K-(btT?4E$CO(ES~Id3ftD^9Ym;%Wun7w=$08{*1B>+nmxO&Dy=68 ziOll@bNKoQabX^GNrIns608q74wfFaHo9IO4_xg% zQNW}m%c^1V6GZG@#*8-4x)U?0y}=(|R=~2P+4a8NteaL;r$YH-(s-T0FKLIr{}|E5 z9B{Qn=^iQ1W5MYr97i5voko*wB-SEfvY;aC>|U)VuuN~BuQ^03D&G%7P3+49B9My} zai6|l&O*^3c}0k>GG%VwW15n^Vbbxano-l-rgxKUSED5o`KLH2)C6;%c^)D$8Tcb= zH~&TrRY{%>wzgiLynlW0qmXNbYIx2HgIbTF5B>UzZ>Shn+eGPnOvx85vkK$_r*9B4 z$&y%uEYKj;7vG!FPS_nKG{;x~5e5=%4O6DcBTLP_vEp~RSeBgEa4;9V`JVZyPx-6` zx0zq^d(#$!o{_8)jLd<#;Jyb7U6Pc<0zC#|G94>J8Yu%Kl3I%F!-+I|G=ftq@B0!% z3#S4EC~x+x6~zX~ElB?mkWX{EHQeJQ>X)1dG)KJcGx;MGl5^QEJTFkWH_ z=irjszd5Q2O1XLDQQPBGDz$EB4nHsk>dnqe91gr%w^Wyrh4zq$rZ-Y{o^pHwVxHXF zGpjGBO-EdiBDmO+Z)D*HgILT+A{k5MwQ+V&f;%^E=wae5%%1@8%>Y=_MTtvz;F2Oe zhnkP%N_f57XRde(vNjhlTtv%otuk{%IkGVN+_igr6ZeN~n@Y*q{L5J>(sY8ld^(E0 zUfe=JL`Ky_6hl6ZUYFfY2#v81SQsiCg^``#5*(w}rGnSYg{T$9ycV;nsXrA(7*DTS z3%S6E#`ds_ECn@x&TW5p^hmX@+a3gnO}lc!)+kFBmI#gB=hut68x}q}xH= z??XW4g1_a>vc~A8pJ5I+kMSr5zh#NT!v&nHTBS@>z3Aq*Q}T{;h@`XUA!lG7^3o^) zOR_Yv*;Peg6La4*ILBg-qYliIpJ|>8*EijGXUep%?>`OJJA6u_FTX=Yq z?j1n(TA8=xT!LMQj`VeMnuAqnZLB+aTk6zFTf@3e>(&&w@kG8SoVxBr&1>2lGF@&l zhw@l`Rw8JQ9+X-xc;>gD6x$SSOV7WWE~FXSqUZx%An|nQ{+N zOp`dqgObfo89jRPyNrzt>hP71-eT-0H!GjT?M@44uk&HdzsqLe=@Yca`;2WM#$_Z& z9ITi0wtdME^BDIjH@c-`W|DN8I{51-&mFasBWmfpbefxw`O1!rgPp;5d`cOWMWJPN z&{w@o0P{ras@o$|QmovMUw{}>d}JpT>N}6v1r%!@g}<}Cc(>8rL*CGmrH1ax*mDH@ihJsCMfnxBiSJyi)q*<98mz18VlI{9B6BOpHUcpp}`;fPJ?C~=IJ}=jCtJ4{YQ@-bOz!L`wOuhYe(PkPW$#Hi7)F2R4f9QByJwF zKbCPQQ0I^vp4ewuw0YcXJjOWxfZ43}9#2E#XPu7(uqC6o=)AtG8`Je+0W0a-S?l(% z@q%J*m3>331$#`|Yg;JbXU5xoo%aP;K`n>umL#m`5L>Tl^@eVvOL9h3eTxw#*UX}M zvROxQXQh3kZ@W}lausLOERtJad>6lD%PW*z9FJ2#B^GuM(lJMwB)4Vx7M{7r3nEwr z|ICfPH{~G!2_D6kEydlYdTcuUWj;h+lO;+PRA2eq<>S#|B6ILX@3< z*Ft_MxFtt%%tkm4=d(BsVb`9IzzH9f7S4uC#^_?QhJe=N(8k#HSpK1nKP$(7T>E-Wi`&X0SHZXO2 z#YY~rP*Hi_c5hV31$^&|_dxB-Vk6z+(OLk2z~BdL#6q_bOEbQwln>L z=cD&+6p`N?*+m2LX82gaeh*2lNdZL2I5iO?8n!@*-ojW?>^)RNcqpHqVSBW2;*3JzNP`?rX^fcgeDNEhPgt zdnYseBVvIcIf;p!RJ7)v(Cyp`+5wlEK|~rnIW|8Qrn%1-`9({Dlvt}$9mOKh9e>u8 zga;);%7njDqbwJ zrXYS-#%hF983!iVs*LoLq5IsZS>tvG0(bR%SpPzsb`nX5XV;yR4WxHO!D9Gpbs`#3;1RBI>`eW=q!qPnx(2T1{^ z>pgv=g%?F!`y7iKhR;24EMCdq9t*Hc6O6D!xEB?bW>TxRqO9%wgbpIpvqmO95B+oT*eSj_J1n9hJQ{#GRLl_^ z_rq2n3zK|P2iwD#@>#F^X84mSs1j{bZN-w1p_ePK9r36G?gnFuXHyPK2Z!XNJr zP}so?3T0aK4-C?Q5ZmdS*Y~HUlZc)=B?V)cDAv z{#)1ot>u$PR5AHMQM*AgiGjZLH{@+fQQ%`vZRz8ysuAI&?=pT*seIu3tXS^Qlw%6G*KLs9JW zGBhud`>vL#t#W6* zls3&u)Y3vuIz79OVmM~%G>mgIiiF63?ipB7q3`t>-5YYxb(b~a4d9U4?IGC6$j|9u z=Vk+48phk(rdk;#x3NVxQhd&`TL?#1Gstlxbi4cjl47z<#=#%V@`*v(avTDkh1oQq z50RrR#1;xA)!WsArw-V5w~oiPo~Q1utU7lxx_ocSuVfI#E+MdIIAwN?KQVucK&-bN zN@d<#a~BRFRAhLl$h*3zohEC%*w7e6vBij$U)YfeT-X=^^0sq2K<|7B`cm#Q0~Os^+FzaBQ^JH_E7|o^5aFu+TkToRvJz%s%_# zcF6GYa{mgvw zf11?4vkE~Rjw^QwLGUBi36k69I$AX8a_)P+P-2OBM0!S6&E)Pu4IVo?Ccg?;pgM9d zJ4ltpq@=V+TN`MiK3Yi#|8OH^b(mWMuu<_c%SOHKP3hYzeBVRDsO6<^66GtA@!jml zyGcdbq{+{caKg%~BPHFg%;>xol%{-KeCTsJoTW#BCeDpzxmv_aIag_qF;N-vVSj&M zjm*yMA1j#*nIU62H&-uPS1)rt|A)4orhg4udN8dI4CL)+P$3fy%L9wp2MuBLN~X&a z)Zvp9lNjj24OjC@>#3s6n$};Rb_BNb(zytmig;IJtfzbyyER6lB7@JI!sCgMXxFOf z^YTcAiBLQWZJk*!!0R*m}QCCUWSZ zl}!>Lh=w}FNHd8WMH8b7I)8#6TvJ8aZY}~aZ-Hq%xy2f3x%-jZEY1*8D-VeGm7jvn zJ$ZyjWWV#^icOqFLz@kKG}<-{v{{D96NC-o>!jQvld?LbST}5r1sqkcUp+~H194jM zOdfnlk-C#mS!}r{TU!t`FnfGV??%KuoLQC4xSwNv?;OF>H_pWRKKG~YPF;liUf{mz zMD%63UuxBnlGy2AJX6BJRBejP4ip=8e$X+<(SB`Lu5TAm+m-*cf8VwI9_RYJjoL;0 zGAiVgQEKJ)&BX5!uIpvgT>i@-$f`m{_@7hy?@_KRD3rGTWurL%ALSpV{k!w^PK6R# zzYGWWH_7$8>vb+iwX$C}i~M@6vHk9No!n4V|H}>u{+sxJ_r8AcPz3nPXo#+P|3inr kyI-#i)am?XZAkF`$61A_VIyHgL&HU0(#W)!{PocP0~fMi5dZ)H literal 0 HcmV?d00001 diff --git a/tailbone/templates/batch/newproduct/configure.mako b/tailbone/templates/batch/newproduct/configure.mako new file mode 100644 index 00000000..e4fa346a --- /dev/null +++ b/tailbone/templates/batch/newproduct/configure.mako @@ -0,0 +1,9 @@ +## -*- coding: utf-8; -*- +<%inherit file="/configure.mako" /> + +<%def name="form_content()"> + ${self.input_file_templates_section()} + + + +${parent.body()} diff --git a/tailbone/views/batch/newproduct.py b/tailbone/views/batch/newproduct.py index e74ffcf6..23f5937b 100644 --- a/tailbone/views/batch/newproduct.py +++ b/tailbone/views/batch/newproduct.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2021 Lance Edgar +# Copyright © 2010-2022 Lance Edgar # # This file is part of Rattail. # @@ -46,6 +46,9 @@ class NewProductBatchView(BatchMasterView): rows_editable = True rows_bulk_deletable = True + configurable = True + has_input_file_templates = True + form_fields = [ 'id', 'input_filename', @@ -64,14 +67,14 @@ class NewProductBatchView(BatchMasterView): row_grid_columns = [ 'sequence', - 'upc', + '_product_key_', 'brand_name', 'description', 'size', 'vendor', 'vendor_item_code', - 'department', - 'subdepartment', + 'department_name', + 'subdepartment_name', 'regular_price', 'status_code', ] @@ -79,16 +82,20 @@ class NewProductBatchView(BatchMasterView): row_form_fields = [ 'sequence', 'product', - 'upc', + '_product_key_', 'brand_name', 'description', 'size', + 'unit_size', + 'unit_of_measure_entry', 'vendor_id', 'vendor', 'vendor_item_code', 'department_number', + 'department_name', 'department', 'subdepartment_number', + 'subdepartment_name', 'subdepartment', 'case_size', 'case_cost', @@ -108,6 +115,14 @@ class NewProductBatchView(BatchMasterView): 'status_text', ] + def get_input_file_templates(self): + return [ + {'key': 'default', + 'label': "Default", + 'default_url': self.request.static_url( + 'tailbone:static/files/newproduct_template.xlsx')}, + ] + def configure_form(self, f): super(NewProductBatchView, self).configure_form(f) @@ -127,6 +142,10 @@ class NewProductBatchView(BatchMasterView): g.set_type('pack_price', 'currency') g.set_type('suggested_price', 'currency') + g.set_link('brand_name') + g.set_link('description') + g.set_link('size') + def row_grid_extra_class(self, row, i): if row.status_code in (row.STATUS_MISSING_KEY, row.STATUS_PRODUCT_EXISTS, @@ -159,5 +178,12 @@ class NewProductBatchView(BatchMasterView): f.set_renderer('report', self.render_report) -def includeme(config): +def defaults(config, **kwargs): + base = globals() + + NewProductBatchView = kwargs.get('NewProductBatchView', base['NewProductBatchView']) NewProductBatchView.defaults(config) + + +def includeme(config): + defaults(config) From ef045607d9d93590df0d70c34b84d92d464fce13 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 11:04:26 -0500 Subject: [PATCH 029/879] Update changelog --- CHANGES.rst | 10 ++++++++++ tailbone/_version.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1bdff255..baf791a6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,16 @@ CHANGELOG ========= +0.8.253 (2022-08-30) +-------------------- + +* Convert value for date filter; only add condition if valid. + +* Add 'warning' flash messages to old jquery base template. + +* Add uom fields, configurable template for newproduct batch. + + 0.8.252 (2022-08-25) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index c2efe75a..2dc92815 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.252' +__version__ = '0.8.253' From 731c2168b0914d07a8ed144d596a9f51a5f240db Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 11:28:16 -0500 Subject: [PATCH 030/879] Improve parsing of purchase order quantities --- tailbone/views/purchasing/ordering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tailbone/views/purchasing/ordering.py b/tailbone/views/purchasing/ordering.py index c864ec35..d772a359 100644 --- a/tailbone/views/purchasing/ordering.py +++ b/tailbone/views/purchasing/ordering.py @@ -390,7 +390,7 @@ class OrderingBatchView(PurchasingBatchView): if cases_ordered == '': cases_ordered = 0 else: - cases_ordered = int(cases_ordered) + cases_ordered = int(float(cases_ordered)) if cases_ordered >= 100000: # TODO: really this depends on underlying column return {'error': "Invalid value for cases ordered: {}".format(cases_ordered)} @@ -401,7 +401,7 @@ class OrderingBatchView(PurchasingBatchView): if units_ordered == '': units_ordered = 0 else: - units_ordered = int(units_ordered) + units_ordered = int(float(units_ordered)) if units_ordered >= 100000: # TODO: really this depends on underlying column return {'error': "Invalid value for units ordered: {}".format(units_ordered)} From 12e4b0a1393d19d39383eede65df1918cb428322 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 13:57:18 -0500 Subject: [PATCH 031/879] Expose more attrs for new product batch rows --- tailbone/views/batch/newproduct.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tailbone/views/batch/newproduct.py b/tailbone/views/batch/newproduct.py index 23f5937b..03ca638b 100644 --- a/tailbone/views/batch/newproduct.py +++ b/tailbone/views/batch/newproduct.py @@ -97,6 +97,10 @@ class NewProductBatchView(BatchMasterView): 'subdepartment_number', 'subdepartment_name', 'subdepartment', + 'weighed', + 'tax1', + 'tax2', + 'tax3', 'case_size', 'case_cost', 'unit_cost', @@ -111,6 +115,7 @@ class NewProductBatchView(BatchMasterView): 'family', 'report_code', 'report', + 'ecommerce_available', 'status_code', 'status_text', ] From 9ea103c0ebe0c1124a6c14f1b8676828f9cfe2f7 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 14:18:57 -0500 Subject: [PATCH 032/879] Update changelog --- CHANGES.rst | 8 ++++++++ tailbone/_version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index baf791a6..96adc463 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,14 @@ CHANGELOG ========= +0.8.254 (2022-08-30) +-------------------- + +* Improve parsing of purchase order quantities. + +* Expose more attrs for new product batch rows. + + 0.8.253 (2022-08-30) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 2dc92815..2867b87f 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.253' +__version__ = '0.8.254' From 960d6279a9c70aa2b750ca8b3ef90cc23181e25f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 21:14:01 -0500 Subject: [PATCH 033/879] Include `WorkOrder.estimated_total` for API --- tailbone/api/workorders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tailbone/api/workorders.py b/tailbone/api/workorders.py index cac9e372..991df36a 100644 --- a/tailbone/api/workorders.py +++ b/tailbone/api/workorders.py @@ -55,6 +55,7 @@ class WorkOrderView(APIMasterView): 'id': workorder.id, 'customer_uuid': workorder.customer.uuid, 'customer_name': workorder.customer.name, + 'estimated_total': workorder.estimated_total, 'notes': workorder.notes, 'status_code': workorder.status_code, 'status_label': self.enum.WORKORDER_STATUS[workorder.status_code], From 35728e20be1898d39c494829170538df30bc65df Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Aug 2022 21:56:46 -0500 Subject: [PATCH 034/879] Add default normalize logic for API views and use common logic for getting field list in traditional Form class --- tailbone/api/master.py | 18 ++++++++++++++++++ tailbone/api/workorders.py | 13 ++++--------- tailbone/forms/core.py | 16 +++------------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/tailbone/api/master.py b/tailbone/api/master.py index 670a6104..97426214 100644 --- a/tailbone/api/master.py +++ b/tailbone/api/master.py @@ -28,7 +28,10 @@ from __future__ import unicode_literals, absolute_import import json +import six + from rattail.config import parse_bool +from rattail.db.util import get_fieldnames from cornice import resource, Service @@ -268,6 +271,21 @@ class APIMasterView(APIView): query = self.Session.query(cls) return query + def get_fieldnames(self): + if not hasattr(self, '_fieldnames'): + self._fieldnames = get_fieldnames( + self.rattail_config, self.model_class, + columns=True, proxies=True, relations=False) + return self._fieldnames + + def normalize(self, obj): + data = {'_str': six.text_type(obj)} + + for field in self.get_fieldnames(): + data[field] = getattr(obj, field) + + return data + def _collection_get(self): from sqlalchemy_filters import apply_filters, apply_sort, apply_pagination diff --git a/tailbone/api/workorders.py b/tailbone/api/workorders.py index 991df36a..eabe4cdb 100644 --- a/tailbone/api/workorders.py +++ b/tailbone/api/workorders.py @@ -49,21 +49,16 @@ class WorkOrderView(APIMasterView): self.workorder_handler = app.get_workorder_handler() def normalize(self, workorder): - return { - '_str': six.text_type(workorder), - 'uuid': workorder.uuid, - 'id': workorder.id, - 'customer_uuid': workorder.customer.uuid, + data = super(WorkOrderView, self).normalize(workorder) + data.update({ 'customer_name': workorder.customer.name, - 'estimated_total': workorder.estimated_total, - 'notes': workorder.notes, - 'status_code': workorder.status_code, 'status_label': self.enum.WORKORDER_STATUS[workorder.status_code], 'date_submitted': six.text_type(workorder.date_submitted or ''), 'date_received': six.text_type(workorder.date_received or ''), 'date_released': six.text_type(workorder.date_released or ''), 'date_delivered': six.text_type(workorder.date_delivered or ''), - } + }) + return data def create_object(self, data): diff --git a/tailbone/forms/core.py b/tailbone/forms/core.py index ac17c1b4..ee916d5f 100644 --- a/tailbone/forms/core.py +++ b/tailbone/forms/core.py @@ -37,6 +37,7 @@ from sqlalchemy.ext.associationproxy import AssociationProxy, ASSOCIATION_PROXY from rattail.time import localtime from rattail.util import prettify, pretty_boolean, pretty_quantity from rattail.core import UNSPECIFIED +from rattail.db.util import get_fieldnames import colander import deform @@ -396,19 +397,8 @@ class Form(object): if not self.model_class: raise ValueError("Must define model_class to use make_fields()") - mapper = orm.class_mapper(self.model_class) - - # first add primary column fields - fields = FieldList([prop.key for prop in mapper.iterate_properties - if not prop.key.startswith('_') - and prop.key != 'versions']) - - # then add association proxy fields - for key, desc in sa.inspect(self.model_class).all_orm_descriptors.items(): - if desc.extension_type == ASSOCIATION_PROXY: - fields.append(key) - - return fields + return get_fieldnames(self.request.rattail_config, self.model_class, + columns=True, proxies=True, relations=True) def make_renderers(self): """ From b5a519d132ef75c5b9366bb4a61c6e91706dcf49 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 31 Aug 2022 16:41:58 -0500 Subject: [PATCH 035/879] Disable "Delete Results" button if no results, for row grid --- tailbone/templates/batch/view.mako | 1 + 1 file changed, 1 insertion(+) diff --git a/tailbone/templates/batch/view.mako b/tailbone/templates/batch/view.mako index 919924f0..66a6881a 100644 --- a/tailbone/templates/batch/view.mako +++ b/tailbone/templates/batch/view.mako @@ -361,6 +361,7 @@ % if use_buefy and master.rows_bulk_deletable and not batch.executed and master.has_perm('delete_rows'): Delete Results From c43a4edec7ef1ea59794021fbf61658fe716f60f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 31 Aug 2022 20:52:17 -0500 Subject: [PATCH 036/879] Move logic for "bulk-delete row objects" into MasterView i guess so far it has only been needed for batch, but some day surely it will be needed for something else..? some of the template logic is still batch only i think.. --- tailbone/views/batch/core.py | 25 +++++++--------------- tailbone/views/master.py | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/tailbone/views/batch/core.py b/tailbone/views/batch/core.py index 24aa94d4..6dc2436d 100644 --- a/tailbone/views/batch/core.py +++ b/tailbone/views/batch/core.py @@ -1264,22 +1264,19 @@ class BatchMasterView(MasterView): """ self.handler.do_remove_row(row) - def bulk_delete_rows(self): - """ - "Delete" all rows matching the current row grid view query. This sets - the ``removed`` flag on the rows but does not truly delete them. - """ + def delete_row_objects(self, rows): + deleted = super(BatchMasterView, self).delete_row_objects(rows) batch = self.get_instance() - query = self.get_effective_row_data(sort=False) - # TODO: this should surely be handled by the handler... + # decrement rowcount for batch if batch.rowcount is not None: - batch.rowcount -= query.count() - query.update({'removed': True}, synchronize_session=False) + batch.rowcount -= deleted + + # refresh batch status self.Session.refresh(batch) self.handler.refresh_batch_status(batch) - return self.redirect(self.get_action_url('view', batch)) + return deleted def execute(self): """ @@ -1505,14 +1502,6 @@ class BatchMasterView(MasterView): config.add_tailbone_permission(permission_prefix, '{}.refresh'.format(permission_prefix), "Refresh data for {}".format(model_title)) - # bulk delete rows - if cls.rows_bulk_deletable: - config.add_route('{}.delete_rows'.format(route_prefix), '{}/{{uuid}}/rows/delete'.format(url_prefix)) - config.add_view(cls, attr='bulk_delete_rows', route_name='{}.delete_rows'.format(route_prefix), - permission='{}.delete_rows'.format(permission_prefix)) - config.add_tailbone_permission(permission_prefix, '{}.delete_rows'.format(permission_prefix), - "Bulk-delete data rows from {}".format(model_title)) - # toggle complete config.add_route('{}.toggle_complete'.format(route_prefix), '{}/{{{}}}/toggle-complete'.format(url_prefix, model_key)) config.add_view(cls, attr='toggle_complete', route_name='{}.toggle_complete'.format(route_prefix), diff --git a/tailbone/views/master.py b/tailbone/views/master.py index ad1d088d..c98d1a0e 100644 --- a/tailbone/views/master.py +++ b/tailbone/views/master.py @@ -4182,6 +4182,30 @@ class MasterView(View): self.delete_row_object(row) return self.redirect(self.get_action_url('view', self.get_parent(row))) + def bulk_delete_rows(self): + """ + Delete all row objects matching the current row grid query. + """ + obj = self.get_instance() + rows = self.get_effective_row_data(sort=False).all() + + # TODO: this should use a separate thread with progress + self.delete_row_objects(rows) + self.Session.refresh(obj) + + return self.redirect(self.get_action_url('view', obj)) + + def delete_row_objects(self, rows): + """ + Perform the actual deletion of given row objects. + """ + deleted = 0 + for row in rows: + if self.row_deletable(row): + self.delete_row_object(row) + deleted += 1 + return deleted + def get_parent(self, row): raise NotImplementedError @@ -4940,6 +4964,22 @@ class MasterView(View): config.add_view(cls, attr='create_row', route_name='{}.create_row'.format(route_prefix), permission='{}.create_row'.format(permission_prefix)) + # bulk-delete rows + # nb. must be defined before view_row b/c of url similarity + if cls.rows_bulk_deletable: + config.add_tailbone_permission(permission_prefix, + '{}.delete_rows'.format(permission_prefix), + "Bulk-delete {} from {}".format( + row_model_title_plural, model_title)) + config.add_route('{}.delete_rows'.format(route_prefix), + '{}/rows/delete'.format(instance_url_prefix), + # TODO: should enforce this + # request_method='POST' + ) + config.add_view(cls, attr='bulk_delete_rows', + route_name='{}.delete_rows'.format(route_prefix), + permission='{}.delete_rows'.format(permission_prefix)) + # view row if cls.has_rows: if cls.rows_viewable: From 365e4a41946eabfd5d79f4d630717c14eed0dd8a Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 6 Sep 2022 13:09:14 -0500 Subject: [PATCH 037/879] Convert value for more date filters; only add condition if valid missed these in 187fea6d1b4deee67e39358915025e09643a7287 --- tailbone/grids/filters.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tailbone/grids/filters.py b/tailbone/grids/filters.py index 00f73e9b..f504664b 100644 --- a/tailbone/grids/filters.py +++ b/tailbone/grids/filters.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2021 Lance Edgar +# Copyright © 2010-2022 Lance Edgar # # This file is part of Rattail. # @@ -699,6 +699,30 @@ class AlchemyDateFilter(AlchemyGridFilter): self.column != self.encode_value(date), )) + def filter_greater_than(self, query, value): + date = self.make_date(value) + if not date: + return query + return query.filter(self.column > self.encode_value(date)) + + def filter_greater_equal(self, query, value): + date = self.make_date(value) + if not date: + return query + return query.filter(self.column >= self.encode_value(date)) + + def filter_less_than(self, query, value): + date = self.make_date(value) + if not date: + return query + return query.filter(self.column < self.encode_value(date)) + + def filter_less_equal(self, query, value): + date = self.make_date(value) + if not date: + return query + return query.filter(self.column <= self.encode_value(date)) + def filter_between(self, query, value): """ Filter data with a "between" query. Really this uses ">=" and "<=" From b37f63a2319700e9ced88523cd1d9227a9afeeb3 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 6 Sep 2022 13:21:29 -0500 Subject: [PATCH 038/879] Update changelog --- CHANGES.rst | 14 ++++++++++++++ tailbone/_version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 96adc463..daa91c4a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,20 @@ CHANGELOG ========= +0.8.255 (2022-09-06) +-------------------- + +* Include ``WorkOrder.estimated_total`` for API. + +* Add default normalize logic for API views. + +* Disable "Delete Results" button if no results, for row grid. + +* Move logic for "bulk-delete row objects" into MasterView. + +* Convert value for more date filters; only add condition if valid. + + 0.8.254 (2022-08-30) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 2867b87f..cc4c6300 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.254' +__version__ = '0.8.255' From 2950827c63e533abf0497e0662333cd3bcbdd53b Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 6 Sep 2022 16:31:59 -0500 Subject: [PATCH 039/879] Add basic per-item discount support for custorders --- tailbone/templates/custorders/configure.mako | 9 ++++ tailbone/templates/custorders/create.mako | 52 +++++++++++++++++++- tailbone/views/custorders/orders.py | 23 ++++++++- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/tailbone/templates/custorders/configure.mako b/tailbone/templates/custorders/configure.mako index 1abbd7b2..0ce07f30 100644 --- a/tailbone/templates/custorders/configure.mako +++ b/tailbone/templates/custorders/configure.mako @@ -88,6 +88,15 @@ + + + Allow per-item discounts + + +
    diff --git a/tailbone/templates/custorders/create.mako b/tailbone/templates/custorders/create.mako index 4a92c063..f8d7096e 100644 --- a/tailbone/templates/custorders/create.mako +++ b/tailbone/templates/custorders/create.mako @@ -805,7 +805,21 @@ - + % if allow_item_discounts: + +
    +
    + + +
    +
    +  % +
    +
    +     + % endif + {{ getItemTotalPriceDisplay() }} @@ -981,6 +995,12 @@ + % if allow_item_discounts: + + {{ props.row.discount_percent }}{{ props.row.discount_percent ? " %" : "" }} + + % endif + { @@ -1882,6 +1922,10 @@ this.productUnitChoices = row.order_uom_choices this.productUOM = row.order_uom + % if allow_item_discounts: + this.productDiscountPercent = row.discount_percent + % endif + this.itemDialogTabIndex = 1 this.showingItemDialog = true }, @@ -1992,6 +2036,7 @@ }, itemDialogSave() { + this.itemDialogSaving = true let params = { product_is_known: this.productIsKnown, @@ -2002,6 +2047,10 @@ order_uom: this.productUOM, } + % if allow_item_discounts: + params.discount_percent = this.productDiscountPercent + % endif + if (this.productIsKnown) { params.product_uuid = this.productUUID } else { @@ -2032,6 +2081,7 @@ // also update the batch total price this.batchTotalPriceDisplay = response.data.batch.total_price_display + this.itemDialogSaving = false this.showingItemDialog = false }) }, diff --git a/tailbone/views/custorders/orders.py b/tailbone/views/custorders/orders.py index cf231374..224ec33a 100644 --- a/tailbone/views/custorders/orders.py +++ b/tailbone/views/custorders/orders.py @@ -348,6 +348,7 @@ class CustomerOrderView(MasterView): 'department_options': self.get_department_options(), 'default_uom_choices': self.batch_handler.uom_choices_for_product(None), 'default_uom': None, + 'allow_item_discounts': self.batch_handler.allow_item_discounts(), }) if self.batch_handler.allow_case_orders(): @@ -695,6 +696,7 @@ class CustomerOrderView(MasterView): 'order_quantity': pretty_quantity(row.order_quantity), 'order_uom': row.order_uom, 'order_uom_choices': self.uom_choices_for_row(row), + 'discount_percent': pretty_quantity(row.discount_percent), 'department_display': row.department_name, @@ -807,6 +809,7 @@ class CustomerOrderView(MasterView): order_quantity = decimal.Decimal(data.get('order_quantity') or '0') order_uom = data.get('order_uom') + discount_percent = decimal.Decimal(data.get('discount_percent') or '0') if data.get('product_is_known'): @@ -822,6 +825,9 @@ class CustomerOrderView(MasterView): if self.batch_handler.product_price_may_be_questionable(): kwargs['price_needs_confirmation'] = data.get('price_needs_confirmation') + if self.batch_handler.allow_item_discounts(): + kwargs['discount_percent'] = discount_percent + row = self.batch_handler.add_product(batch, product, order_quantity, order_uom, **kwargs) @@ -838,9 +844,14 @@ class CustomerOrderView(MasterView): pending_info['user'] = self.request.user + kwargs = {} + if self.batch_handler.allow_item_discounts(): + kwargs['discount_percent'] = discount_percent + row = self.batch_handler.add_pending_product(batch, pending_info, - order_quantity, order_uom) + order_quantity, order_uom, + **kwargs) self.Session.flush() return {'batch': self.normalize_batch(batch), @@ -860,6 +871,7 @@ class CustomerOrderView(MasterView): order_quantity = decimal.Decimal(data.get('order_quantity') or '0') order_uom = data.get('order_uom') + discount_percent = decimal.Decimal(data.get('discount_percent') or '0') if data.get('product_is_known'): @@ -879,6 +891,9 @@ class CustomerOrderView(MasterView): if self.batch_handler.product_price_may_be_questionable(): row.price_needs_confirmation = data.get('price_needs_confirmation') + if self.batch_handler.allow_item_discounts(): + row.discount_percent = discount_percent + self.batch_handler.refresh_row(row) else: # product is not known @@ -887,6 +902,9 @@ class CustomerOrderView(MasterView): row.order_quantity = order_quantity row.order_uom = order_uom + if self.batch_handler.allow_item_discounts(): + row.discount_percent = discount_percent + # nb. this will refresh the row pending_info = dict(data['pending_product']) self.batch_handler.update_pending_product(row, pending_info) @@ -965,6 +983,9 @@ class CustomerOrderView(MasterView): {'section': 'rattail.custorders', 'option': 'allow_unknown_product', 'type': bool}, + {'section': 'rattail.custorders', + 'option': 'allow_item_discounts', + 'type': bool}, ] @classmethod From f7a019ed83e0b1657ef66e8b34ebce34325e935d Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 6 Sep 2022 16:44:26 -0500 Subject: [PATCH 040/879] Make past item lookup optional for custorders --- tailbone/templates/custorders/configure.mako | 9 +++++++++ tailbone/templates/custorders/create.mako | 12 ++++++++++++ tailbone/views/custorders/orders.py | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/tailbone/templates/custorders/configure.mako b/tailbone/templates/custorders/configure.mako index 0ce07f30..6d51e433 100644 --- a/tailbone/templates/custorders/configure.mako +++ b/tailbone/templates/custorders/configure.mako @@ -97,6 +97,15 @@ + + + Allow re-order via past item lookup + + +
    diff --git a/tailbone/templates/custorders/create.mako b/tailbone/templates/custorders/create.mako index f8d7096e..cdbf584c 100644 --- a/tailbone/templates/custorders/create.mako +++ b/tailbone/templates/custorders/create.mako @@ -485,12 +485,14 @@ @click="showAddItemDialog()"> Add Item + % if allow_past_item_reorder: Add Past Item + % endif
    @@ -851,6 +853,7 @@ @selected="productLookupSelected"> + % if allow_past_item_reorder:
    @@ -953,6 +956,7 @@
    + % endif Date: Tue, 6 Sep 2022 22:19:01 -0500 Subject: [PATCH 041/879] Do not convert date if already a date --- tailbone/grids/filters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tailbone/grids/filters.py b/tailbone/grids/filters.py index f504664b..edce41dd 100644 --- a/tailbone/grids/filters.py +++ b/tailbone/grids/filters.py @@ -675,6 +675,9 @@ class AlchemyDateFilter(AlchemyGridFilter): Convert user input to a proper ``datetime.date`` object. """ if value: + if isinstance(value, datetime.date): + return value + try: dt = datetime.datetime.strptime(value, '%Y-%m-%d') except ValueError: From e67cde4255c53761c9dba630b3ddd150a2eec517 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 7 Sep 2022 20:46:18 -0500 Subject: [PATCH 042/879] Avoid use of `self.handler` within batch API views --- tailbone/api/batch/core.py | 41 ++++++++++++++++++++++----------- tailbone/api/batch/inventory.py | 8 +++---- tailbone/api/batch/ordering.py | 12 +++++----- tailbone/api/batch/receiving.py | 24 +++++++++++-------- 4 files changed, 52 insertions(+), 33 deletions(-) diff --git a/tailbone/api/batch/core.py b/tailbone/api/batch/core.py index bbba1fb3..5b6102ed 100644 --- a/tailbone/api/batch/core.py +++ b/tailbone/api/batch/core.py @@ -27,6 +27,7 @@ Tailbone Web API - Batch Views from __future__ import unicode_literals, absolute_import import logging +import warnings import six @@ -84,7 +85,14 @@ class APIBatchView(APIBatchMixin, APIMasterView): def __init__(self, request, **kwargs): super(APIBatchView, self).__init__(request, **kwargs) - self.handler = self.get_handler() + self.batch_handler = self.get_handler() + + @property + def handler(self): + warnings.warn("the `handler` property is deprecated; " + "please use `batch_handler` instead", + DeprecationWarning, stacklevel=2) + return self.batch_handler def normalize(self, batch): app = self.get_rattail_app() @@ -115,7 +123,7 @@ class APIBatchView(APIBatchMixin, APIMasterView): 'executed_display': self.pretty_datetime(executed) if executed else None, 'executed_by_uuid': batch.executed_by_uuid, 'executed_by_display': six.text_type(batch.executed_by or ''), - 'mutable': self.handler.is_mutable(batch), + 'mutable': self.batch_handler.is_mutable(batch), } def create_object(self, data): @@ -128,9 +136,9 @@ class APIBatchView(APIBatchMixin, APIMasterView): user = self.request.user kwargs = dict(data) kwargs['user'] = user - batch = self.handler.make_batch(self.Session(), **kwargs) - if self.handler.should_populate(batch): - self.handler.do_populate(batch, user) + batch = self.batch_handler.make_batch(self.Session(), **kwargs) + if self.batch_handler.should_populate(batch): + self.batch_handler.do_populate(batch, user) return batch def update_object(self, batch, data): @@ -198,7 +206,7 @@ class APIBatchView(APIBatchMixin, APIMasterView): kwargs = dict(self.request.json_body) kwargs.pop('user', None) kwargs.pop('progress', None) - result = self.handler.do_execute(batch, self.request.user, **kwargs) + result = self.batch_handler.do_execute(batch, self.request.user, **kwargs) return {'ok': bool(result), 'batch': self.normalize(batch)} @classmethod @@ -252,7 +260,14 @@ class APIBatchRowView(APIBatchMixin, APIMasterView): def __init__(self, request, **kwargs): super(APIBatchRowView, self).__init__(request, **kwargs) - self.handler = self.get_handler() + self.batch_handler = self.get_handler() + + @property + def handler(self): + warnings.warn("the `handler` property is deprecated; " + "please use `batch_handler` instead", + DeprecationWarning, stacklevel=2) + return self.batch_handler def normalize(self, row): batch = row.batch @@ -267,7 +282,7 @@ class APIBatchRowView(APIBatchMixin, APIMasterView): 'batch_description': batch.description, 'batch_complete': batch.complete, 'batch_executed': bool(batch.executed), - 'batch_mutable': self.handler.is_mutable(batch), + 'batch_mutable': self.batch_handler.is_mutable(batch), 'sequence': row.sequence, 'status_code': row.status_code, 'status_display': row.STATUS.get(row.status_code, six.text_type(row.status_code)), @@ -280,14 +295,14 @@ class APIBatchRowView(APIBatchMixin, APIMasterView): Invokes the batch handler's ``refresh_row()`` method after updating the row's field data per usual. """ - if not self.handler.is_mutable(row.batch): + if not self.batch_handler.is_mutable(row.batch): return {'error': "Batch is not mutable"} # update row per usual row = super(APIBatchRowView, self).update_object(row, data) # okay now we apply handler refresh logic - self.handler.refresh_row(row) + self.batch_handler.refresh_row(row) return row def delete_object(self, row): @@ -296,7 +311,7 @@ class APIBatchRowView(APIBatchMixin, APIMasterView): Delegates deletion of the row to the batch handler. """ - self.handler.do_remove_row(row) + self.batch_handler.do_remove_row(row) def quick_entry(self): """ @@ -312,10 +327,10 @@ class APIBatchRowView(APIBatchMixin, APIMasterView): entry = data['quick_entry'] try: - row = self.handler.quick_entry(self.Session(), batch, entry) + row = self.batch_handler.quick_entry(self.Session(), batch, entry) except Exception as error: log.warning("quick entry failed for '%s' batch %s: %s", - self.handler.batch_key, batch.id_str, entry, + self.batch_handler.batch_key, batch.id_str, entry, exc_info=True) msg = six.text_type(error) if not msg and isinstance(error, NotImplementedError): diff --git a/tailbone/api/batch/inventory.py b/tailbone/api/batch/inventory.py index f0c68030..5e56fe46 100644 --- a/tailbone/api/batch/inventory.py +++ b/tailbone/api/batch/inventory.py @@ -67,9 +67,9 @@ class InventoryBatchViews(APIBatchView): """ permission_prefix = self.get_permission_prefix() if self.request.is_root: - modes = self.handler.get_count_modes() + modes = self.batch_handler.get_count_modes() else: - modes = self.handler.get_allowed_count_modes( + modes = self.batch_handler.get_allowed_count_modes( self.Session(), self.request.user, permission_prefix=permission_prefix) return modes @@ -79,7 +79,7 @@ class InventoryBatchViews(APIBatchView): Retrieve info about the available "reasons" for inventory adjustment batches. """ - raw_reasons = self.handler.get_adjustment_reasons(self.Session()) + raw_reasons = self.batch_handler.get_adjustment_reasons(self.Session()) reasons = [] for reason in raw_reasons: reasons.append({ @@ -149,7 +149,7 @@ class InventoryBatchRowViews(APIBatchRowView): pretty_quantity(row.cases or row.units), 'CS' if row.cases else data['unit_uom']) - data['allow_cases'] = self.handler.allow_cases(batch) + data['allow_cases'] = self.batch_handler.allow_cases(batch) return data diff --git a/tailbone/api/batch/ordering.py b/tailbone/api/batch/ordering.py index b7bd45cb..9ab9617c 100644 --- a/tailbone/api/batch/ordering.py +++ b/tailbone/api/batch/ordering.py @@ -104,10 +104,10 @@ class OrderingBatchViews(APIBatchView): # organize vendor catalog costs by dept / subdept departments = {} - costs = self.handler.get_order_form_costs(self.Session(), batch.vendor) - costs = self.handler.sort_order_form_costs(costs) + costs = self.batch_handler.get_order_form_costs(self.Session(), batch.vendor) + costs = self.batch_handler.sort_order_form_costs(costs) costs = list(costs) # we must have a stable list for the rest of this - self.handler.decorate_order_form_costs(batch, costs) + self.batch_handler.decorate_order_form_costs(batch, costs) for cost in costs: department = cost.product.department @@ -175,7 +175,7 @@ class OrderingBatchViews(APIBatchView): sorted_departments.append(dept) # fetch recent purchase history, sort/pad for template convenience - history = self.handler.get_order_form_history(batch, costs, 6) + history = self.batch_handler.get_order_form_history(batch, costs, 6) for i in range(6 - len(history)): history.append(None) history = list(reversed(history)) @@ -266,10 +266,10 @@ class OrderingBatchRowViews(APIBatchRowView): Note that the "normal" logic for this method is not invoked at all. """ - if not self.handler.is_mutable(row.batch): + if not self.batch_handler.is_mutable(row.batch): return {'error': "Batch is not mutable"} - self.handler.update_row_quantity(row, **data) + self.batch_handler.update_row_quantity(row, **data) return row diff --git a/tailbone/api/batch/receiving.py b/tailbone/api/batch/receiving.py index ce7c34f6..c755de65 100644 --- a/tailbone/api/batch/receiving.py +++ b/tailbone/api/batch/receiving.py @@ -73,7 +73,7 @@ class ReceivingBatchViews(APIBatchView): data['invoice_total'] = batch.invoice_total data['invoice_total_calculated'] = batch.invoice_total_calculated - data['can_auto_receive'] = self.handler.can_auto_receive(batch) + data['can_auto_receive'] = self.batch_handler.can_auto_receive(batch) return data @@ -89,7 +89,7 @@ class ReceivingBatchViews(APIBatchView): a pending batch. """ batch = self.get_object() - self.handler.auto_receive_all_items(batch) + self.batch_handler.auto_receive_all_items(batch) return self._get(obj=batch) def mark_receiving_complete(self): @@ -119,7 +119,7 @@ class ReceivingBatchViews(APIBatchView): if not vendor: return {'error': "Vendor not found"} - purchases = self.handler.get_eligible_purchases( + purchases = self.batch_handler.get_eligible_purchases( vendor, self.enum.PURCHASE_BATCH_MODE_RECEIVING) purchases = [self.normalize_eligible_purchase(p) @@ -128,10 +128,10 @@ class ReceivingBatchViews(APIBatchView): return {'purchases': purchases} def normalize_eligible_purchase(self, purchase): - return self.handler.normalize_eligible_purchase(purchase) + return self.batch_handler.normalize_eligible_purchase(purchase) def render_eligible_purchase(self, purchase): - return self.handler.render_eligible_purchase(purchase) + return self.batch_handler.render_eligible_purchase(purchase) @classmethod def defaults(cls, config): @@ -321,6 +321,10 @@ class ReceivingBatchRowViews(APIBatchRowView): data['cases_expired'] = row.cases_expired data['units_expired'] = row.units_expired + cases, units = self.batch_handler.get_unconfirmed_counts(row) + data['cases_unconfirmed'] = cases + data['units_unconfirmed'] = units + data['po_unit_cost'] = row.po_unit_cost data['po_total'] = row.po_total @@ -328,7 +332,7 @@ class ReceivingBatchRowViews(APIBatchRowView): data['invoice_total'] = row.invoice_total data['invoice_total_calculated'] = row.invoice_total_calculated - data['allow_cases'] = self.handler.allow_cases() + data['allow_cases'] = self.batch_handler.allow_cases() data['quick_receive'] = self.rattail_config.getbool( 'rattail.batch', 'purchase.mobile_quick_receive', @@ -346,8 +350,8 @@ class ReceivingBatchRowViews(APIBatchRowView): raise NotImplementedError("TODO: add CS support for quick_receive_all") else: data['quick_receive_uom'] = data['unit_uom'] - accounted_for = self.handler.get_units_accounted_for(row) - remainder = self.handler.get_units_ordered(row) - accounted_for + accounted_for = self.batch_handler.get_units_accounted_for(row) + remainder = self.batch_handler.get_units_ordered(row) - accounted_for if accounted_for: # some product accounted for; button should receive "remainder" only @@ -389,7 +393,7 @@ class ReceivingBatchRowViews(APIBatchRowView): default=False) if alert_received: data['received_alert'] = None - if self.handler.get_units_confirmed(row): + if self.batch_handler.get_units_confirmed(row): msg = "You have already received some of this product; last update was {}.".format( humanize.naturaltime(app.make_utc() - row.modified)) data['received_alert'] = msg @@ -418,7 +422,7 @@ class ReceivingBatchRowViews(APIBatchRowView): # handler takes care of the row receiving logic for us kwargs = dict(form.validated) del kwargs['row'] - self.handler.receive_row(row, **kwargs) + self.batch_handler.receive_row(row, **kwargs) self.Session.flush() return self._get(obj=row) From 3877346b3a377dd35098819a66ff865de845ff5c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 9 Sep 2022 14:53:47 -0500 Subject: [PATCH 043/879] Update changelog --- CHANGES.rst | 12 ++++++++++++ tailbone/_version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index daa91c4a..c3cf9d7e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,18 @@ CHANGELOG ========= +0.8.256 (2022-09-09) +-------------------- + +* Add basic per-item discount support for custorders. + +* Make past item lookup optional for custorders. + +* Do not convert date if already a date (for grid filters). + +* Avoid use of ``self.handler`` within batch API views. + + 0.8.255 (2022-09-06) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index cc4c6300..2383e66f 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.255' +__version__ = '0.8.256' From 733e7ee00c1de7f0cc890eecc79314cba60fb308 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 24 Sep 2022 10:34:32 -0500 Subject: [PATCH 044/879] Add template method for rendering row grid component so custom event hooks can be added more easily, when needed --- tailbone/templates/master/view.mako | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tailbone/templates/master/view.mako b/tailbone/templates/master/view.mako index 32176712..7b0b2de5 100644 --- a/tailbone/templates/master/view.mako +++ b/tailbone/templates/master/view.mako @@ -107,13 +107,17 @@ % if rows_title:

    ${rows_title}

    % endif - + ${self.render_row_grid_component()} % else: ${rows_grid|n} % endif % endif +<%def name="render_row_grid_component()"> + + + <%def name="render_this_page_template()"> % if master.has_rows: ## TODO: stop using |n filter From 620447f02912ddad09f0beeee97bd6812ef1db2c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 25 Sep 2022 09:18:34 -0500 Subject: [PATCH 045/879] Add version workaround for sphinx-rtd-theme bug --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f65ca97..3328785e 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,9 @@ extras = { # # package # low high - 'Sphinx', # 1.2 + # TODO: remove version workaround after next sphinx[-rtd-theme] release + # cf. https://github.com/readthedocs/sphinx_rtd_theme/issues/1343 + 'Sphinx!=5.2.0.post0', # 1.2 'sphinx-rtd-theme', # 0.2.4 ], From 9b101963e5a944f42727a56d7fed239c6022ab84 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 18 Oct 2022 10:55:47 -0500 Subject: [PATCH 046/879] Use people handler to update address --- tailbone/views/people.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tailbone/views/people.py b/tailbone/views/people.py index 1993c2e3..6d517e3a 100644 --- a/tailbone/views/people.py +++ b/tailbone/views/people.py @@ -859,16 +859,8 @@ class PersonView(MasterView): data = dict(self.request.json_body) # update person address - address = person.address - if not address: - address = person.add_address() - address.street = data['street'] - address.street2 = data['street2'] - address.city = data['city'] - address.state = data['state'] - address.zipcode = data['zipcode'] - - self.handler.mark_address_invalid(person, address, data['invalid']) + address = self.people_handler.ensure_address(person) + self.people_handler.update_address(person, address, **data) self.Session.flush() return { From 22c33b58c7dcc81ead922c7a0bfed2f2a7805dce Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 19 Oct 2022 16:26:05 -0500 Subject: [PATCH 047/879] Fix start_date param for pricing batch upload --- tailbone/views/batch/pricing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tailbone/views/batch/pricing.py b/tailbone/views/batch/pricing.py index cb0f3be9..6ba28889 100644 --- a/tailbone/views/batch/pricing.py +++ b/tailbone/views/batch/pricing.py @@ -193,6 +193,7 @@ class PricingBatchView(BatchMasterView): def get_batch_kwargs(self, batch, **kwargs): kwargs = super(PricingBatchView, self).get_batch_kwargs(batch, **kwargs) + kwargs['start_date'] = batch.start_date kwargs['min_diff_threshold'] = batch.min_diff_threshold kwargs['min_diff_percent'] = batch.min_diff_percent kwargs['calculate_for_manual'] = batch.calculate_for_manual From c2b2d1114187f264102f95e6989a6ad0b417d483 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 29 Oct 2022 13:40:35 -0500 Subject: [PATCH 048/879] Use shared logic for rendering percentage values --- tailbone/forms/core.py | 5 ++--- tailbone/grids/core.py | 5 ++--- tailbone/views/products.py | 4 +++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tailbone/forms/core.py b/tailbone/forms/core.py index ee916d5f..fb11ffba 100644 --- a/tailbone/forms/core.py +++ b/tailbone/forms/core.py @@ -1006,10 +1006,9 @@ class Form(object): return pretty_quantity(value) def render_percent(self, obj, field): + app = self.request.rattail_config.get_app() value = self.obtain_value(obj, field) - if value is None: - return "" - return "{:0.3f} %".format(value * 100) + return app.render_percent(value, places=3) def render_gpc(self, obj, field): value = self.obtain_value(obj, field) diff --git a/tailbone/grids/core.py b/tailbone/grids/core.py index b15dcafd..db976432 100644 --- a/tailbone/grids/core.py +++ b/tailbone/grids/core.py @@ -369,10 +369,9 @@ class Grid(object): return value.pretty() def render_percent(self, obj, column_name): + app = self.request.rattail_config.get_app() value = self.obtain_value(obj, column_name) - if value is None: - return "" - return "{:0.3f} %".format(value * 100) + return app.render_percent(value, places=3) def render_quantity(self, obj, column_name): value = self.obtain_value(obj, column_name) diff --git a/tailbone/views/products.py b/tailbone/views/products.py index 8f1ea545..ab9f55c6 100644 --- a/tailbone/views/products.py +++ b/tailbone/views/products.py @@ -670,7 +670,9 @@ class ProductView(MasterView): return "" if product.volatile.true_margin is None: return "" - return "{:0.3f} %".format(product.volatile.true_margin * 100) + app = self.get_rattail_app() + return app.render_percent(product.volatile.true_margin, + places=3) def render_on_hand(self, product, column): inventory = product.inventory From 38e6441b61cafdda81b744c888738fa966d7d89e Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 31 Oct 2022 21:41:01 -0500 Subject: [PATCH 049/879] Log a warning to troubleshoot luigi restart failure --- tailbone/views/luigi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tailbone/views/luigi.py b/tailbone/views/luigi.py index dfc68d2f..054f24ee 100644 --- a/tailbone/views/luigi.py +++ b/tailbone/views/luigi.py @@ -118,6 +118,7 @@ class LuigiTaskView(MasterView): self.request.session.flash("Luigi scheduler has been restarted.") except Exception as error: + log.warning("restart failed", exc_info=True) self.request.session.flash(simple_error(error), 'error') return self.redirect(self.request.get_referrer( From be533922a2c2dbea83e670ae8092a4170519a3f7 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 3 Nov 2022 11:28:38 -0500 Subject: [PATCH 050/879] Show UPC for receiving line item if no product reference to help with troubleshooting invoice file parsing etc. --- tailbone/templates/receiving/view_row.mako | 5 ++++- tailbone/views/purchasing/receiving.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tailbone/templates/receiving/view_row.mako b/tailbone/templates/receiving/view_row.mako index bb4275b8..dca71c35 100644 --- a/tailbone/templates/receiving/view_row.mako +++ b/tailbone/templates/receiving/view_row.mako @@ -85,8 +85,11 @@ ${form.render_field_readonly(product_key_field)} ${form.render_field_readonly('product')} % else: - ${form.render_field_readonly('item_entry')} ${form.render_field_readonly(product_key_field)} + ${form.render_field_readonly('item_entry')} + % if product_key_field != 'upc': + ${form.render_field_readonly('upc')} + % endif ${form.render_field_readonly('brand_name')} ${form.render_field_readonly('description')} ${form.render_field_readonly('size')} diff --git a/tailbone/views/purchasing/receiving.py b/tailbone/views/purchasing/receiving.py index af96448f..2fe692f0 100644 --- a/tailbone/views/purchasing/receiving.py +++ b/tailbone/views/purchasing/receiving.py @@ -1479,6 +1479,14 @@ class ReceivingBatchView(PurchasingBatchView): super(ReceivingBatchView, self).configure_row_form(f) batch = self.get_instance() + # when viewing a row which has no product reference, enable + # the 'upc' field to help with troubleshooting + # TODO: this maybe should be optional..? + if self.viewing and 'upc' not in f: + row = self.get_row_instance() + if not row.product: + f.append('upc') + # allow input for certain fields only; all others are readonly mutable = [ 'invoice_unit_cost', From 3b64950a3852bd0e2ee49d8e73e1bae3e6072a82 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 3 Nov 2022 11:34:32 -0500 Subject: [PATCH 051/879] Update changelog --- CHANGES.rst | 16 ++++++++++++++++ tailbone/_version.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c3cf9d7e..a1a03d46 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,22 @@ CHANGELOG ========= +0.8.257 (2022-11-03) +-------------------- + +* Add template method for rendering row grid component. + +* Use people handler to update address. + +* Fix start_date param for pricing batch upload. + +* Use shared logic for rendering percentage values. + +* Log a warning to troubleshoot luigi restart failure. + +* Show UPC for receiving line item if no product reference. + + 0.8.256 (2022-09-09) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 2383e66f..8f293897 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.256' +__version__ = '0.8.257' From fec259629e164e0be9e301b286970de3c54445aa Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 15 Nov 2022 13:37:37 -0600 Subject: [PATCH 052/879] Let the auth handler manage user merge --- tailbone/views/users.py | 50 +++++++---------------------------------- 1 file changed, 8 insertions(+), 42 deletions(-) diff --git a/tailbone/views/users.py b/tailbone/views/users.py index 0c5821b5..31842d0b 100644 --- a/tailbone/views/users.py +++ b/tailbone/views/users.py @@ -52,6 +52,7 @@ class UserView(PrincipalMasterView): model_row_class = UserEvent has_versions = True touchable = True + mergeable = True grid_columns = [ 'username', @@ -78,23 +79,13 @@ class UserView(PrincipalMasterView): 'occurred', ] - mergeable = True - merge_additive_fields = [ - 'sent_message_count', - 'received_message_count', - ] - merge_coalesce_fields = [ - 'person_uuid', - 'person_name', - 'active', - ] - merge_fields = merge_additive_fields + [ - 'uuid', - 'username', - 'person_uuid', - 'person_name', - 'role_count', - ] + def __init__(self, request): + super(UserView, self).__init__(request) + app = self.get_rattail_app() + + # always get a reference to the auth/merge handler + self.auth_handler = app.get_auth_handler() + self.merge_handler = self.auth_handler def query(self, session): query = super(UserView, self).query(session) @@ -441,31 +432,6 @@ class UserView(PrincipalMasterView): users.append(user) return users - def get_merge_data(self, user): - return { - 'uuid': user.uuid, - 'username': user.username, - 'person_uuid': user.person_uuid, - 'person_name': user.person.display_name if user.person else None, - '_roles': user.roles, - 'role_count': len(user.roles), - 'active': user.active, - 'sent_message_count': len(user.sent_messages), - 'received_message_count': len(user._messages), - } - - def get_merge_resulting_data(self, remove, keep): - result = super(UserView, self).get_merge_resulting_data(remove, keep) - result['role_count'] = len(set(remove['_roles'] + keep['_roles'])) - return result - - def merge_objects(self, removing, keeping): - # TODO: merge roles, messages - assert not removing.sent_messages - assert not removing._messages - assert not removing._roles - self.Session.delete(removing) - def preferences(self, user=None): """ View to modify preferences for a particular user. From 3e8924e7ccb248df6f35898e6349a216715ffd6f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 15 Nov 2022 13:39:17 -0600 Subject: [PATCH 053/879] Update changelog --- CHANGES.rst | 6 ++++++ tailbone/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a1a03d46..8eca2ac4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,12 @@ CHANGELOG ========= +0.8.258 (2022-11-15) +-------------------- + +* Let the auth handler manage user merge. + + 0.8.257 (2022-11-03) -------------------- diff --git a/tailbone/_version.py b/tailbone/_version.py index 8f293897..3447d6bf 100644 --- a/tailbone/_version.py +++ b/tailbone/_version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8; -*- -__version__ = '0.8.257' +__version__ = '0.8.258' From deed2111fbd3d31cc44c8bd4cf668358e1facc45 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 15 Nov 2022 16:29:15 -0600 Subject: [PATCH 054/879] Add "between" verb for numeric grid filters --- tailbone/grids/filters.py | 57 +++++++++++++++++++++-- tailbone/static/js/tailbone.buefy.grid.js | 49 +++++++++++++++++++ tailbone/templates/grids/buefy.mako | 31 ++++++++++++ 3 files changed, 134 insertions(+), 3 deletions(-) diff --git a/tailbone/grids/filters.py b/tailbone/grids/filters.py index edce41dd..2818b78a 100644 --- a/tailbone/grids/filters.py +++ b/tailbone/grids/filters.py @@ -76,8 +76,7 @@ class NumericValueRenderer(FilterValueRenderer): """ Input renderer for numeric values. """ - # TODO - # data_type = 'number' + data_type = 'number' def render(self, value=None, **kwargs): kwargs.setdefault('step', '0.001') @@ -137,6 +136,7 @@ class GridFilter(object): 'less_equal': "less than or equal to", 'is_empty': "is empty", 'is_not_empty': "is not empty", + 'between': "between", 'is_null': "is null", 'is_not_null': "is not null", 'is_true': "is true", @@ -378,6 +378,47 @@ class AlchemyGridFilter(GridFilter): return query return query.filter(self.column <= self.encode_value(value)) + def filter_between(self, query, value): + """ + Filter data with a "between" query. Really this uses ">=" and + "<=" (inclusive) logic instead of SQL "between" keyword. + """ + if value is None or value == '': + return query + + if '|' not in value: + return query + + values = value.split('|') + if len(values) != 2: + return query + + start_value, end_value = values + + # we'll only filter if we have start and/or end value + if not start_value and not end_value: + return query + + return self.filter_for_range(query, start_value, end_value) + + def filter_for_range(self, query, start_value, end_value): + """ + This method should actually apply filter(s) to the query, + according to the given value range. Subclasses may override + this logic. + """ + if start_value: + if self.value_invalid(start_value): + return query + query = query.filter(self.column >= start_value) + + if end_value: + if self.value_invalid(end_value): + return query + query = query.filter(self.column <= end_value) + + return query + class AlchemyStringFilter(AlchemyGridFilter): """ @@ -532,7 +573,8 @@ class AlchemyNumericFilter(AlchemyGridFilter): # expose greater-than / less-than verbs in addition to core default_verbs = ['equal', 'not_equal', 'greater_than', 'greater_equal', - 'less_than', 'less_equal', 'is_null', 'is_not_null', 'is_any'] + 'less_than', 'less_equal', 'between', + 'is_null', 'is_not_null', 'is_any'] # TODO: what follows "works" in that it prevents an error...but from the # user's perspective it still fails silently...need to improve on front-end @@ -541,6 +583,13 @@ class AlchemyNumericFilter(AlchemyGridFilter): # term for integer field... def value_invalid(self, value): + + # first just make sure it's somewhat numeric + try: + float(value) + except ValueError: + return True + return bool(value and len(six.text_type(value)) > 8) def filter_equal(self, query, value): @@ -726,6 +775,7 @@ class AlchemyDateFilter(AlchemyGridFilter): return query return query.filter(self.column <= self.encode_value(date)) + # TODO: this should be merged into parent class def filter_between(self, query, value): """ Filter data with a "between" query. Really this uses ">=" and "<=" @@ -753,6 +803,7 @@ class AlchemyDateFilter(AlchemyGridFilter): return self.filter_date_range(query, start_date, end_date) + # TODO: this should be merged into parent class def filter_date_range(self, query, start_date, end_date): """ This method should actually apply filter(s) to the query, according to diff --git a/tailbone/static/js/tailbone.buefy.grid.js b/tailbone/static/js/tailbone.buefy.grid.js index a4139bc6..75037448 100644 --- a/tailbone/static/js/tailbone.buefy.grid.js +++ b/tailbone/static/js/tailbone.buefy.grid.js @@ -1,4 +1,53 @@ +const GridFilterNumericValue = { + template: '#grid-filter-numeric-value-template', + props: { + value: String, + wantsRange: Boolean, + }, + data() { + return { + startValue: null, + endValue: null, + } + }, + mounted() { + if (this.wantsRange) { + if (this.value.includes('|')) { + let values = this.value.split('|') + if (values.length == 2) { + this.startValue = values[0] + this.endValue = values[1] + } else { + this.startValue = this.value + } + } else { + this.startValue = this.value + } + } else { + this.startValue = this.value + } + }, + methods: { + focus() { + this.$refs.startValue.focus() + }, + startValueChanged(value) { + if (this.wantsRange) { + value += '|' + this.endValue + } + this.$emit('input', value) + }, + endValueChanged(value) { + value = this.startValue + '|' + value + this.$emit('input', value) + }, + }, +} + +Vue.component('grid-filter-numeric-value', GridFilterNumericValue) + + const GridFilterDateValue = { template: '#grid-filter-date-value-template', props: { diff --git a/tailbone/templates/grids/buefy.mako b/tailbone/templates/grids/buefy.mako index 11b9a86b..ec1a4875 100644 --- a/tailbone/templates/grids/buefy.mako +++ b/tailbone/templates/grids/buefy.mako @@ -1,5 +1,29 @@ ## -*- coding: utf-8; -*- + + + % endif + + <%def name="object_helpers()"> ${self.render_status_breakdown()} ${self.render_po_vs_invoice_helper()} @@ -418,13 +452,128 @@ % endif + % if allow_edit_catalog_unit_cost: + + let ReceivingCostEditor = { + template: '#receiving-cost-editor-template', + props: { + row: Object, + value: String, + }, + data() { + return { + inputValue: this.value, + editing: false, + } + }, + methods: { + + startEdit() { + this.inputValue = this.value + this.editing = true + this.$nextTick(() => { + this.$refs.input.focus() + }) + }, + + inputKeyDown(event) { + + // when user presses Enter while editing cost value, submit + // value to server for immediate persistence + if (event.which == 13) { + this.submitEdit() + + // when user presses Escape, cancel the edit + } else if (event.which == 27) { + this.cancelEdit() + } + }, + + inputBlur(event) { + // always assume user meant to cancel + this.cancelEdit() + }, + + cancelEdit() { + // reset input to discard any user entry + this.inputValue = this.value + this.editing = false + this.$emit('cancel-edit') + }, + + submitEdit() { + let url = '${url('{}.update_row_cost'.format(route_prefix), uuid=batch.uuid)}' + + // TODO: should get csrf token from parent component? + let csrftoken = ${json.dumps(request.session.get_csrf_token() or request.session.new_csrf_token())|n} + let headers = {'${csrf_header_name}': csrftoken} + + let params = { + row_uuid: this.$props.row.uuid, + catalog_unit_cost: this.inputValue, + } + + this.$http.post(url, params, {headers: headers}).then(response => { + if (!response.data.error) { + + // let parent know cost value has changed + // (this in turn will update data in *this* + // component, and display will refresh) + this.$emit('input', response.data.row.catalog_unit_cost, + this.$props.row._index) + + // and hide the input box + this.editing = false + + } else { + this.$buefy.toast.open({ + message: "Submit failed: " + response.data.error, + type: 'is-warning', + duration: 4000, // 4 seconds + }) + } + + }, response => { + this.$buefy.toast.open({ + message: "Submit failed: (unknown error)", + type: 'is-warning', + duration: 4000, // 4 seconds + }) + }) + }, + }, + } + + Vue.component('receiving-cost-editor', ReceivingCostEditor) + + ${rows_grid.component_studly}.methods.catalogUnitCostClicked = function(row) { + + // start edit for clicked cell + this.$refs['catalogUnitCost_' + row.uuid].startEdit() + } + + ${rows_grid.component_studly}.methods.catalogCostConfirmed = function(amount, index) { + + // update display to indicate cost was confirmed + this.addRowClass(index, 'catalog_cost_confirmed') + + // start editing next row, unless there are no more + let nextRow = index + 1 + if (this.data.length > nextRow) { + nextRow = this.data[nextRow] + this.$refs['catalogUnitCost_' + nextRow.uuid].startEdit() + } + } + + % endif + ${parent.body()} -% if master.handler.allow_truck_dump_receiving() and master.has_perm('edit_row'): +% if not use_buefy and master.handler.allow_truck_dump_receiving() and master.has_perm('edit_row'): ${h.form(url('{}.transform_unit_row'.format(route_prefix), uuid=batch.uuid), name='transform-unit-form')} ${h.csrf_token(request)} ${h.hidden('row_uuid')} diff --git a/tailbone/util.py b/tailbone/util.py index cd6c9237..5dee997f 100644 --- a/tailbone/util.py +++ b/tailbone/util.py @@ -64,6 +64,21 @@ def csrf_token(request, name='_csrf'): return HTML.tag("div", tags.hidden(name, value=token), style="display:none;") +def get_form_data(request): + """ + Returns the effective form data for the given request. Mostly + this is a convenience, to return either POST or JSON depending on + the type of request. + """ + # nb. we prefer JSON only if no POST is present + # TODO: this seems to work for our use case at least, but perhaps + # there is a better way? see also + # https://docs.pylonsproject.org/projects/pyramid/en/latest/api/request.html#pyramid.request.Request.is_xhr + if request.is_xhr and not request.POST: + return request.json_body + return request.POST + + def should_use_buefy(request): """ Returns a flag indicating whether or not the current theme supports (and diff --git a/tailbone/views/purchasing/receiving.py b/tailbone/views/purchasing/receiving.py index 78136ef3..09a28099 100644 --- a/tailbone/views/purchasing/receiving.py +++ b/tailbone/views/purchasing/receiving.py @@ -46,6 +46,7 @@ from pyramid import httpexceptions from webhelpers2.html import tags, HTML from tailbone import forms, grids +from tailbone.util import get_form_data from tailbone.views.purchasing import PurchasingBatchView @@ -715,6 +716,11 @@ class ReceivingBatchView(PurchasingBatchView): return breakdown + def allow_edit_catalog_unit_cost(self, batch): + return (not batch.executed + and self.has_perm('edit_row') + and self.batch_handler.allow_receiving_edit_catalog_unit_cost()) + def template_kwargs_view(self, **kwargs): kwargs = super(ReceivingBatchView, self).template_kwargs_view(**kwargs) batch = kwargs['instance'] @@ -739,6 +745,8 @@ class ReceivingBatchView(PurchasingBatchView): data=breakdown, columns=['title', 'count']) + kwargs['allow_edit_catalog_unit_cost'] = self.allow_edit_catalog_unit_cost(batch) + return kwargs def get_context_credits(self, row): @@ -933,6 +941,7 @@ class ReceivingBatchView(PurchasingBatchView): def configure_row_grid(self, g): super(ReceivingBatchView, self).configure_row_grid(g) + use_buefy = self.get_use_buefy() batch = self.get_instance() # vendor_code @@ -943,6 +952,10 @@ class ReceivingBatchView(PurchasingBatchView): if (self.handler.has_purchase_order(batch) or self.handler.has_invoice_file(batch)): g.remove('catalog_unit_cost') + elif use_buefy and self.allow_edit_catalog_unit_cost(batch): + g.set_raw_renderer('catalog_unit_cost', self.render_catalog_unit_cost) + g.set_click_handler('catalog_unit_cost', + 'catalogUnitCostClicked(props.row)') # po_unit_cost if self.handler.has_invoice_file(batch): @@ -1001,6 +1014,14 @@ class ReceivingBatchView(PurchasingBatchView): else: g.set_enum('truck_dump_status', model.PurchaseBatchRow.STATUS) + def render_catalog_unit_cost(self): + return HTML.tag('receiving-cost-editor', **{ + 'v-model': 'props.row.catalog_unit_cost', + ':ref': "'catalogUnitCost_' + props.row.uuid", + ':row': 'props.row', + '@input': 'catalogCostConfirmed', + }) + def row_grid_extra_class(self, row, i): css_class = super(ReceivingBatchView, self).row_grid_extra_class(row, i) @@ -1790,10 +1811,10 @@ class ReceivingBatchView(PurchasingBatchView): def update_row_cost(self): """ - AJAX view for updating the invoice (actual) unit cost for a row. + AJAX view for updating various cost fields in a data row. """ batch = self.get_instance() - data = dict(self.request.POST) + data = dict(get_form_data(self.request)) # validate row uuid = data.get('row_uuid') @@ -1939,6 +1960,9 @@ class ReceivingBatchView(PurchasingBatchView): {'section': 'rattail.batch', 'option': 'purchase.receiving.should_autofix_invoice_case_vs_unit', 'type': bool}, + {'section': 'rattail.batch', + 'option': 'purchase.receiving.allow_edit_catalog_unit_cost', + 'type': bool}, # mobile interface {'section': 'rattail.batch', From 9c54a4ada16289043cc6b0a7c335437bf50afce4 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 5 Dec 2022 15:22:59 -0600 Subject: [PATCH 079/879] Add receiving workflow as param when making receiving batch --- tailbone/views/purchasing/receiving.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tailbone/views/purchasing/receiving.py b/tailbone/views/purchasing/receiving.py index 09a28099..26156516 100644 --- a/tailbone/views/purchasing/receiving.py +++ b/tailbone/views/purchasing/receiving.py @@ -116,6 +116,7 @@ class ReceivingBatchView(PurchasingBatchView): 'batch_type', # TODO: ideally would get rid of this one 'store', 'vendor', + 'description', 'receiving_workflow', 'truck_dump', 'truck_dump_children_first', @@ -126,6 +127,7 @@ class ReceivingBatchView(PurchasingBatchView): 'invoice_parser_key', 'department', 'purchase', + 'params', 'vendor_email', 'vendor_fax', 'vendor_contact', @@ -138,7 +140,6 @@ class ReceivingBatchView(PurchasingBatchView): 'invoice_number', 'invoice_total', 'invoice_total_calculated', - 'description', 'notes', 'created', 'created_by', @@ -647,6 +648,8 @@ class ReceivingBatchView(PurchasingBatchView): if 'vendor_uuid' in self.request.matchdict: kwargs['vendor_uuid'] = self.request.matchdict['vendor_uuid'] + # TODO: ugh should just have workflow and no batch_type + kwargs['receiving_workflow'] = batch_type if batch_type == 'from_scratch': kwargs.pop('truck_dump_batch', None) kwargs.pop('truck_dump_batch_uuid', None) From 36a5f2ab492c46d3ea4e5086690409425248d51a Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 5 Dec 2022 16:05:27 -0600 Subject: [PATCH 080/879] Show invoice cost in receiving batch, if "from scratch" --- tailbone/views/purchasing/receiving.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tailbone/views/purchasing/receiving.py b/tailbone/views/purchasing/receiving.py index 26156516..4937b80f 100644 --- a/tailbone/views/purchasing/receiving.py +++ b/tailbone/views/purchasing/receiving.py @@ -960,13 +960,13 @@ class ReceivingBatchView(PurchasingBatchView): g.set_click_handler('catalog_unit_cost', 'catalogUnitCostClicked(props.row)') - # po_unit_cost - if self.handler.has_invoice_file(batch): - g.remove('po_unit_cost') - - # invoice_unit_cost - if not self.handler.has_invoice_file(batch): + # nb. only show PO *or* invoice cost; prefer the latter unless + # we have a PO and no invoice + if (self.batch_handler.has_purchase_order(batch) + and not self.batch_handler.has_invoice(batch)): g.remove('invoice_unit_cost') + else: + g.remove('po_unit_cost') # credits # note that sorting by credits involves a subquery with group by clause. From cceb66e50024c7d55310db6021441b91fc3492ec Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 5 Dec 2022 16:25:55 -0600 Subject: [PATCH 081/879] Add support for editing invoice cost in receiving batch, per new theme --- tailbone/templates/receiving/configure.mako | 9 +++ tailbone/templates/receiving/view.mako | 67 ++++++++++++++++----- tailbone/views/purchasing/receiving.py | 25 ++++++++ 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/tailbone/templates/receiving/configure.mako b/tailbone/templates/receiving/configure.mako index 9d06d811..9f4a6c3b 100644 --- a/tailbone/templates/receiving/configure.mako +++ b/tailbone/templates/receiving/configure.mako @@ -133,6 +133,15 @@ + + + Allow edit of Invoice Unit Cost + + +

    Mobile Interface

    diff --git a/tailbone/templates/receiving/view.mako b/tailbone/templates/receiving/view.mako index d7a2a287..b16aa5b8 100644 --- a/tailbone/templates/receiving/view.mako +++ b/tailbone/templates/receiving/view.mako @@ -264,19 +264,26 @@ <%def name="extra_styles()"> ${parent.extra_styles()} - % if use_buefy and allow_edit_catalog_unit_cost: + % if use_buefy: % elif not use_buefy and not batch.executed and master.has_perm('edit_row'): - - <%def name="page_content()">
    -
    ${rendered_result or ""|n}
    +
    ${rendered_result or ""|n}
    diff --git a/tailbone/templates/page_help.mako b/tailbone/templates/page_help.mako new file mode 100644 index 00000000..cd13011e --- /dev/null +++ b/tailbone/templates/page_help.mako @@ -0,0 +1,204 @@ +## -*- coding: utf-8; -*- + +<%def name="render_template()"> + + + +<%def name="declare_vars()"> + + + +<%def name="make_component()"> + + diff --git a/tailbone/templates/themes/falafel/base.mako b/tailbone/templates/themes/falafel/base.mako index fe3ef429..e46be1a5 100644 --- a/tailbone/templates/themes/falafel/base.mako +++ b/tailbone/templates/themes/falafel/base.mako @@ -3,6 +3,7 @@ <%namespace file="/autocomplete.mako" import="tailbone_autocomplete_template" /> <%namespace name="base_meta" file="/base_meta.mako" /> <%namespace file="/formposter.mako" import="declare_formposter_mixin" /> +<%namespace name="page_help" file="/page_help.mako" /> @@ -383,17 +384,9 @@ % endif - ## Help Button - % if help_url is not Undefined and help_url: -
    - - Help - -
    - % endif +
    + +
    ## Feedback Button / Dialog % if request.has_perm('common.feedback'): @@ -466,6 +459,8 @@ + ${page_help.render_template()} + + % endif + + <%def name="finalize_this_page_vars()"> ${parent.finalize_this_page_vars()} % if form is not Undefined: diff --git a/tailbone/templates/forms/deform_buefy.mako b/tailbone/templates/forms/deform_buefy.mako index c387d965..0b1e8d90 100644 --- a/tailbone/templates/forms/deform_buefy.mako +++ b/tailbone/templates/forms/deform_buefy.mako @@ -66,6 +66,47 @@ % if not form.readonly: ${h.end_form()} % endif + + % if can_edit_help: + +
    + +
    +

    Field: {{ configureFieldName }}

    +
    + +
    + + + + + + + + + + +
    + +
    + + Cancel + + + {{ configureFieldSaving ? "Working, please wait..." : "Save" }} + +
    +
    +     + % endif + @@ -85,7 +126,29 @@ submit${form.component_studly}() { this.${form.component_studly}Submitting = true this.${form.component_studly}ButtonText = "Working, please wait..." - } + }, + % endif + + % if can_edit_help: + configureFieldSave() { + this.configureFieldSaving = true + let url = '${edit_help_url}' + let params = { + field_name: this.configureFieldName, + markdown_text: this.configureFieldMarkdown, + } + this.submitForm(url, params, response => { + this.configureFieldShowDialog = false + this.$buefy.toast.open({ + message: "Info was saved; please refresh page to see changes.", + type: 'is-info', + duration: 4000, // 4 seconds + }) + this.configureFieldSaving = false + }, response => { + this.configureFieldSaving = false + }) + }, % endif } } @@ -95,6 +158,16 @@ ## TODO: should find a better way to handle CSRF token csrftoken: ${json.dumps(request.session.get_csrf_token() or request.session.new_csrf_token())|n}, + % if can_edit_help: + fieldLabels: ${json.dumps(field_labels)|n}, + fieldMarkdowns: ${json.dumps(field_markdowns)|n}, + configureFieldShowDialog: false, + configureFieldSaving: false, + configureFieldName: null, + configureFieldLabel: null, + configureFieldMarkdown: null, + % endif + ## TODO: ugh, this seems pretty hacky. need to declare some data models ## for various field components to bind to... % if not form.readonly: diff --git a/tailbone/templates/page_help.mako b/tailbone/templates/page_help.mako index cd13011e..b745965a 100644 --- a/tailbone/templates/page_help.mako +++ b/tailbone/templates/page_help.mako @@ -108,7 +108,7 @@     - + diff --git a/tailbone/util.py b/tailbone/util.py index f5457149..ca8d0933 100644 --- a/tailbone/util.py +++ b/tailbone/util.py @@ -183,12 +183,14 @@ def raw_datetime(config, value, verbose=False, as_date=False): return HTML.tag('span', **kwargs) -def render_markdown(text, **kwargs): +def render_markdown(text, raw=False, **kwargs): """ Render the given markdown text as HTML. """ kwargs.setdefault('extensions', ['fenced_code', 'codehilite']) md = markdown.markdown(text, **kwargs) + if raw: + return md md = HTML.literal(md) return HTML.tag('div', class_='rendered-markdown', c=[md]) diff --git a/tailbone/views/master.py b/tailbone/views/master.py index 396c953e..2431b437 100644 --- a/tailbone/views/master.py +++ b/tailbone/views/master.py @@ -2333,6 +2333,40 @@ class MasterView(View): info.markdown_text = form.validated['markdown_text'] return {'ok': True} + def edit_field_help(self): + if (not self.has_perm('edit_help') + and not self.request.has_perm('common.edit_help')): + raise self.forbidden() + + model = self.model + route_prefix = self.get_route_prefix() + schema = colander.Schema() + + schema.add(colander.SchemaNode(colander.String(), + name='field_name')) + + schema.add(colander.SchemaNode(colander.String(), + name='markdown_text', + missing=None)) + + factory = self.get_form_factory() + form = factory(schema=schema, request=self.request) + if not form.validate(newstyle=True): + return {'error': "Form did not validate"} + + # nb. self.Session may differ, so use tailbone.db.Session + info = Session.query(model.TailboneFieldInfo)\ + .filter(model.TailboneFieldInfo.route_prefix == route_prefix)\ + .filter(model.TailboneFieldInfo.field_name == form.validated['field_name'])\ + .first() + if not info: + info = model.TailboneFieldInfo(route_prefix=route_prefix, + field_name=form.validated['field_name']) + Session.add(info) + + info.markdown_text = form.validated['markdown_text'] + return {'ok': True} + def render_to_response(self, template, data, **kwargs): """ Return a response with the given template rendered with the given data. @@ -3944,6 +3978,7 @@ class MasterView(View): Return a dictionary of kwargs to be passed to the factory when creating new form instances. """ + route_prefix = self.get_route_prefix() defaults = { 'request': self.request, 'readonly': self.viewing, @@ -3951,12 +3986,21 @@ class MasterView(View): 'action_url': self.request.current_route_url(_query=None), 'use_buefy': self.get_use_buefy(), 'assume_local_times': self.has_local_times, + 'route_prefix': route_prefix, + 'can_edit_help': (self.has_perm('edit_help') + or self.request.has_perm('common.edit_help')), } + + if defaults['can_edit_help']: + defaults['edit_help_url'] = self.request.route_url( + '{}.edit_field_help'.format(route_prefix)) + if self.creating: kwargs.setdefault('cancel_url', self.get_index_url()) else: instance = kwargs['model_instance'] kwargs.setdefault('cancel_url', self.get_action_url('view', instance)) + defaults.update(kwargs) return defaults @@ -4832,6 +4876,12 @@ class MasterView(View): config.add_view(cls, attr='edit_help', route_name='{}.edit_help'.format(route_prefix), renderer='json') + config.add_route('{}.edit_field_help'.format(route_prefix), + '{}/edit-field-help'.format(url_prefix), + request_method='POST') + config.add_view(cls, attr='edit_field_help', + route_name='{}.edit_field_help'.format(route_prefix), + renderer='json') # list/search if cls.listable: From b04c1054fcbd6e8acb4f626f235a92e02a8d00f9 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 25 Dec 2022 12:25:55 -0600 Subject: [PATCH 116/879] Override document title when upgrading when using websockets, to mimic old behavior without them --- tailbone/templates/upgrades/view.mako | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tailbone/templates/upgrades/view.mako b/tailbone/templates/upgrades/view.mako index c6ae11f2..a5b6445e 100644 --- a/tailbone/templates/upgrades/view.mako +++ b/tailbone/templates/upgrades/view.mako @@ -66,7 +66,7 @@

    - Upgrading (please wait) ... + Upgrading ${app_title} (please wait) ... {{ executeUpgradeComplete ? "DONE!" : "" }}

    { this.adjustTextoutHeight() }) From cd466a64e53406d98aca0f6e9af8724a398d27f3 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 25 Dec 2022 12:45:23 -0600 Subject: [PATCH 117/879] Filter by person instead of user, for Generated Reports "Created by" --- tailbone/views/exports.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tailbone/views/exports.py b/tailbone/views/exports.py index 3f6d417c..82591099 100644 --- a/tailbone/views/exports.py +++ b/tailbone/views/exports.py @@ -31,12 +31,9 @@ import shutil import six -from rattail.db import model - from pyramid.response import FileResponse -from webhelpers2.html import HTML, tags +from webhelpers2.html import tags -from tailbone import forms from tailbone.views import MasterView @@ -49,6 +46,11 @@ class ExportMasterView(MasterView): downloadable = False delete_export_files = False + labels = { + 'id': "ID", + 'created_by': "Created by", + } + grid_columns = [ 'id', 'created', @@ -82,19 +84,23 @@ class ExportMasterView(MasterView): def configure_grid(self, g): super(ExportMasterView, self).configure_grid(g) + model = self.model - g.joiners['created_by'] = lambda q: q.join(model.User) - g.sorters['created_by'] = g.make_sorter(model.User.username) - g.filters['created_by'] = g.make_filter('created_by', model.User.username) + # id + g.set_renderer('id', self.render_id) + g.set_link('id') + + # filename + g.set_link('filename') + + # created g.set_sort_defaults('created', 'desc') - g.set_renderer('id', self.render_id) - - g.set_label('id', "ID") - g.set_label('created_by', "Created by") - - g.set_link('id') - g.set_link('filename') + # created_by + g.set_joiner('created_by', + lambda q: q.join(model.User).outerjoin(model.Person)) + g.set_sorter('created_by', model.Person.display_name) + g.set_filter('created_by', model.Person.display_name) def render_id(self, export, field): return export.id_str From 8264a69ceca86ee95a049687f7dab0d0542f8a36 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 25 Dec 2022 14:41:58 -0600 Subject: [PATCH 118/879] Add "direct link" support for master grids --- tailbone/grids/core.py | 4 +- tailbone/templates/grids/buefy.mako | 133 ++++++++++++++------ tailbone/templates/grids/filters_buefy.mako | 2 +- tailbone/views/master.py | 4 + 4 files changed, 105 insertions(+), 38 deletions(-) diff --git a/tailbone/grids/core.py b/tailbone/grids/core.py index 54f578ed..78fd2cc6 100644 --- a/tailbone/grids/core.py +++ b/tailbone/grids/core.py @@ -189,6 +189,7 @@ class Grid(object): clicking_row_checks_box=False, click_handlers=None, main_actions=[], more_actions=[], delete_speedbump=False, ajax_data_url=None, component='tailbone-grid', + expose_direct_link=False, **kwargs): self.key = key @@ -256,11 +257,12 @@ class Grid(object): if ajax_data_url: self.ajax_data_url = ajax_data_url elif self.request: - self.ajax_data_url = self.request.current_route_url() + self.ajax_data_url = self.request.current_route_url(_query=None) else: self.ajax_data_url = '' self.component = component + self.expose_direct_link = expose_direct_link self._whgrid_kwargs = kwargs @property diff --git a/tailbone/templates/grids/buefy.mako b/tailbone/templates/grids/buefy.mako index 12231606..c99d0f70 100644 --- a/tailbone/templates/grids/buefy.mako +++ b/tailbone/templates/grids/buefy.mako @@ -289,26 +289,41 @@ - % if grid.pageable: -