Add "worksheet file" pattern for editing batches

lets user download a worksheet, edit, then upload back to update the batch

tailbone/forms/core.py

@@ -623,6 +623,9 @@ class Form(object):
             if 'required' in kwargs and not kwargs['required']:
                 kw['missing'] = colander.null
             self.set_node(key, colander.SchemaNode(deform.FileData(), **kw))
+            # must explicitly replace node, if we already have a schema
+            if self.schema:
+                self.schema[key] = self.nodes[key]
         else:
             raise ValueError("unknown type for '{}' field: {}".format(key, type_))
 

tailbone/templates/batch/view.mako

@@ -10,7 +10,7 @@
     var has_execution_options = ${'true' if master.has_execution_options(batch) else 'false'};
 
     $(function() {
-        % if master.has_worksheet:
+        % if master.has_worksheet and master.allow_worksheet(batch) and master.has_perm('worksheet'):
             $('.load-worksheet').click(function() {
                 disable_button(this);
                 location.href = '${url('{}.worksheet'.format(route_prefix), uuid=batch.uuid)}';
@@ -24,6 +24,36 @@
                 location.href = '${url('{}.refresh'.format(route_prefix), uuid=batch.uuid)}';
             });
         % endif
+        % if master.has_worksheet_file and master.allow_worksheet(batch) and master.has_perm('worksheet'):
+            $('.upload-worksheet').click(function() {
+                $('#upload-worksheet-dialog').dialog({
+                    title: "Upload Worksheet",
+                    width: 600,
+                    modal: true,
+                    buttons: [
+                        {
+                            text: "Upload & Update Batch",
+                            click: function(event) {
+                                var form = $('form[name="upload-worksheet"]');
+                                var field = form.find('input[type="file"]').get(0);
+                                if (!field.value) {
+                                    alert("Please choose a file to upload.");
+                                    return
+                                }
+                                disable_button(dialog_button(event));
+                                form.submit();
+                            }
+                        },
+                        {
+                            text: "Cancel",
+                            click: function() {
+                                $(this).dialog('close');
+                            }
+                        }
+                    ]
+                });
+            });
+        % endif
     });
 
   </script>
@@ -59,6 +89,7 @@
     <div class="buttons">
       ${self.leading_buttons()}
       ${refresh_button()}
+      ${self.trailing_buttons()}
     </div>
 </%def>
 
@@ -81,7 +112,8 @@
           ## TODO: this should surely use a POST request?
           <once-button tag="a"
                        href="${url('{}.refresh'.format(route_prefix), uuid=batch.uuid)}"
-                       text="Refresh Data">
+                       text="Refresh Data"
+                       icon-left="fas fa-redo">
           </once-button>
       % else:
           <button type="button" class="button" id="refresh-data">Refresh Data</button>
@@ -89,6 +121,28 @@
   % endif
 </%def>
 
+<%def name="trailing_buttons()">
+  % if master.has_worksheet_file and master.allow_worksheet(batch) and master.has_perm('worksheet'):
+      % if use_buefy:
+          <b-button tag="a"
+                    href="${master.get_action_url('download_worksheet', batch)}"
+                    icon-pack="fas"
+                    icon-left="fas fa-download">
+            Download Worksheet
+          </b-button>
+          <b-button type="is-primary"
+                    icon-pack="fas"
+                    icon-left="fas fa-upload"
+                    @click="$emit('show-upload')">
+            Upload Worksheet
+          </b-button>
+      % else:
+          ${h.link_to("Download Worksheet", master.get_action_url('download_worksheet', batch), class_='button')}
+          <button type="button" class="upload-worksheet">Upload Worksheet</button>
+      % endif
+  % endif
+</%def>
+
 <%def name="object_helpers()">
   ${self.render_status_breakdown()}
   ${self.render_execute_helper()}
@@ -206,6 +260,54 @@
 
 <%def name="render_this_page()">
   ${parent.render_this_page()}
+
+  % if master.has_worksheet_file and master.allow_worksheet(batch) and master.has_perm('worksheet'):
+      % if use_buefy:
+          <b-modal has-modal-card
+                   :active.sync="showUploadDialog">
+            <div class="modal-card">
+
+              <header class="modal-card-head">
+                <p class="modal-card-title">Upload Worksheet</p>
+              </header>
+
+              <section class="modal-card-body">
+                <p>
+                  This will <span class="has-text-weight-bold">update</span>
+                  the batch data with the worksheet file you provide.&nbsp;
+                  Please be certain to use the right one!
+                </p>
+                <br />
+                <${upload_worksheet_form.component} ref="uploadForm">
+                </${upload_worksheet_form.component}>
+              </section>
+
+              <footer class="modal-card-foot">
+                <b-button @click="showUploadDialog = false">
+                  Cancel
+                </b-button>
+                <b-button type="is-primary"
+                          @click="submitUpload()"
+                          icon-pack="fas"
+                          icon-left="fas fa-upload"
+                          :disabled="uploadButtonDisabled">
+                  {{ uploadButtonText }}
+                </b-button>
+              </footer>
+
+            </div>
+          </b-modal>
+      % else:
+          <div id="upload-worksheet-dialog" style="display: none;">
+            <p>
+              This will <strong>update</strong> the batch data with the worksheet
+              file you provide.&nbsp; Please be certain to use the right one!
+            </p>
+            ${upload_worksheet_form.render_deform(buttons=False, form_kwargs={'name': 'upload-worksheet'})|n}
+          </div>
+      % endif
+  % endif
+
   % if not use_buefy:
       % if master.handler.executable(batch) and master.has_perm('execute'):
           <div id="execution-options-dialog" style="display: none;">
@@ -213,22 +315,59 @@
           </div>
       % endif
   % endif
+
 </%def>
 
 <%def name="render_this_page_template()">
   ${parent.render_this_page_template()}
-  % if use_buefy and master.handler.executable(batch) and master.has_perm('execute'):
+  % if use_buefy:
-      ## TODO: stop using |n filter
+      % if master.has_worksheet_file and master.allow_worksheet(batch) and master.has_perm('worksheet'):
-      ${execute_form.render_deform(form_kwargs={'ref': 'actualExecuteForm'}, buttons=False)|n}
+          ${upload_worksheet_form.render_deform(buttons=False, form_kwargs={'ref': 'actualUploadForm'})|n}
+      % endif
+      % if master.handler.executable(batch) and master.has_perm('execute'):
+          ${execute_form.render_deform(form_kwargs={'ref': 'actualExecuteForm'}, buttons=False)|n}
+      % endif
   % endif
 </%def>
 
+<%def name="render_buefy_form()">
+  <div class="form">
+    <${form.component} @show-upload="showUploadDialog = true">
+    </${form.component}>
+  </div>
+</%def>
+
 <%def name="modify_this_page_vars()">
   ${parent.modify_this_page_vars()}
   <script type="text/javascript">
 
     ThisPageData.statusBreakdownData = ${json.dumps(status_breakdown_grid.get_buefy_data()['data'])|n}
 
+    % if master.has_worksheet_file and master.allow_worksheet(batch) and master.has_perm('worksheet'):
+
+        ThisPageData.showUploadDialog = false
+        ThisPageData.uploadButtonText = "Upload & Update Batch"
+        ThisPageData.uploadButtonDisabled = false
+
+        ThisPage.methods.submitUpload = function() {
+            let form = this.$refs.uploadForm
+            let value = form.field_model_worksheet_file
+            if (!value) {
+                alert("Please choose a file to upload.")
+                return
+            }
+            this.uploadButtonDisabled = true
+            this.uploadButtonText = "Working, please wait..."
+            form.submit()
+        }
+
+        ${upload_worksheet_form.component_studly}.methods.submit = function() {
+            this.$refs.actualUploadForm.submit()
+        }
+
+    ## end 'external_worksheet'
+    % endif
+
     % if not batch.executed and master.has_perm('execute'):
 
         ThisPageData.showExecutionDialog = false
@@ -247,6 +386,16 @@
 
 <%def name="make_this_page_component()">
   ${parent.make_this_page_component()}
+  % if master.has_worksheet_file and master.allow_worksheet(batch) and master.has_perm('worksheet'):
+      <script type="text/javascript">
+
+        ## UploadForm
+        ${upload_worksheet_form.component_studly}.data = function() { return ${upload_worksheet_form.component_studly}Data }
+        Vue.component('${upload_worksheet_form.component}', ${upload_worksheet_form.component_studly})
+
+      </script>
+  % endif
+
   % if not batch.executed and master.has_perm('execute'):
       <script type="text/javascript">
 

tailbone/templates/form.mako

@@ -11,7 +11,7 @@
 
 <%def name="render_buefy_form()">
   <div class="form">
-    <tailbone-form></tailbone-form>
+    <${form.component}></${form.component}>
   </div>
 </%def>
 

tailbone/views/batch/core.py

@@ -87,6 +87,7 @@ class BatchMasterView(MasterView):
     mobile_filterable = True
     mobile_rows_viewable = True
     has_worksheet = False
+    has_worksheet_file = False
 
     grid_columns = [
         'id',
@@ -168,6 +169,10 @@ class BatchMasterView(MasterView):
         batch = kwargs['instance']
         kwargs['batch'] = batch
         kwargs['handler'] = self.handler
+
+        if self.has_worksheet_file and self.allow_worksheet(batch) and self.has_perm('worksheet'):
+            kwargs['upload_worksheet_form'] = self.make_upload_worksheet_form(batch)
+
         kwargs['execute_title'] = self.get_execute_title(batch)
         kwargs['execute_enabled'] = self.instance_executable(batch)
         if kwargs['mobile']:
@@ -181,6 +186,7 @@ class BatchMasterView(MasterView):
             kwargs['execute_form'] = self.make_execute_form(batch, action_url=url)
         else:
             kwargs['why_not_execute'] = self.handler.why_not_execute(batch)
+
         kwargs['status_breakdown'] = self.make_status_breakdown(batch)
         if use_buefy:
             data = [{'title': title, 'count': count}
@@ -190,6 +196,71 @@ class BatchMasterView(MasterView):
                                                    data, ['title', 'count'])
         return kwargs
 
+    def make_upload_worksheet_form(self, batch):
+        action_url = self.get_action_url('upload_worksheet', batch)
+        use_buefy = self.get_use_buefy()
+        form = forms.Form(schema=UploadWorksheet(),
+                          request=self.request,
+                          action_url=action_url,
+                          use_buefy=use_buefy,
+                          component='upload-worksheet-form')
+        form.set_type('worksheet_file', 'file')
+        # TODO: must set these to avoid some default Buefy code
+        form.auto_disable = False
+        form.auto_disable_save = False
+        return form
+
+    def download_worksheet(self):
+        batch = self.get_instance()
+        path = self.handler.write_worksheet(batch)
+        root, ext = os.path.splitext(path)
+        # we present a more descriptive filename for download
+        filename = '{}.worksheet.{}{}'.format(batch.batch_key, batch.id_str, ext)
+        return self.file_response(path, filename=filename)
+
+    def upload_worksheet(self):
+        batch = self.get_instance()
+        form = self.make_upload_worksheet_form(batch)
+        if self.validate_form(form):
+            uploads = self.normalize_uploads(form)
+            path = uploads['worksheet_file']['temp_path']
+            return self.handler_action(batch, 'update_from_worksheet', path=path)
+        self.request.session.flash("Upload form did not validate!", 'error')
+        return self.redirect(self.get_action_url('view', batch))
+
+    def update_from_worksheet_thread(self, batch_uuid, user_uuid, progress, path=None):
+        """
+        Thread target for updating a batch from worksheet.
+        """
+        session = self.make_isolated_session()
+        batch = session.query(self.model_class).get(batch_uuid)
+        try:
+            self.handler.update_from_worksheet(batch, path, progress=progress)
+
+        except Exception as error:
+            session.rollback()
+            log.exception("upload/update failed for '{}' batch: {}".format(self.batch_key, batch))
+            session.close()
+            if progress:
+                progress.session.load()
+                progress.session['error'] = True
+                progress.session['error_msg'] = "Upload processing failed: {}".format(
+                    simple_error(error))
+                progress.session.save()
+
+        else:
+            session.commit()
+            success_msg = "Batch has been updated: {}".format(batch)
+            success_url = self.get_action_url('view', batch)
+            session.close()
+
+            if progress:
+                progress.session.load()
+                progress.session['complete'] = True
+                progress.session['success_msg'] = success_msg
+                progress.session['success_url'] = success_url
+                progress.session.save()
+
     def make_status_breakdown(self, batch, rows=None, status_enum=None):
         """
         Returns a simple list of 2-tuples, each of which has the status display
@@ -1375,11 +1446,11 @@ class BatchMasterView(MasterView):
 
         # If no error, check result flag (false means user canceled).
         else:
+            success_msg = None
             if result:
                 session.commit()
-                # TODO: this doesn't always work...?
+                success_msg = "{} has been executed: {}".format(
-                self.request.session.flash("{} has been executed: {}".format(
+                    self.get_model_title(), batch.id_str)
-                    self.get_model_title(), batch.id_str))
             else:
                 session.rollback()
 
@@ -1391,6 +1462,8 @@ class BatchMasterView(MasterView):
                 progress.session.load()
                 progress.session['complete'] = True
                 progress.session['success_url'] = success_url
+                if success_msg:
+                    progress.session['success_msg'] = success_msg
                 progress.session.save()
 
     def get_execute_success_url(self, batch, result, **kwargs):
@@ -1499,6 +1572,7 @@ class BatchMasterView(MasterView):
         model_key = cls.get_model_key()
         route_prefix = cls.get_route_prefix()
         url_prefix = cls.get_url_prefix()
+        instance_url_prefix = cls.get_instance_url_prefix()
         permission_prefix = cls.get_permission_prefix()
         model_title = cls.get_model_title()
         model_title_plural = cls.get_model_title_plural()
@@ -1514,9 +1588,10 @@ class BatchMasterView(MasterView):
                         permission='{}.create'.format(permission_prefix))
 
         # worksheet
-        if cls.has_worksheet:
+        if cls.has_worksheet or cls.has_worksheet_file:
             config.add_tailbone_permission(permission_prefix, '{}.worksheet'.format(permission_prefix),
                                            "Edit {} data as worksheet".format(model_title))
+        if cls.has_worksheet:
             config.add_route('{}.worksheet'.format(route_prefix), '{}/{{{}}}/worksheet'.format(url_prefix, model_key))
             config.add_view(cls, attr='worksheet', route_name='{}.worksheet'.format(route_prefix),
                             permission='{}.worksheet'.format(permission_prefix))
@@ -1525,6 +1600,20 @@ class BatchMasterView(MasterView):
             config.add_view(cls, attr='worksheet_update', route_name='{}.worksheet_update'.format(route_prefix),
                             renderer='json', permission='{}.worksheet'.format(permission_prefix))
 
+        # worksheet file
+        if cls.has_worksheet_file:
+
+            # download worksheet
+            config.add_route('{}.download_worksheet'.format(route_prefix), '{}/download-worksheet'.format(instance_url_prefix))
+            config.add_view(cls, attr='download_worksheet', route_name='{}.download_worksheet'.format(route_prefix),
+                            permission='{}.worksheet'.format(permission_prefix))
+
+            # upload worksheet
+            config.add_route('{}.upload_worksheet'.format(route_prefix), '{}/upload-worksheet'.format(instance_url_prefix),
+                             request_method='POST')
+            config.add_view(cls, attr='upload_worksheet', route_name='{}.upload_worksheet'.format(route_prefix),
+                            permission='{}.worksheet'.format(permission_prefix))
+
         # refresh batch data
         if cls.refreshable:
             config.add_route('{}.refresh'.format(route_prefix), '{}/{{uuid}}/refresh'.format(url_prefix))
@@ -1611,6 +1700,12 @@ class FileBatchMasterView(BatchMasterView):
             f.set_renderer('filename', self.render_downloadable_file)
 
 
+class UploadWorksheet(colander.Schema):
+
+    # this node is actually "replaced" when form is configured
+    worksheet_file = colander.SchemaNode(colander.String())
+
+
 class ToggleComplete(colander.MappingSchema):
 
     complete = colander.SchemaNode(colander.Boolean())

tailbone/views/core.py

@@ -144,7 +144,7 @@ class View(object):
         return render_to_response('json', data,
                                   request=self.request)
 
-    def file_response(self, path):
+    def file_response(self, path, filename=None):
         """
         Returns a generic FileResponse from the given path
         """
@@ -152,9 +152,10 @@ class View(object):
             return self.notfound()
         response = FileResponse(path, request=self.request)
         response.content_length = os.path.getsize(path)
-        filename = os.path.basename(path)
+        if not filename:
-        if six.PY2:
+            filename = os.path.basename(path)
-            filename = filename.encode('ascii', 'replace')
+            if six.PY2:
+                filename = filename.encode('ascii', 'replace')
         response.content_disposition = str('attachment; filename="{}"'.format(filename))
         return response