Add feature to "download rows for results" in master index view

tailbone/templates/master/index.mako

@@ -18,6 +18,15 @@
   <script type="text/javascript">
     $(function() {
 
+        % if download_results_rows_path:
+            function downloadResultsRowsRedirect() {
+                location.href = '${url('{}.download_results_rows'.format(route_prefix))}?filename=${h.os.path.basename(download_results_rows_path)}';
+            }
+            // we give this 1 second before attempting the redirect; so this
+            // way the page should fully render before redirecting
+            window.setTimeout(downloadResultsRowsRedirect, 1000);
+        % endif
+
         $('.grid-wrapper').gridwrapper();
 
         % if master.deletable and request.has_perm('{}.delete'.format(permission_prefix)) and master.delete_confirm == 'simple':
@@ -55,6 +64,20 @@
 
         % endif
 
+        % if master.has_rows and master.results_rows_downloadable:
+
+            $('#download-row-results-button').click(function() {
+                if (confirm("This will generate an Excel file which contains "
+                            + "not the results themselves, but the *rows* for "
+                            + "each.\n\nAre you sure you want this?")) {
+                    disable_button(this);
+                    var form = $(this).parents('form');
+                    form.submit();
+                }
+            });
+
+        % endif
+
         % if master.bulk_deletable and request.has_perm('{}.bulk_delete'.format(permission_prefix)):
 
         $('form[name="bulk-delete"] button').click(function() {
@@ -292,6 +315,29 @@
       % endif
   % endif
 
+  ## download rows for search results
+  % if master.has_rows and master.results_rows_downloadable:
+      % if use_buefy:
+          <b-button type="is-primary"
+                    icon-pack="fas"
+                    icon-left="fas fa-download"
+                    @click="downloadResultsRows()"
+                    :disabled="downloadResultsRowsButtonDisabled">
+            {{ downloadResultsRowsButtonText }}
+          </b-button>
+          ${h.form(url('{}.download_results_rows'.format(route_prefix)), ref='downloadResultsRowsForm')}
+          ${h.csrf_token(request)}
+          ${h.end_form()}
+      % else:
+          ${h.form(url('{}.download_results_rows'.format(route_prefix)))}
+          ${h.csrf_token(request)}
+          <button type="button" id="download-row-results-button">
+            Download Rows for Results
+          </button>
+          ${h.end_form()}
+      % endif
+  % endif
+
   ## merge 2 objects
   % if master.mergeable and request.has_perm('{}.merge'.format(permission_prefix)):
 
@@ -413,6 +459,13 @@
       </b-notification>
   % endif
 
+  % if download_results_rows_path:
+      <b-notification type="is-info">
+        Your download should start automatically, or you can
+        ${h.link_to("click here", '{}?filename={}'.format(url('{}.download_results_rows'.format(route_prefix)), h.os.path.basename(download_results_rows_path)))}
+      </b-notification>
+  % endif
+
   <${grid.component} :csrftoken="csrftoken"
      % if master.deletable and request.has_perm('{}.delete'.format(permission_prefix)) and master.delete_confirm == 'simple':
      @deleteActionClicked="deleteObject"
@@ -465,6 +518,19 @@
         }
     % endif
 
+    ## maybe auto-redirect to download latest "rows for results" file
+    % if download_results_rows_path and use_buefy:
+        ThisPage.methods.downloadResultsRowsRedirect = function() {
+            location.href = '${url('{}.download_results_rows'.format(route_prefix))}?filename=${h.os.path.basename(download_results_rows_path)}';
+        }
+        ThisPage.mounted = function() {
+            // we give this 1 second before attempting the redirect; otherwise
+            // the FontAwesome icons do not seem to load properly.  so this way
+            // the page should fully render before redirecting
+            window.setTimeout(this.downloadResultsRowsRedirect, 1000)
+        }
+    % endif
+
     ## TODO: stop checking for buefy here once we only have the one session.pop()
     % if use_buefy and request.session.pop('{}.results_csv.generated'.format(route_prefix), False):
         ThisPage.mounted = function() {
@@ -565,6 +631,23 @@
         }
     % endif
 
+    ## download rows for results
+    % if master.has_rows and master.results_rows_downloadable and master.has_perm('download_results_rows'):
+
+        ${grid.component_studly}Data.downloadResultsRowsButtonDisabled = false
+        ${grid.component_studly}Data.downloadResultsRowsButtonText = "Download Rows for Results"
+
+        ${grid.component_studly}.methods.downloadResultsRows = function() {
+            if (confirm("This will generate an Excel file which contains "
+                        + "not the results themselves, but the *rows* for "
+                        + "each.\n\nAre you sure you want this?")) {
+                this.downloadResultsRowsButtonDisabled = true
+                this.downloadResultsRowsButtonText = "Working, please wait..."
+                this.$refs.downloadResultsRowsForm.submit()
+            }
+        }
+    % endif
+
     ## enable / disable selected objects
     % if master.supports_set_enabled_toggle and master.has_perm('enable_disable_set'):
 
@@ -705,6 +788,17 @@
 
 % else:
     ## no buefy, so do the traditional thing
+
+    % if download_results_rows_path:
+        <div class="flash-messages">
+          <div class="ui-state-highlight ui-corner-all">
+            <span style="float: left; margin-right: .3em;" class="ui-icon ui-icon-info"></span>
+            Your download should start automatically, or you can
+            ${h.link_to("click here", '{}?filename={}'.format(url('{}.download_results_rows'.format(route_prefix)), h.os.path.basename(download_results_rows_path)))}
+          </div>
+        </div>
+    % endif
+
     ${grid.render_complete(tools=capture(self.grid_tools).strip(), context_menu=capture(self.context_menu_items).strip())|n}
 
     % if master.deletable and request.has_perm('{}.delete'.format(permission_prefix)) and master.delete_confirm == 'simple':

tailbone/views/master.py

@@ -84,6 +84,7 @@ class MasterView(View):
     results_downloadable = False
     results_downloadable_csv = False
     results_downloadable_xlsx = False
+    results_rows_downloadable = False
     creatable = True
     show_create_link = True
     viewable = True
@@ -356,6 +357,14 @@ class MasterView(View):
             context['download_results_fields_available'] = available
             context['download_results_fields_default'] = self.download_results_fields_default(available)
 
+        if self.has_rows and self.results_rows_downloadable and self.has_perm('download_results_rows'):
+            route_prefix = self.get_route_prefix()
+            context['download_results_rows_path'] = self.request.session.pop(
+                '{}.results_rows.generated'.format(route_prefix), None)
+            available = self.download_results_fields_available()
+            context['download_results_rows_fields_available'] = available
+            context['download_results_rows_fields_default'] = self.download_results_rows_fields_default(available)
+
         return self.render_to_response('index', context)
 
     def make_grid(self, factory=None, key=None, data=None, columns=None, **kwargs):
@@ -2877,8 +2886,8 @@ class MasterView(View):
 
             def write(obj, i):
                 data = self.download_results_normalize(obj, fields, fmt=fmt)
-                row = self.download_results_coerce_csv(data, fields)
+                csvrow = self.download_results_coerce_csv(data, fields)
-                writer.writerow(row)
+                writer.writerow(csvrow)
 
             self.progress_loop(write, results, progress,
                                message="Writing data to CSV file")
@@ -3201,6 +3210,297 @@ class MasterView(View):
             row[field] = getattr(obj, field, None)
         return row
 
+    def download_results_rows_supported_formats(self):
+        # TODO: default formats should be configurable?
+        return OrderedDict([
+            ('xlsx', "Excel (XLSX)"),
+            ('csv', "CSV"),
+        ])
+
+    def download_results_rows_default_format(self):
+        # TODO: default format should be configurable
+        return 'xlsx'
+
+    def download_results_rows(self):
+        """
+        View for saving *rows* of current (filtered) data results into a file,
+        and downloading that file.
+        """
+        route_prefix = self.get_route_prefix()
+        user_uuid = self.request.user.uuid
+
+        # POST means generate a new results file for download
+        if self.request.method == 'POST':
+
+            # make sure a valid format was requested
+            supported = self.download_results_rows_supported_formats()
+            if not supported:
+                self.request.session.flash("There are no supported download formats!",
+                                           'error')
+                return self.redirect(self.get_index_url())
+            fmt = self.request.POST.get('fmt')
+            if not fmt:
+                fmt = self.download_results_rows_default_format() or list(supported)[0]
+            if fmt not in supported:
+                self.request.session.flash("Unsupported download format: {}".format(fmt),
+                                           'error')
+                return self.redirect(self.get_index_url())
+
+            # parse field list if one was given
+            fields = self.request.POST.get('fields')
+            if fields:
+                fields = fields.split(',')
+            if not fields:
+                if fmt == 'csv':
+                    fields = self.get_row_csv_fields()
+                elif fmt == 'xlsx':
+                    fields = self.get_row_xlsx_fields()
+                else:
+                    self.request.session.flash("No fields were specified", 'error')
+                    return self.redirect(self.get_index_url())
+
+            # start thread to actually do work / report progress
+            key = '{}.download_results_rows'.format(route_prefix)
+            progress = self.make_progress(key)
+            results = self.get_effective_data()
+            thread = Thread(target=self.download_results_rows_thread,
+                            args=(results, fmt, fields, user_uuid, progress))
+            thread.start()
+
+            # show user the progress page
+            return self.render_progress(progress, {
+                'cancel_url': self.get_index_url(),
+                'cancel_msg': "Download was canceled.",
+            })
+
+        # not POST, so just download a file (if specified)
+        filename = self.request.GET.get('filename')
+        if not filename:
+            return self.redirect(self.get_index_url())
+        path = self.download_results_rows_path(user_uuid, filename)
+        return self.file_response(path)
+
+    def download_results_rows_filename(self, fmt):
+        """
+        Must return an appropriate "download results" filename for the given
+        format.  E.g. ``'products.csv'``
+        """
+        route_prefix = self.get_route_prefix()
+        if fmt == 'csv':
+            return '{}.rows.csv'.format(route_prefix)
+        if fmt == 'xlsx':
+            return '{}.rows.xlsx'.format(route_prefix)
+
+    def download_results_rows_path(self, user_uuid, filename=None,
+                              typ='results', makedirs=False):
+        """
+        Returns an absolute path for the "results" data file, specific to the
+        given user UUID.
+        """
+        route_prefix = self.get_route_prefix()
+        path = os.path.join(self.rattail_config.datadir(), 'downloads',
+                            typ, route_prefix,
+                            user_uuid[:2], user_uuid[2:])
+        if makedirs and not os.path.exists(path):
+            os.makedirs(path)
+
+        if filename:
+            path = os.path.join(path, filename)
+        return path
+
+    def download_results_rows_fields_available(self, **kwargs):
+        """
+        Return the list of fields which are *available* to be written to
+        download file.  Default field list will be constructed from the
+        underlying table columns.
+        """
+        fields = []
+        mapper = orm.class_mapper(self.model_class)
+        for prop in mapper.iterate_properties:
+            if isinstance(prop, orm.ColumnProperty):
+                fields.append(prop.key)
+        return fields
+
+    def download_results_rows_fields_default(self, fields, **kwargs):
+        """
+        Return the default list of fields to be written to download file.
+        Unless you override, all "available" fields will be included by
+        default.
+        """
+        return fields
+
+    def download_results_rows_thread(self, results, fmt, fields, user_uuid, progress):
+        """
+        Thread target, which invokes :meth:`download_results_generate()` to
+        officially generate the data file which is then to be downloaded.
+        """
+        route_prefix = self.get_route_prefix()
+        session = self.make_isolated_session()
+        try:
+
+            # create folder(s) for output; make sure file doesn't exist
+            filename = self.download_results_rows_filename(fmt)
+            path = self.download_results_rows_path(user_uuid, filename, makedirs=True)
+            if os.path.exists(path):
+                os.remove(path)
+
+            # generate file for download
+            results = results.with_session(session).all()
+            self.download_results_rows_setup(fields, progress=progress)
+            self.download_results_rows_generate(session, results, path, fmt, fields,
+                                                progress=progress)
+
+            session.commit()
+
+        except Exception as error:
+            msg = "failed to generate results file for download!"
+            log.warning(msg, exc_info=True)
+            session.rollback()
+            if progress:
+                progress.session.load()
+                progress.session['error'] = True
+                progress.session['error_msg'] = "{}: {}".format(
+                    msg, simple_error(error))
+                progress.session.save()
+            return
+
+        finally:
+            session.close()
+
+        if progress:
+            progress.session.load()
+            progress.session['complete'] = True
+            progress.session['success_url'] = self.get_index_url()
+            progress.session['extra_session_bits'] = {
+                '{}.results_rows.generated'.format(route_prefix): path,
+            }
+            progress.session.save()
+
+    def download_results_rows_setup(self, fields, progress=None):
+        """
+        Perform any up-front caching or other setup required, just prior to
+        generating a new results data file for download.
+        """
+
+    def download_results_rows_generate(self, session, results, path, fmt, fields, progress=None):
+        """
+        This method is responsible for actually generating the data file for a
+        "download rows for results" operation, according to the given params.
+        """
+        # we really are concerned with "rows of results" here, so let's just
+        # replace the 'results' list with a list of rows
+        original_results = results
+        results = []
+
+        def collect(obj, i):
+            results.extend(self.get_row_data(obj).all())
+
+        self.progress_loop(collect, original_results, progress,
+                           message="Collecting data for {}".format(self.get_row_model_title_plural()))
+
+        if fmt == 'csv':
+
+            if six.PY2:
+                csv_file = open(path, 'wb')
+                writer = UnicodeDictWriter(csv_file, fields, encoding='utf_8')
+            else: # PY3
+                csv_file = open(path, 'wt', encoding='utf_8')
+                writer = csv.DictWriter(csv_file, fields)
+            writer.writeheader()
+
+            def write(obj, i):
+                data = self.download_results_rows_normalize(obj, fields, fmt=fmt)
+                csvrow = self.download_results_rows_coerce_csv(data, fields)
+                writer.writerow(csvrow)
+
+            self.progress_loop(write, results, progress,
+                               message="Writing data to CSV file")
+            csv_file.close()
+
+        elif fmt == 'xlsx':
+
+            writer = ExcelWriter(path, fields,
+                                 sheet_title=self.get_row_model_title_plural())
+            writer.write_header()
+
+            xlrows = []
+            def write(obj, i):
+                data = self.download_results_rows_normalize(obj, fields, fmt=fmt)
+                row = self.download_results_rows_coerce_xlsx(data, fields)
+                xlrow = [row[field] for field in fields]
+                xlrows.append(xlrow)
+
+            self.progress_loop(write, results, progress,
+                               message="Collecting data for Excel")
+
+            def finalize(x, i):
+                writer.write_rows(xlrows)
+                writer.auto_freeze()
+                writer.auto_filter()
+                writer.auto_resize()
+                writer.save()
+
+            self.progress_loop(finalize, [1], progress,
+                               message="Writing Excel file to disk")
+
+    def download_results_rows_normalize(self, row, fields, **kwargs):
+        """
+        Normalize the given row object into a data dict, for use when writing
+        to the results file for download.
+        """
+        data = {}
+        for field in fields:
+            value = getattr(row, field, None)
+
+            # make timestamps zone-aware
+            if isinstance(value, datetime.datetime):
+                value = localtime(self.rattail_config, value,
+                                  from_utc=not self.has_local_times)
+
+            data[field] = value
+
+        return data
+
+    def download_results_rows_coerce_csv(self, data, fields, **kwargs):
+        """
+        Coerce the given data dict record, to a "row" dict suitable for use
+        when writing directly to CSV file.  Each value in the dict should be a
+        string type.
+        """
+        csvrow = dict(data)
+        for field in fields:
+            value = csvrow.get(field)
+
+            if value is None:
+                value = ''
+            else:
+                value = six.text_type(value)
+
+            csvrow[field] = value
+
+        return csvrow
+
+    def download_results_rows_coerce_xlsx(self, data, fields, **kwargs):
+        """
+        Coerce the given data dict record, to a "row" dict suitable for use
+        when writing directly to XLSX file.
+        """
+        data = dict(data)
+        for key in data:
+            value = data[key]
+
+            # convert GPC to pretty string
+            if isinstance(value, GPC):
+                value = value.pretty()
+
+            # make timestamps local, "zone-naive"
+            elif isinstance(value, datetime.datetime):
+                value = localtime(self.rattail_config, value, tzinfo=False)
+
+            data[key] = value
+
+        return data
+
     def row_results_xlsx(self):
         """
         Download current *row* results as XLSX.
@@ -4152,6 +4452,14 @@ class MasterView(View):
                 config.add_view(cls, attr='results_xlsx_download', route_name='{}.results_xlsx_download'.format(route_prefix),
                                 permission='{}.results_xlsx'.format(permission_prefix))
 
+            # download rows for results
+            if cls.has_rows and cls.results_rows_downloadable:
+                config.add_tailbone_permission(permission_prefix, '{}.download_results_rows'.format(permission_prefix),
+                                               "Download *rows* for {} search results".format(model_title))
+                config.add_route('{}.download_results_rows'.format(route_prefix), '{}/download-rows-for-results'.format(url_prefix))
+                config.add_view(cls, attr='download_results_rows', route_name='{}.download_results_rows'.format(route_prefix),
+                                permission='{}.download_results_rows'.format(permission_prefix))
+
         # quickie (search)
         if cls.supports_quickie_search:
             config.add_tailbone_permission(permission_prefix, '{}.quickie'.format(permission_prefix),