Add new/flexible "download results" feature

tailbone/helpers.py

@@ -26,6 +26,7 @@ Template Context Helpers
 
 from __future__ import unicode_literals, absolute_import
 
+import os
 import datetime
 from decimal import Decimal
 

tailbone/templates/master/index.mako

@@ -143,6 +143,155 @@
 
 <%def name="grid_tools()">
 
+  ## download search results
+  % if master.results_downloadable and master.has_perm('download_results'):
+      % if use_buefy:
+          <b-button type="is-primary"
+                    icon-pack="fas"
+                    icon-left="fas fa-download"
+                    @click="showDownloadResultsDialog = true"
+                    :disabled="!total">
+            Download Results
+          </b-button>
+
+          ${h.form(url('{}.download_results'.format(route_prefix)), ref='download_results_form')}
+          ${h.csrf_token(request)}
+          <input type="hidden" name="fmt" :value="downloadResultsFormat" />
+          <input type="hidden" name="fields" :value="downloadResultsFieldsIncluded" />
+          ${h.end_form()}
+
+          <b-modal :active.sync="showDownloadResultsDialog">
+            <div class="card">
+
+              <div class="card-content">
+                <p>
+                  There are
+                  <span class="is-size-4 has-text-weight-bold">
+                    {{ total.toLocaleString('en') }} ${model_title_plural}
+                  </span>
+                  matching your current filters.
+                </p>
+                <p>
+                  You may download this set as a single data file if you like.
+                </p>
+                <br />
+
+                <b-notification type="is-warning" :closable="false"
+                                v-if="downloadResultsFormat == 'xlsx' && total >= 1000">
+                  Excel downloads for large data sets can take a long time to
+                  generate, and bog down the server in the meantime.  You are
+                  encouraged to choose CSV for a large data set, even though
+                  the end result (file size) may be larger with CSV.
+                </b-notification>
+
+                <div style="display: flex;">
+
+                  <div style="flex-grow: 1;">
+                    <b-field horizontal label="Format">
+                      <b-select v-model="downloadResultsFormat">
+                        % for key, label in six.iteritems(master.download_results_supported_formats()):
+                        <option value="${key}">${label}</option>
+                        % endfor
+                      </b-select>
+                    </b-field>
+                  </div>
+
+                  <div>
+                    <div v-show="downloadResultsFieldsMode != 'choose'"
+                         class="has-text-right">
+                      <p v-if="downloadResultsFieldsMode == 'default'">
+                        Will use DEFAULT fields.
+                      </p>
+                      <p v-if="downloadResultsFieldsMode == 'all'">
+                        Will use ALL fields.
+                      </p>
+                    </div>
+                    <br />
+                    <div class="buttons">
+                      <b-button type="is-primary"
+                                v-show="downloadResultsFieldsMode != 'default'"
+                                @click="downloadResultsUseDefaultFields()">
+                        Use Default Fields
+                      </b-button>
+                      <b-button type="is-primary"
+                                v-show="downloadResultsFieldsMode != 'all'"
+                                @click="downloadResultsUseAllFields()">
+                        Use All Fields
+                      </b-button>
+                      <b-button type="is-primary"
+                                v-show="downloadResultsFieldsMode != 'choose'"
+                                @click="downloadResultsFieldsMode = 'choose'">
+                        Choose Fields
+                      </b-button>
+                    </div>
+                  </div>
+
+                </div>
+                <br />
+
+                <div v-show="downloadResultsFieldsMode == 'choose'">
+                  <div style="display: flex;">
+                    <div>
+                      <b-field label="Excluded Fields">
+                        <b-select multiple native-size="8"
+                                  expanded
+                                  ref="downloadResultsExcludedFields">
+                          <option v-for="field in downloadResultsFieldsAvailable"
+                                  v-if="!downloadResultsFieldsIncluded.includes(field)"
+                                  :key="field"
+                                  :value="field">
+                            {{ field }}
+                          </option>
+                        </b-select>
+                      </b-field>
+                    </div>
+                    <div>
+                      <br /><br />
+                      <b-button style="margin: 0.5rem;"
+                                @click="downloadResultsExcludeFields()">
+                        &lt;
+                      </b-button>
+                      <br />
+                      <b-button style="margin: 0.5rem;"
+                                @click="downloadResultsIncludeFields()">
+                        &gt;
+                      </b-button>
+                    </div>
+                    <div>
+                      <b-field label="Included Fields">
+                        <b-select multiple native-size="8"
+                                  expanded
+                                  ref="downloadResultsIncludedFields">
+                          <option v-for="field in downloadResultsFieldsIncluded"
+                                  :key="field"
+                                  :value="field">
+                            {{ field }}
+                          </option>
+                        </b-select>
+                      </b-field>
+                    </div>
+                  </div>
+                </div>
+
+              </div> <!-- card-content -->
+
+              <footer class="modal-card-foot">
+                <b-button @click="showDownloadResultsDialog = false">
+                  Cancel
+                </b-button>
+                <once-button type="is-primary"
+                             @click="downloadResultsSubmit()"
+                             icon-pack="fas"
+                             icon-left="fas fa-download"
+                             :disabled="!downloadResultsFieldsIncluded.length"
+                             text="Download Results">
+                </once-button>
+              </footer>
+            </div>
+          </b-modal>
+      % endif
+  % endif
+
   ## merge 2 objects
   % if master.mergeable and request.has_perm('{}.merge'.format(permission_prefix)):
 
@@ -256,6 +405,14 @@
 </%def>
 
 <%def name="page_content()">
+
+  % if download_results_path:
+      <b-notification type="is-info">
+        Your download should start automatically, or you can
+        ${h.link_to("click here", '{}?filename={}'.format(url('{}.download_results'.format(route_prefix)), h.os.path.basename(download_results_path)))}
+      </b-notification>
+  % endif
+
   <${grid.component} :csrftoken="csrftoken"
      % if master.deletable and request.has_perm('{}.delete'.format(permission_prefix)) and master.delete_confirm == 'simple':
      @deleteActionClicked="deleteObject"
@@ -295,6 +452,19 @@
   ${parent.modify_this_page_vars()}
   <script type="text/javascript">
 
+    ## maybe auto-redirect to download latest results file
+    % if download_results_path and use_buefy:
+        ThisPage.methods.downloadResultsRedirect = function() {
+            location.href = '${url('{}.download_results'.format(route_prefix))}?filename=${h.os.path.basename(download_results_path)}';
+        }
+        ThisPage.mounted = function() {
+            // we give this 1 second before attempting the redirect; otherwise
+            // the FontAwesome icons do not seem to load properly.  so this way
+            // the page should fully render before redirecting
+            window.setTimeout(this.downloadResultsRedirect, 1000)
+        }
+    % endif
+
     ## TODO: stop checking for buefy here once we only have the one session.pop()
     % if use_buefy and request.session.pop('{}.results_csv.generated'.format(route_prefix), False):
         ThisPage.mounted = function() {
@@ -318,6 +488,83 @@
         }
     % endif
 
+    ## download results
+    % if master.results_downloadable and master.has_perm('download_results'):
+
+        ${grid.component_studly}Data.downloadResultsFormat = '${master.download_results_default_format()}'
+        ${grid.component_studly}Data.showDownloadResultsDialog = false
+        ${grid.component_studly}Data.downloadResultsFieldsMode = 'default'
+        ${grid.component_studly}Data.downloadResultsFieldsAvailable = ${json.dumps(download_results_fields_available)|n}
+        ${grid.component_studly}Data.downloadResultsFieldsDefault = ${json.dumps(download_results_fields_default)|n}
+        ${grid.component_studly}Data.downloadResultsFieldsIncluded = ${json.dumps(download_results_fields_default)|n}
+
+        ${grid.component_studly}.computed.downloadResultsFieldsExcluded = function() {
+            let excluded = []
+            this.downloadResultsFieldsAvailable.forEach(field => {
+                if (!this.downloadResultsFieldsIncluded.includes(field)) {
+                    excluded.push(field)
+                }
+            }, this)
+            return excluded
+        }
+
+        ${grid.component_studly}.methods.downloadResultsExcludeFields = function() {
+            let selected = this.$refs.downloadResultsIncludedFields.selected
+            if (!selected) {
+                return
+            }
+            selected = Array.from(selected)
+            selected.forEach(field => {
+
+                // de-select the entry within "included" field input
+                let index = this.$refs.downloadResultsIncludedFields.selected.indexOf(field)
+                if (index > -1) {
+                    this.$refs.downloadResultsIncludedFields.selected.splice(index, 1)
+                }
+
+                // remove field from official "included" list
+                index = this.downloadResultsFieldsIncluded.indexOf(field)
+                if (index > -1) {
+                    this.downloadResultsFieldsIncluded.splice(index, 1)
+                }
+            }, this)
+        }
+
+        ${grid.component_studly}.methods.downloadResultsIncludeFields = function() {
+            let selected = this.$refs.downloadResultsExcludedFields.selected
+            if (!selected) {
+                return
+            }
+            selected = Array.from(selected)
+            selected.forEach(field => {
+
+                // de-select the entry within "excluded" field input
+                let index = this.$refs.downloadResultsExcludedFields.selected.indexOf(field)
+                if (index > -1) {
+                    this.$refs.downloadResultsExcludedFields.selected.splice(index, 1)
+                }
+
+                // add field to official "included" list
+                this.downloadResultsFieldsIncluded.push(field)
+
+            }, this)
+        }
+
+        ${grid.component_studly}.methods.downloadResultsUseDefaultFields = function() {
+            this.downloadResultsFieldsIncluded = Array.from(this.downloadResultsFieldsDefault)
+            this.downloadResultsFieldsMode = 'default'
+        }
+
+        ${grid.component_studly}.methods.downloadResultsUseAllFields = function() {
+            this.downloadResultsFieldsIncluded = Array.from(this.downloadResultsFieldsAvailable)
+            this.downloadResultsFieldsMode = 'all'
+        }
+
+        ${grid.component_studly}.methods.downloadResultsSubmit = function() {
+            this.$refs.download_results_form.submit()
+        }
+    % endif
+
     ## enable / disable selected objects
     % if master.supports_set_enabled_toggle and master.has_perm('enable_disable_set'):
 

tailbone/views/master.py

@@ -76,8 +76,12 @@ class MasterView(View):
     # set to True in order to encode search values as utf-8
     use_byte_string_filters = False
 
+    # set to True if all timestamps are "local" instead of UTC
+    has_local_times = False
+
     listable = True
     sortable = True
+    results_downloadable = False
     results_downloadable_csv = False
     results_downloadable_xlsx = False
     creatable = True
@@ -191,6 +195,22 @@ class MasterView(View):
         from tailbone.db import Session
         return Session
 
+    def make_isolated_session(self):
+        """
+        This method should return a newly-created SQLAlchemy Session instance.
+        The use case here is primarily for secondary threads, which may be
+        employed for long-running processes such as executing a batch.  The
+        session returned should *not* have any web hooks to auto-commit with
+        the request/response cycle etc.  It should just be a plain old session,
+        "isolated" from the rest of the web app in a sense.
+
+        So whereas ``self.Session`` by default will return a reference to
+        ``tailbone.db.Session``, which is a "scoped" session wrapper specific
+        to the current thread (one per request), this method should instead
+        return e.g. a new independent ``rattail.db.Session`` instance.
+        """
+        return RattailSession()
+
     @classmethod
     def get_grid_factory(cls):
         """
@@ -327,6 +347,15 @@ class MasterView(View):
         context = {
             'grid': grid,
         }
+
+        if self.results_downloadable and self.has_perm('download_results'):
+            route_prefix = self.get_route_prefix()
+            context['download_results_path'] = self.request.session.pop(
+                '{}.results.generated'.format(route_prefix), None)
+            available = self.download_results_fields_available()
+            context['download_results_fields_available'] = available
+            context['download_results_fields_default'] = self.download_results_fields_default(available)
+
         return self.render_to_response('index', context)
 
     def make_grid(self, factory=None, key=None, data=None, columns=None, **kwargs):
@@ -1900,7 +1929,7 @@ class MasterView(View):
                            message="Deleting objects")
 
     def get_bulk_delete_session(self):
-        return RattailSession()
+        return self.make_isolated_session()
 
     def bulk_delete_thread(self, objects, progress):
         """
@@ -2676,6 +2705,264 @@ class MasterView(View):
         """
         return False
 
+    def download_results_path(self, user_uuid, filename=None,
+                              typ='results', makedirs=False):
+        """
+        Returns an absolute path for the "results" data file, specific to the
+        given user UUID.
+        """
+        route_prefix = self.get_route_prefix()
+        path = os.path.join(self.rattail_config.datadir(), 'downloads',
+                            typ, route_prefix,
+                            user_uuid[:2], user_uuid[2:])
+        if makedirs and not os.path.exists(path):
+            os.makedirs(path)
+
+        if filename:
+            path = os.path.join(path, filename)
+        return path
+
+    def download_results_filename(self, fmt):
+        """
+        Must return an appropriate "download results" filename for the given
+        format.  E.g. ``'products.csv'``
+        """
+        route_prefix = self.get_route_prefix()
+        if fmt == 'csv':
+            return '{}.csv'.format(route_prefix)
+        if fmt == 'xlsx':
+            return '{}.xlsx'.format(route_prefix)
+
+    def download_results_supported_formats(self):
+        # TODO: default formats should be configurable?
+        return OrderedDict([
+            ('xlsx', "Excel (XLSX)"),
+            ('csv', "CSV"),
+        ])
+
+    def download_results_default_format(self):
+        # TODO: default format should be configurable
+        return 'xlsx'
+
+    def download_results(self):
+        """
+        View for saving current (filtered) data results into a file, and
+        downloading that file.
+        """
+        route_prefix = self.get_route_prefix()
+        user_uuid = self.request.user.uuid
+
+        # POST means generate a new results file for download
+        if self.request.method == 'POST':
+
+            # make sure a valid format was requested
+            supported = self.download_results_supported_formats()
+            if not supported:
+                self.request.session.flash("There are no supported download formats!",
+                                           'error')
+                return self.redirect(self.get_index_url())
+            fmt = self.request.POST.get('fmt')
+            if not fmt:
+                fmt = self.download_results_default_format() or list(supported)[0]
+            if fmt not in supported:
+                self.request.session.flash("Unsupported download format: {}".format(fmt),
+                                           'error')
+                return self.redirect(self.get_index_url())
+
+            # parse field list if one was given
+            fields = self.request.POST.get('fields')
+            if fields:
+                fields = fields.split(',')
+
+            # start thread to actually do work / report progress
+            key = '{}.download_results'.format(route_prefix)
+            progress = self.make_progress(key)
+            results = self.get_effective_data()
+            thread = Thread(target=self.download_results_thread,
+                            args=(results, fmt, fields, user_uuid, progress))
+            thread.start()
+
+            # show user the progress page
+            return self.render_progress(progress, {
+                'cancel_url': self.get_index_url(),
+                'cancel_msg': "Download was canceled.",
+            })
+
+        # not POST, so just download a file (if specified)
+        filename = self.request.GET.get('filename')
+        if not filename:
+            return self.redirect(self.get_index_url())
+        path = self.download_results_path(user_uuid, filename)
+        return self.file_response(path)
+
+    def download_results_thread(self, results, fmt, fields, user_uuid, progress):
+        """
+        Thread target, which invokes :meth:`download_results_generate()` to
+        officially generate the data file which is then to be downloaded.
+        """
+        route_prefix = self.get_route_prefix()
+        session = self.make_isolated_session()
+        try:
+
+            # create folder(s) for output; make sure file doesn't exist
+            filename = self.download_results_filename(fmt)
+            path = self.download_results_path(user_uuid, filename, makedirs=True)
+            if os.path.exists(path):
+                os.remove(path)
+
+            # generate file for download
+            results = results.with_session(session).all()
+            self.download_results_setup(fields, progress=progress)
+            self.download_results_generate(session, results, path, fmt, fields,
+                                           progress=progress)
+
+            session.commit()
+
+        except Exception as error:
+            msg = "failed to generate results file for download!"
+            log.warning(msg, exc_info=True)
+            session.rollback()
+            if progress:
+                progress.session.load()
+                progress.session['error'] = True
+                progress.session['error_msg'] = "{}: {}".format(
+                    msg, simple_error(error))
+                progress.session.save()
+            return
+
+        finally:
+            session.close()
+
+        if progress:
+            progress.session.load()
+            progress.session['complete'] = True
+            progress.session['success_url'] = self.get_index_url()
+            progress.session['extra_session_bits'] = {
+                '{}.results.generated'.format(route_prefix): path,
+            }
+            progress.session.save()
+
+    def download_results_setup(self, fields, progress=None):
+        """
+        Perform any up-front caching or other setup required, just prior to
+        generating a new results data file for download.
+        """
+
+    def download_results_generate(self, session, results, path, fmt, fields, progress=None):
+        """
+        This method is responsible for actually generating the data file for a
+        "download results" operation, according to the given params.
+        """
+        if fmt == 'csv':
+
+            if six.PY2:
+                csv_file = open(path, 'wb')
+                writer = UnicodeDictWriter(csv_file, fields, encoding='utf_8')
+            else: # PY3
+                csv_file = open(path, 'wt', encoding='utf_8')
+                writer = csv.DictWriter(csv_file, fields)
+            writer.writeheader()
+
+            def write(obj, i):
+                data = self.download_results_normalize(obj, fields, fmt=fmt)
+                row = self.download_results_coerce_csv(data, fields)
+                writer.writerow(row)
+
+            self.progress_loop(write, results, progress,
+                               message="Writing data to CSV file")
+            csv_file.close()
+
+        elif fmt == 'xlsx':
+
+            writer = ExcelWriter(path, fields,
+                                 sheet_title=self.get_model_title_plural())
+            writer.write_header()
+
+            xlrows = []
+            def write(obj, i):
+                data = self.download_results_normalize(obj, fields, fmt=fmt)
+                row = self.download_results_coerce_xlsx(data, fields)
+                xlrow = [row[field] for field in fields]
+                xlrows.append(xlrow)
+
+            self.progress_loop(write, results, progress,
+                               message="Collecting data for Excel")
+
+            def finalize(x, i):
+                writer.write_rows(xlrows)
+                writer.auto_freeze()
+                writer.auto_filter()
+                writer.auto_resize()
+                writer.save()
+
+            self.progress_loop(finalize, [1], progress,
+                               message="Writing Excel file to disk")
+
+    def download_results_fields_available(self, **kwargs):
+        """
+        Return the list of fields which are *available* to be written to
+        download file.  Default field list will be constructed from the
+        underlying table columns.
+        """
+        fields = []
+        mapper = orm.class_mapper(self.model_class)
+        for prop in mapper.iterate_properties:
+            if isinstance(prop, orm.ColumnProperty):
+                fields.append(prop.key)
+        return fields
+
+    def download_results_fields_default(self, fields, **kwargs):
+        """
+        Return the default list of fields to be written to download file.
+        Unless you override, all "available" fields will be included by
+        default.
+        """
+        return fields
+
+    def download_results_normalize(self, obj, fields, **kwargs):
+        """
+        Normalize the given object into a data dict, for use when writing to
+        the results file for download.
+        """
+        data = {}
+        for field in fields:
+            value = getattr(obj, field, None)
+
+            # make timestamps zone-aware
+            if isinstance(value, datetime.datetime):
+                value = localtime(self.rattail_config, value,
+                                  from_utc=not self.has_local_times)
+
+            data[field] = value
+
+        return data
+
+    def download_results_coerce_csv(self, data, fields, **kwargs):
+        """
+        Coerce the given data dict record, to a "row" dict suitable for use
+        when writing directly to CSV file.  Each value in the dict should be a
+        string type.
+        """
+        csvrow = dict(data)
+        for field in fields:
+            value = csvrow.get(field)
+
+            if value is None:
+                value = ''
+            else:
+                value = six.text_type(value)
+
+            csvrow[field] = value
+
+        return csvrow
+
+    def download_results_coerce_xlsx(self, data, fields, **kwargs):
+        """
+        Coerce the given data dict record, to a "row" dict suitable for use
+        when writing directly to XLSX file.
+        """
+        return data
+
     def results_csv(self):
         """
         Download current list results as CSV.
@@ -2697,7 +2984,7 @@ class MasterView(View):
         })
 
     def results_csv_session(self):
-        return RattailSession()
+        return self.make_isolated_session()
 
     def results_csv_thread(self, results, user_uuid, progress):
         """
@@ -2793,7 +3080,7 @@ class MasterView(View):
         })
 
     def results_xlsx_session(self):
-        return RattailSession()
+        return self.make_isolated_session()
 
     def results_xlsx_thread(self, results, user_uuid, progress):
         """
@@ -3807,6 +4094,20 @@ class MasterView(View):
                 config.add_view(cls, attr='mobile_index', route_name='mobile.{}'.format(route_prefix),
                                 permission='{}.list'.format(permission_prefix))
 
+            # download results
+            # this is the "new" more flexible approach, but we only want to
+            # enable it if the class declares it, *and* does *not* declare the
+            # older style(s).  that way each class must explicitly choose
+            # *only* the new style in order to use it
+            if cls.results_downloadable and not (
+                    cls.results_downloadable_csv or cls.results_downloadable_xlsx):
+                config.add_tailbone_permission(permission_prefix, '{}.download_results'.format(permission_prefix),
+                                               "Download search results for {}".format(model_title_plural))
+                config.add_route('{}.download_results'.format(route_prefix), '{}/download-results'.format(url_prefix))
+                config.add_view(cls, attr='download_results', route_name='{}.download_results'.format(route_prefix),
+                                permission='{}.download_results'.format(permission_prefix))
+
+            # download results as CSV (deprecated)
             if cls.results_downloadable_csv:
                 config.add_tailbone_permission(permission_prefix, '{}.results_csv'.format(permission_prefix),
                                                "Download {} as CSV".format(model_title_plural))
@@ -3817,6 +4118,7 @@ class MasterView(View):
                 config.add_view(cls, attr='results_csv_download', route_name='{}.results_csv_download'.format(route_prefix),
                                 permission='{}.results_csv'.format(permission_prefix))
 
+            # download results as XLSX (deprecated)
             if cls.results_downloadable_xlsx:
                 config.add_tailbone_permission(permission_prefix, '{}.results_xlsx'.format(permission_prefix),
                                                "Download {} as XLSX".format(model_title_plural))