rattail/tailbone
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add new/flexible "download results" feature

Browse source
This commit is contained in:
Lance Edgar 2020-08-22 12:25:02 -05:00
parent 43472c7eb6
commit 922cbe4451
3 changed files with 553 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
tailbone/helpers.py
View file

@ -26,6 +26,7 @@ Template Context Helpers
from __future__ import unicode_literals, absolute_import
import os
import datetime
from decimal import Decimal

247
tailbone/templates/master/index.mako
View file

@ -143,6 +143,155 @@
<%def name="grid_tools()">
## download search results
% if master.results_downloadable and master.has_perm('download_results'):
% if use_buefy:
<b-button type="is-primary"
icon-pack="fas"
icon-left="fas fa-download"
@click="showDownloadResultsDialog = true"
:disabled="!total">
Download Results
</b-button>
${h.form(url('{}.download_results'.format(route_prefix)), ref='download_results_form')}
${h.csrf_token(request)}
<input type="hidden" name="fmt" :value="downloadResultsFormat" />
<input type="hidden" name="fields" :value="downloadResultsFieldsIncluded" />
${h.end_form()}
<b-modal :active.sync="showDownloadResultsDialog">
<div class="card">
<div class="card-content">
<p>
There are
<span class="is-size-4 has-text-weight-bold">
{{ total.toLocaleString('en') }} ${model_title_plural}
</span>
matching your current filters.
</p>
<p>
You may download this set as a single data file if you like.
</p>
<br />
<b-notification type="is-warning" :closable="false"
v-if="downloadResultsFormat == 'xlsx' && total >= 1000">
Excel downloads for large data sets can take a long time to
generate, and bog down the server in the meantime. You are
encouraged to choose CSV for a large data set, even though
the end result (file size) may be larger with CSV.
</b-notification>
<div style="display: flex;">
<div style="flex-grow: 1;">
<b-field horizontal label="Format">
<b-select v-model="downloadResultsFormat">
% for key, label in six.iteritems(master.download_results_supported_formats()):
<option value="${key}">${label}</option>
% endfor
</b-select>
</b-field>
</div>
<div>
<div v-show="downloadResultsFieldsMode != 'choose'"
class="has-text-right">
<p v-if="downloadResultsFieldsMode == 'default'">
Will use DEFAULT fields.
</p>
<p v-if="downloadResultsFieldsMode == 'all'">
Will use ALL fields.
</p>
</div>
<br />
<div class="buttons">
<b-button type="is-primary"
v-show="downloadResultsFieldsMode != 'default'"
@click="downloadResultsUseDefaultFields()">
Use Default Fields
</b-button>
<b-button type="is-primary"
v-show="downloadResultsFieldsMode != 'all'"
@click="downloadResultsUseAllFields()">
Use All Fields
</b-button>
<b-button type="is-primary"
v-show="downloadResultsFieldsMode != 'choose'"
@click="downloadResultsFieldsMode = 'choose'">
Choose Fields
</b-button>
</div>
</div>
</div>
<br />
<div v-show="downloadResultsFieldsMode == 'choose'">
<div style="display: flex;">
<div>
<b-field label="Excluded Fields">
<b-select multiple native-size="8"
expanded
ref="downloadResultsExcludedFields">
<option v-for="field in downloadResultsFieldsAvailable"
v-if="!downloadResultsFieldsIncluded.includes(field)"
:key="field"
:value="field">
{{ field }}
</option>
</b-select>
</b-field>
</div>
<div>
<br /><br />
<b-button style="margin: 0.5rem;"
@click="downloadResultsExcludeFields()">
&lt;
</b-button>
<br />
<b-button style="margin: 0.5rem;"
@click="downloadResultsIncludeFields()">
&gt;
</b-button>
</div>
<div>
<b-field label="Included Fields">
<b-select multiple native-size="8"
expanded
ref="downloadResultsIncludedFields">
<option v-for="field in downloadResultsFieldsIncluded"
:key="field"
:value="field">
{{ field }}
</option>
</b-select>
</b-field>
</div>
</div>
</div>
</div> <!-- card-content -->
<footer class="modal-card-foot">
<b-button @click="showDownloadResultsDialog = false">
Cancel
</b-button>
<once-button type="is-primary"
@click="downloadResultsSubmit()"
icon-pack="fas"
icon-left="fas fa-download"
:disabled="!downloadResultsFieldsIncluded.length"
text="Download Results">
</once-button>
</footer>
</div>
</b-modal>
% endif
% endif
## merge 2 objects
% if master.mergeable and request.has_perm('{}.merge'.format(permission_prefix)):
@ -256,6 +405,14 @@
</%def>
<%def name="page_content()">
% if download_results_path:
<b-notification type="is-info">
Your download should start automatically, or you can
${h.link_to("click here", '{}?filename={}'.format(url('{}.download_results'.format(route_prefix)), h.os.path.basename(download_results_path)))}
</b-notification>
% endif
<${grid.component} :csrftoken="csrftoken"
% if master.deletable and request.has_perm('{}.delete'.format(permission_prefix)) and master.delete_confirm == 'simple':
@deleteActionClicked="deleteObject"
@ -295,6 +452,19 @@
${parent.modify_this_page_vars()}
<script type="text/javascript">
## maybe auto-redirect to download latest results file
% if download_results_path and use_buefy:
ThisPage.methods.downloadResultsRedirect = function() {
location.href = '${url('{}.download_results'.format(route_prefix))}?filename=${h.os.path.basename(download_results_path)}';
}
ThisPage.mounted = function() {
// we give this 1 second before attempting the redirect; otherwise
// the FontAwesome icons do not seem to load properly. so this way
// the page should fully render before redirecting
window.setTimeout(this.downloadResultsRedirect, 1000)
}
% endif
## TODO: stop checking for buefy here once we only have the one session.pop()
% if use_buefy and request.session.pop('{}.results_csv.generated'.format(route_prefix), False):
ThisPage.mounted = function() {
@ -318,6 +488,83 @@
}
% endif
## download results
% if master.results_downloadable and master.has_perm('download_results'):
${grid.component_studly}Data.downloadResultsFormat = '${master.download_results_default_format()}'
${grid.component_studly}Data.showDownloadResultsDialog = false
${grid.component_studly}Data.downloadResultsFieldsMode = 'default'
${grid.component_studly}Data.downloadResultsFieldsAvailable = ${json.dumps(download_results_fields_available)|n}
${grid.component_studly}Data.downloadResultsFieldsDefault = ${json.dumps(download_results_fields_default)|n}
${grid.component_studly}Data.downloadResultsFieldsIncluded = ${json.dumps(download_results_fields_default)|n}
${grid.component_studly}.computed.downloadResultsFieldsExcluded = function() {
let excluded = []
this.downloadResultsFieldsAvailable.forEach(field => {
if (!this.downloadResultsFieldsIncluded.includes(field)) {
excluded.push(field)
}
}, this)
return excluded
}
${grid.component_studly}.methods.downloadResultsExcludeFields = function() {
let selected = this.$refs.downloadResultsIncludedFields.selected
if (!selected) {
return
}
selected = Array.from(selected)
selected.forEach(field => {
// de-select the entry within "included" field input
let index = this.$refs.downloadResultsIncludedFields.selected.indexOf(field)
if (index > -1) {
this.$refs.downloadResultsIncludedFields.selected.splice(index, 1)
}
// remove field from official "included" list
index = this.downloadResultsFieldsIncluded.indexOf(field)
if (index > -1) {
this.downloadResultsFieldsIncluded.splice(index, 1)
}
}, this)
}
${grid.component_studly}.methods.downloadResultsIncludeFields = function() {
let selected = this.$refs.downloadResultsExcludedFields.selected
if (!selected) {
return
}
selected = Array.from(selected)
selected.forEach(field => {
// de-select the entry within "excluded" field input
let index = this.$refs.downloadResultsExcludedFields.selected.indexOf(field)
if (index > -1) {
this.$refs.downloadResultsExcludedFields.selected.splice(index, 1)
}
// add field to official "included" list
this.downloadResultsFieldsIncluded.push(field)
}, this)
}
${grid.component_studly}.methods.downloadResultsUseDefaultFields = function() {
this.downloadResultsFieldsIncluded = Array.from(this.downloadResultsFieldsDefault)
this.downloadResultsFieldsMode = 'default'
}
${grid.component_studly}.methods.downloadResultsUseAllFields = function() {
this.downloadResultsFieldsIncluded = Array.from(this.downloadResultsFieldsAvailable)
this.downloadResultsFieldsMode = 'all'
}
${grid.component_studly}.methods.downloadResultsSubmit = function() {
this.$refs.download_results_form.submit()
}
% endif
## enable / disable selected objects
% if master.supports_set_enabled_toggle and master.has_perm('enable_disable_set'):

308
tailbone/views/master.py
View file

@ -76,8 +76,12 @@ class MasterView(View):
# set to True in order to encode search values as utf-8
use_byte_string_filters = False
# set to True if all timestamps are "local" instead of UTC
has_local_times = False
listable = True
sortable = True
results_downloadable = False
results_downloadable_csv = False
results_downloadable_xlsx = False
creatable = True
@ -191,6 +195,22 @@ class MasterView(View):
from tailbone.db import Session
return Session
def make_isolated_session(self):
"""
This method should return a newly-created SQLAlchemy Session instance.
The use case here is primarily for secondary threads, which may be
employed for long-running processes such as executing a batch. The
session returned should *not* have any web hooks to auto-commit with
the request/response cycle etc. It should just be a plain old session,
"isolated" from the rest of the web app in a sense.
So whereas ``self.Session`` by default will return a reference to
``tailbone.db.Session``, which is a "scoped" session wrapper specific
to the current thread (one per request), this method should instead
return e.g. a new independent ``rattail.db.Session`` instance.
"""
return RattailSession()
@classmethod
def get_grid_factory(cls):
"""
@ -327,6 +347,15 @@ class MasterView(View):
context = {
'grid': grid,
}
if self.results_downloadable and self.has_perm('download_results'):
route_prefix = self.get_route_prefix()
context['download_results_path'] = self.request.session.pop(
'{}.results.generated'.format(route_prefix), None)
available = self.download_results_fields_available()
context['download_results_fields_available'] = available
context['download_results_fields_default'] = self.download_results_fields_default(available)
return self.render_to_response('index', context)
def make_grid(self, factory=None, key=None, data=None, columns=None, **kwargs):
@ -1900,7 +1929,7 @@ class MasterView(View):
message="Deleting objects")
def get_bulk_delete_session(self):
return RattailSession()
return self.make_isolated_session()
def bulk_delete_thread(self, objects, progress):
"""
@ -2676,6 +2705,264 @@ class MasterView(View):
"""
return False
def download_results_path(self, user_uuid, filename=None,
typ='results', makedirs=False):
"""
Returns an absolute path for the "results" data file, specific to the
given user UUID.
"""
route_prefix = self.get_route_prefix()
path = os.path.join(self.rattail_config.datadir(), 'downloads',
typ, route_prefix,
user_uuid[:2], user_uuid[2:])
if makedirs and not os.path.exists(path):
os.makedirs(path)
if filename:
path = os.path.join(path, filename)
return path
def download_results_filename(self, fmt):
"""
Must return an appropriate "download results" filename for the given
format. E.g. ``'products.csv'``
"""
route_prefix = self.get_route_prefix()
if fmt == 'csv':
return '{}.csv'.format(route_prefix)
if fmt == 'xlsx':
return '{}.xlsx'.format(route_prefix)
def download_results_supported_formats(self):
# TODO: default formats should be configurable?
return OrderedDict([
('xlsx', "Excel (XLSX)"),
('csv', "CSV"),
])
def download_results_default_format(self):
# TODO: default format should be configurable
return 'xlsx'
def download_results(self):
"""
View for saving current (filtered) data results into a file, and
downloading that file.
"""
route_prefix = self.get_route_prefix()
user_uuid = self.request.user.uuid
# POST means generate a new results file for download
if self.request.method == 'POST':
# make sure a valid format was requested
supported = self.download_results_supported_formats()
if not supported:
self.request.session.flash("There are no supported download formats!",
'error')
return self.redirect(self.get_index_url())
fmt = self.request.POST.get('fmt')
if not fmt:
fmt = self.download_results_default_format() or list(supported)[0]
if fmt not in supported:
self.request.session.flash("Unsupported download format: {}".format(fmt),
'error')
return self.redirect(self.get_index_url())
# parse field list if one was given
fields = self.request.POST.get('fields')
if fields:
fields = fields.split(',')
# start thread to actually do work / report progress
key = '{}.download_results'.format(route_prefix)
progress = self.make_progress(key)
results = self.get_effective_data()
thread = Thread(target=self.download_results_thread,
args=(results, fmt, fields, user_uuid, progress))
thread.start()
# show user the progress page
return self.render_progress(progress, {
'cancel_url': self.get_index_url(),
'cancel_msg': "Download was canceled.",
})
# not POST, so just download a file (if specified)
filename = self.request.GET.get('filename')
if not filename:
return self.redirect(self.get_index_url())
path = self.download_results_path(user_uuid, filename)
return self.file_response(path)
def download_results_thread(self, results, fmt, fields, user_uuid, progress):
"""
Thread target, which invokes :meth:`download_results_generate()` to
officially generate the data file which is then to be downloaded.
"""
route_prefix = self.get_route_prefix()
session = self.make_isolated_session()
try:
# create folder(s) for output; make sure file doesn't exist
filename = self.download_results_filename(fmt)
path = self.download_results_path(user_uuid, filename, makedirs=True)
if os.path.exists(path):
os.remove(path)
# generate file for download
results = results.with_session(session).all()
self.download_results_setup(fields, progress=progress)
self.download_results_generate(session, results, path, fmt, fields,
progress=progress)
session.commit()
except Exception as error:
msg = "failed to generate results file for download!"
log.warning(msg, exc_info=True)
session.rollback()
if progress:
progress.session.load()
progress.session['error'] = True
progress.session['error_msg'] = "{}: {}".format(
msg, simple_error(error))
progress.session.save()
return
finally:
session.close()
if progress:
progress.session.load()
progress.session['complete'] = True
progress.session['success_url'] = self.get_index_url()
progress.session['extra_session_bits'] = {
'{}.results.generated'.format(route_prefix): path,
}
progress.session.save()
def download_results_setup(self, fields, progress=None):
"""
Perform any up-front caching or other setup required, just prior to
generating a new results data file for download.
"""
def download_results_generate(self, session, results, path, fmt, fields, progress=None):
"""
This method is responsible for actually generating the data file for a
"download results" operation, according to the given params.
"""
if fmt == 'csv':
if six.PY2:
csv_file = open(path, 'wb')
writer = UnicodeDictWriter(csv_file, fields, encoding='utf_8')
else: # PY3
csv_file = open(path, 'wt', encoding='utf_8')
writer = csv.DictWriter(csv_file, fields)
writer.writeheader()
def write(obj, i):
data = self.download_results_normalize(obj, fields, fmt=fmt)
row = self.download_results_coerce_csv(data, fields)
writer.writerow(row)
self.progress_loop(write, results, progress,
message="Writing data to CSV file")
csv_file.close()
elif fmt == 'xlsx':
writer = ExcelWriter(path, fields,
sheet_title=self.get_model_title_plural())
writer.write_header()
xlrows = []
def write(obj, i):
data = self.download_results_normalize(obj, fields, fmt=fmt)
row = self.download_results_coerce_xlsx(data, fields)
xlrow = [row[field] for field in fields]
xlrows.append(xlrow)
self.progress_loop(write, results, progress,
message="Collecting data for Excel")
def finalize(x, i):
writer.write_rows(xlrows)
writer.auto_freeze()
writer.auto_filter()
writer.auto_resize()
writer.save()
self.progress_loop(finalize, [1], progress,
message="Writing Excel file to disk")
def download_results_fields_available(self, **kwargs):
"""
Return the list of fields which are *available* to be written to
download file. Default field list will be constructed from the
underlying table columns.
"""
fields = []
mapper = orm.class_mapper(self.model_class)
for prop in mapper.iterate_properties:
if isinstance(prop, orm.ColumnProperty):
fields.append(prop.key)
return fields
def download_results_fields_default(self, fields, **kwargs):
"""
Return the default list of fields to be written to download file.
Unless you override, all "available" fields will be included by
default.
"""
return fields
def download_results_normalize(self, obj, fields, **kwargs):
"""
Normalize the given object into a data dict, for use when writing to
the results file for download.
"""
data = {}
for field in fields:
value = getattr(obj, field, None)
# make timestamps zone-aware
if isinstance(value, datetime.datetime):
value = localtime(self.rattail_config, value,
from_utc=not self.has_local_times)
data[field] = value
return data
def download_results_coerce_csv(self, data, fields, **kwargs):
"""
Coerce the given data dict record, to a "row" dict suitable for use
when writing directly to CSV file. Each value in the dict should be a
string type.
"""
csvrow = dict(data)
for field in fields:
value = csvrow.get(field)
if value is None:
value = ''
else:
value = six.text_type(value)
csvrow[field] = value
return csvrow
def download_results_coerce_xlsx(self, data, fields, **kwargs):
"""
Coerce the given data dict record, to a "row" dict suitable for use
when writing directly to XLSX file.
"""
return data
def results_csv(self):
"""
Download current list results as CSV.
@ -2697,7 +2984,7 @@ class MasterView(View):
})
def results_csv_session(self):
return RattailSession()
return self.make_isolated_session()
def results_csv_thread(self, results, user_uuid, progress):
"""
@ -2793,7 +3080,7 @@ class MasterView(View):
})
def results_xlsx_session(self):
return RattailSession()
return self.make_isolated_session()
def results_xlsx_thread(self, results, user_uuid, progress):
"""
@ -3807,6 +4094,20 @@ class MasterView(View):
config.add_view(cls, attr='mobile_index', route_name='mobile.{}'.format(route_prefix),
permission='{}.list'.format(permission_prefix))
# download results
# this is the "new" more flexible approach, but we only want to
# enable it if the class declares it, *and* does *not* declare the
# older style(s). that way each class must explicitly choose
# *only* the new style in order to use it
if cls.results_downloadable and not (
cls.results_downloadable_csv or cls.results_downloadable_xlsx):
config.add_tailbone_permission(permission_prefix, '{}.download_results'.format(permission_prefix),
"Download search results for {}".format(model_title_plural))
config.add_route('{}.download_results'.format(route_prefix), '{}/download-results'.format(url_prefix))
config.add_view(cls, attr='download_results', route_name='{}.download_results'.format(route_prefix),
permission='{}.download_results'.format(permission_prefix))
# download results as CSV (deprecated)
if cls.results_downloadable_csv:
config.add_tailbone_permission(permission_prefix, '{}.results_csv'.format(permission_prefix),
"Download {} as CSV".format(model_title_plural))
@ -3817,6 +4118,7 @@ class MasterView(View):
config.add_view(cls, attr='results_csv_download', route_name='{}.results_csv_download'.format(route_prefix),
permission='{}.results_csv'.format(permission_prefix))
# download results as XLSX (deprecated)
if cls.results_downloadable_xlsx:
config.add_tailbone_permission(permission_prefix, '{}.results_xlsx'.format(permission_prefix),
"Download {} as XLSX".format(model_title_plural))