Add new/flexible "download results" feature
This commit is contained in:
Lance Edgar
parent
43472c7eb6
commit
922cbe4451
3 changed files with 553 additions and 3 deletions
|
|
@ -76,8 +76,12 @@ class MasterView(View):
|
|||
# set to True in order to encode search values as utf-8
|
||||
use_byte_string_filters = False
|
||||
|
||||
# set to True if all timestamps are "local" instead of UTC
|
||||
has_local_times = False
|
||||
|
||||
listable = True
|
||||
sortable = True
|
||||
results_downloadable = False
|
||||
results_downloadable_csv = False
|
||||
results_downloadable_xlsx = False
|
||||
creatable = True
|
||||
|
|
@ -191,6 +195,22 @@ class MasterView(View):
|
|||
from tailbone.db import Session
|
||||
return Session
|
||||
|
||||
def make_isolated_session(self):
|
||||
"""
|
||||
This method should return a newly-created SQLAlchemy Session instance.
|
||||
The use case here is primarily for secondary threads, which may be
|
||||
employed for long-running processes such as executing a batch. The
|
||||
session returned should *not* have any web hooks to auto-commit with
|
||||
the request/response cycle etc. It should just be a plain old session,
|
||||
"isolated" from the rest of the web app in a sense.
|
||||
|
||||
So whereas ``self.Session`` by default will return a reference to
|
||||
``tailbone.db.Session``, which is a "scoped" session wrapper specific
|
||||
to the current thread (one per request), this method should instead
|
||||
return e.g. a new independent ``rattail.db.Session`` instance.
|
||||
"""
|
||||
return RattailSession()
|
||||
|
||||
@classmethod
|
||||
def get_grid_factory(cls):
|
||||
"""
|
||||
|
|
@ -327,6 +347,15 @@ class MasterView(View):
|
|||
context = {
|
||||
'grid': grid,
|
||||
}
|
||||
|
||||
if self.results_downloadable and self.has_perm('download_results'):
|
||||
route_prefix = self.get_route_prefix()
|
||||
context['download_results_path'] = self.request.session.pop(
|
||||
'{}.results.generated'.format(route_prefix), None)
|
||||
available = self.download_results_fields_available()
|
||||
context['download_results_fields_available'] = available
|
||||
context['download_results_fields_default'] = self.download_results_fields_default(available)
|
||||
|
||||
return self.render_to_response('index', context)
|
||||
|
||||
def make_grid(self, factory=None, key=None, data=None, columns=None, **kwargs):
|
||||
|
|
@ -1900,7 +1929,7 @@ class MasterView(View):
|
|||
message="Deleting objects")
|
||||
|
||||
def get_bulk_delete_session(self):
|
||||
return RattailSession()
|
||||
return self.make_isolated_session()
|
||||
|
||||
def bulk_delete_thread(self, objects, progress):
|
||||
"""
|
||||
|
|
@ -2676,6 +2705,264 @@ class MasterView(View):
|
|||
"""
|
||||
return False
|
||||
|
||||
def download_results_path(self, user_uuid, filename=None,
|
||||
typ='results', makedirs=False):
|
||||
"""
|
||||
Returns an absolute path for the "results" data file, specific to the
|
||||
given user UUID.
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
path = os.path.join(self.rattail_config.datadir(), 'downloads',
|
||||
typ, route_prefix,
|
||||
user_uuid[:2], user_uuid[2:])
|
||||
if makedirs and not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
if filename:
|
||||
path = os.path.join(path, filename)
|
||||
return path
|
||||
|
||||
def download_results_filename(self, fmt):
|
||||
"""
|
||||
Must return an appropriate "download results" filename for the given
|
||||
format. E.g. ``'products.csv'``
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
if fmt == 'csv':
|
||||
return '{}.csv'.format(route_prefix)
|
||||
if fmt == 'xlsx':
|
||||
return '{}.xlsx'.format(route_prefix)
|
||||
|
||||
def download_results_supported_formats(self):
|
||||
# TODO: default formats should be configurable?
|
||||
return OrderedDict([
|
||||
('xlsx', "Excel (XLSX)"),
|
||||
('csv', "CSV"),
|
||||
])
|
||||
|
||||
def download_results_default_format(self):
|
||||
# TODO: default format should be configurable
|
||||
return 'xlsx'
|
||||
|
||||
def download_results(self):
|
||||
"""
|
||||
View for saving current (filtered) data results into a file, and
|
||||
downloading that file.
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
user_uuid = self.request.user.uuid
|
||||
|
||||
# POST means generate a new results file for download
|
||||
if self.request.method == 'POST':
|
||||
|
||||
# make sure a valid format was requested
|
||||
supported = self.download_results_supported_formats()
|
||||
if not supported:
|
||||
self.request.session.flash("There are no supported download formats!",
|
||||
'error')
|
||||
return self.redirect(self.get_index_url())
|
||||
fmt = self.request.POST.get('fmt')
|
||||
if not fmt:
|
||||
fmt = self.download_results_default_format() or list(supported)[0]
|
||||
if fmt not in supported:
|
||||
self.request.session.flash("Unsupported download format: {}".format(fmt),
|
||||
'error')
|
||||
return self.redirect(self.get_index_url())
|
||||
|
||||
# parse field list if one was given
|
||||
fields = self.request.POST.get('fields')
|
||||
if fields:
|
||||
fields = fields.split(',')
|
||||
|
||||
# start thread to actually do work / report progress
|
||||
key = '{}.download_results'.format(route_prefix)
|
||||
progress = self.make_progress(key)
|
||||
results = self.get_effective_data()
|
||||
thread = Thread(target=self.download_results_thread,
|
||||
args=(results, fmt, fields, user_uuid, progress))
|
||||
thread.start()
|
||||
|
||||
# show user the progress page
|
||||
return self.render_progress(progress, {
|
||||
'cancel_url': self.get_index_url(),
|
||||
'cancel_msg': "Download was canceled.",
|
||||
})
|
||||
|
||||
# not POST, so just download a file (if specified)
|
||||
filename = self.request.GET.get('filename')
|
||||
if not filename:
|
||||
return self.redirect(self.get_index_url())
|
||||
path = self.download_results_path(user_uuid, filename)
|
||||
return self.file_response(path)
|
||||
|
||||
def download_results_thread(self, results, fmt, fields, user_uuid, progress):
|
||||
"""
|
||||
Thread target, which invokes :meth:`download_results_generate()` to
|
||||
officially generate the data file which is then to be downloaded.
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
session = self.make_isolated_session()
|
||||
try:
|
||||
|
||||
# create folder(s) for output; make sure file doesn't exist
|
||||
filename = self.download_results_filename(fmt)
|
||||
path = self.download_results_path(user_uuid, filename, makedirs=True)
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
# generate file for download
|
||||
results = results.with_session(session).all()
|
||||
self.download_results_setup(fields, progress=progress)
|
||||
self.download_results_generate(session, results, path, fmt, fields,
|
||||
progress=progress)
|
||||
|
||||
session.commit()
|
||||
|
||||
except Exception as error:
|
||||
msg = "failed to generate results file for download!"
|
||||
log.warning(msg, exc_info=True)
|
||||
session.rollback()
|
||||
if progress:
|
||||
progress.session.load()
|
||||
progress.session['error'] = True
|
||||
progress.session['error_msg'] = "{}: {}".format(
|
||||
msg, simple_error(error))
|
||||
progress.session.save()
|
||||
return
|
||||
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if progress:
|
||||
progress.session.load()
|
||||
progress.session['complete'] = True
|
||||
progress.session['success_url'] = self.get_index_url()
|
||||
progress.session['extra_session_bits'] = {
|
||||
'{}.results.generated'.format(route_prefix): path,
|
||||
}
|
||||
progress.session.save()
|
||||
|
||||
def download_results_setup(self, fields, progress=None):
|
||||
"""
|
||||
Perform any up-front caching or other setup required, just prior to
|
||||
generating a new results data file for download.
|
||||
"""
|
||||
|
||||
def download_results_generate(self, session, results, path, fmt, fields, progress=None):
|
||||
"""
|
||||
This method is responsible for actually generating the data file for a
|
||||
"download results" operation, according to the given params.
|
||||
"""
|
||||
if fmt == 'csv':
|
||||
|
||||
if six.PY2:
|
||||
csv_file = open(path, 'wb')
|
||||
writer = UnicodeDictWriter(csv_file, fields, encoding='utf_8')
|
||||
else: # PY3
|
||||
csv_file = open(path, 'wt', encoding='utf_8')
|
||||
writer = csv.DictWriter(csv_file, fields)
|
||||
writer.writeheader()
|
||||
|
||||
def write(obj, i):
|
||||
data = self.download_results_normalize(obj, fields, fmt=fmt)
|
||||
row = self.download_results_coerce_csv(data, fields)
|
||||
writer.writerow(row)
|
||||
|
||||
self.progress_loop(write, results, progress,
|
||||
message="Writing data to CSV file")
|
||||
csv_file.close()
|
||||
|
||||
elif fmt == 'xlsx':
|
||||
|
||||
writer = ExcelWriter(path, fields,
|
||||
sheet_title=self.get_model_title_plural())
|
||||
writer.write_header()
|
||||
|
||||
xlrows = []
|
||||
def write(obj, i):
|
||||
data = self.download_results_normalize(obj, fields, fmt=fmt)
|
||||
row = self.download_results_coerce_xlsx(data, fields)
|
||||
xlrow = [row[field] for field in fields]
|
||||
xlrows.append(xlrow)
|
||||
|
||||
self.progress_loop(write, results, progress,
|
||||
message="Collecting data for Excel")
|
||||
|
||||
def finalize(x, i):
|
||||
writer.write_rows(xlrows)
|
||||
writer.auto_freeze()
|
||||
writer.auto_filter()
|
||||
writer.auto_resize()
|
||||
writer.save()
|
||||
|
||||
self.progress_loop(finalize, [1], progress,
|
||||
message="Writing Excel file to disk")
|
||||
|
||||
def download_results_fields_available(self, **kwargs):
|
||||
"""
|
||||
Return the list of fields which are *available* to be written to
|
||||
download file. Default field list will be constructed from the
|
||||
underlying table columns.
|
||||
"""
|
||||
fields = []
|
||||
mapper = orm.class_mapper(self.model_class)
|
||||
for prop in mapper.iterate_properties:
|
||||
if isinstance(prop, orm.ColumnProperty):
|
||||
fields.append(prop.key)
|
||||
return fields
|
||||
|
||||
def download_results_fields_default(self, fields, **kwargs):
|
||||
"""
|
||||
Return the default list of fields to be written to download file.
|
||||
Unless you override, all "available" fields will be included by
|
||||
default.
|
||||
"""
|
||||
return fields
|
||||
|
||||
def download_results_normalize(self, obj, fields, **kwargs):
|
||||
"""
|
||||
Normalize the given object into a data dict, for use when writing to
|
||||
the results file for download.
|
||||
"""
|
||||
data = {}
|
||||
for field in fields:
|
||||
value = getattr(obj, field, None)
|
||||
|
||||
# make timestamps zone-aware
|
||||
if isinstance(value, datetime.datetime):
|
||||
value = localtime(self.rattail_config, value,
|
||||
from_utc=not self.has_local_times)
|
||||
|
||||
data[field] = value
|
||||
|
||||
return data
|
||||
|
||||
def download_results_coerce_csv(self, data, fields, **kwargs):
|
||||
"""
|
||||
Coerce the given data dict record, to a "row" dict suitable for use
|
||||
when writing directly to CSV file. Each value in the dict should be a
|
||||
string type.
|
||||
"""
|
||||
csvrow = dict(data)
|
||||
for field in fields:
|
||||
value = csvrow.get(field)
|
||||
|
||||
if value is None:
|
||||
value = ''
|
||||
else:
|
||||
value = six.text_type(value)
|
||||
|
||||
csvrow[field] = value
|
||||
|
||||
return csvrow
|
||||
|
||||
def download_results_coerce_xlsx(self, data, fields, **kwargs):
|
||||
"""
|
||||
Coerce the given data dict record, to a "row" dict suitable for use
|
||||
when writing directly to XLSX file.
|
||||
"""
|
||||
return data
|
||||
|
||||
def results_csv(self):
|
||||
"""
|
||||
Download current list results as CSV.
|
||||
|
|
@ -2697,7 +2984,7 @@ class MasterView(View):
|
|||
})
|
||||
|
||||
def results_csv_session(self):
|
||||
return RattailSession()
|
||||
return self.make_isolated_session()
|
||||
|
||||
def results_csv_thread(self, results, user_uuid, progress):
|
||||
"""
|
||||
|
|
@ -2793,7 +3080,7 @@ class MasterView(View):
|
|||
})
|
||||
|
||||
def results_xlsx_session(self):
|
||||
return RattailSession()
|
||||
return self.make_isolated_session()
|
||||
|
||||
def results_xlsx_thread(self, results, user_uuid, progress):
|
||||
"""
|
||||
|
|
@ -3807,6 +4094,20 @@ class MasterView(View):
|
|||
config.add_view(cls, attr='mobile_index', route_name='mobile.{}'.format(route_prefix),
|
||||
permission='{}.list'.format(permission_prefix))
|
||||
|
||||
# download results
|
||||
# this is the "new" more flexible approach, but we only want to
|
||||
# enable it if the class declares it, *and* does *not* declare the
|
||||
# older style(s). that way each class must explicitly choose
|
||||
# *only* the new style in order to use it
|
||||
if cls.results_downloadable and not (
|
||||
cls.results_downloadable_csv or cls.results_downloadable_xlsx):
|
||||
config.add_tailbone_permission(permission_prefix, '{}.download_results'.format(permission_prefix),
|
||||
"Download search results for {}".format(model_title_plural))
|
||||
config.add_route('{}.download_results'.format(route_prefix), '{}/download-results'.format(url_prefix))
|
||||
config.add_view(cls, attr='download_results', route_name='{}.download_results'.format(route_prefix),
|
||||
permission='{}.download_results'.format(permission_prefix))
|
||||
|
||||
# download results as CSV (deprecated)
|
||||
if cls.results_downloadable_csv:
|
||||
config.add_tailbone_permission(permission_prefix, '{}.results_csv'.format(permission_prefix),
|
||||
"Download {} as CSV".format(model_title_plural))
|
||||
|
|
@ -3817,6 +4118,7 @@ class MasterView(View):
|
|||
config.add_view(cls, attr='results_csv_download', route_name='{}.results_csv_download'.format(route_prefix),
|
||||
permission='{}.results_csv'.format(permission_prefix))
|
||||
|
||||
# download results as XLSX (deprecated)
|
||||
if cls.results_downloadable_xlsx:
|
||||
config.add_tailbone_permission(permission_prefix, '{}.results_xlsx'.format(permission_prefix),
|
||||
"Download {} as XLSX".format(model_title_plural))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue