Add feature to "download rows for results" in master index view
This commit is contained in:
Lance Edgar
parent
bcb4bda7e6
commit
e0d1e39824
2 changed files with 404 additions and 2 deletions
|
|
@ -84,6 +84,7 @@ class MasterView(View):
|
|||
results_downloadable = False
|
||||
results_downloadable_csv = False
|
||||
results_downloadable_xlsx = False
|
||||
results_rows_downloadable = False
|
||||
creatable = True
|
||||
show_create_link = True
|
||||
viewable = True
|
||||
|
|
@ -356,6 +357,14 @@ class MasterView(View):
|
|||
context['download_results_fields_available'] = available
|
||||
context['download_results_fields_default'] = self.download_results_fields_default(available)
|
||||
|
||||
if self.has_rows and self.results_rows_downloadable and self.has_perm('download_results_rows'):
|
||||
route_prefix = self.get_route_prefix()
|
||||
context['download_results_rows_path'] = self.request.session.pop(
|
||||
'{}.results_rows.generated'.format(route_prefix), None)
|
||||
available = self.download_results_fields_available()
|
||||
context['download_results_rows_fields_available'] = available
|
||||
context['download_results_rows_fields_default'] = self.download_results_rows_fields_default(available)
|
||||
|
||||
return self.render_to_response('index', context)
|
||||
|
||||
def make_grid(self, factory=None, key=None, data=None, columns=None, **kwargs):
|
||||
|
|
@ -2877,8 +2886,8 @@ class MasterView(View):
|
|||
|
||||
def write(obj, i):
|
||||
data = self.download_results_normalize(obj, fields, fmt=fmt)
|
||||
row = self.download_results_coerce_csv(data, fields)
|
||||
writer.writerow(row)
|
||||
csvrow = self.download_results_coerce_csv(data, fields)
|
||||
writer.writerow(csvrow)
|
||||
|
||||
self.progress_loop(write, results, progress,
|
||||
message="Writing data to CSV file")
|
||||
|
|
@ -3201,6 +3210,297 @@ class MasterView(View):
|
|||
row[field] = getattr(obj, field, None)
|
||||
return row
|
||||
|
||||
def download_results_rows_supported_formats(self):
|
||||
# TODO: default formats should be configurable?
|
||||
return OrderedDict([
|
||||
('xlsx', "Excel (XLSX)"),
|
||||
('csv', "CSV"),
|
||||
])
|
||||
|
||||
def download_results_rows_default_format(self):
|
||||
# TODO: default format should be configurable
|
||||
return 'xlsx'
|
||||
|
||||
def download_results_rows(self):
|
||||
"""
|
||||
View for saving *rows* of current (filtered) data results into a file,
|
||||
and downloading that file.
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
user_uuid = self.request.user.uuid
|
||||
|
||||
# POST means generate a new results file for download
|
||||
if self.request.method == 'POST':
|
||||
|
||||
# make sure a valid format was requested
|
||||
supported = self.download_results_rows_supported_formats()
|
||||
if not supported:
|
||||
self.request.session.flash("There are no supported download formats!",
|
||||
'error')
|
||||
return self.redirect(self.get_index_url())
|
||||
fmt = self.request.POST.get('fmt')
|
||||
if not fmt:
|
||||
fmt = self.download_results_rows_default_format() or list(supported)[0]
|
||||
if fmt not in supported:
|
||||
self.request.session.flash("Unsupported download format: {}".format(fmt),
|
||||
'error')
|
||||
return self.redirect(self.get_index_url())
|
||||
|
||||
# parse field list if one was given
|
||||
fields = self.request.POST.get('fields')
|
||||
if fields:
|
||||
fields = fields.split(',')
|
||||
if not fields:
|
||||
if fmt == 'csv':
|
||||
fields = self.get_row_csv_fields()
|
||||
elif fmt == 'xlsx':
|
||||
fields = self.get_row_xlsx_fields()
|
||||
else:
|
||||
self.request.session.flash("No fields were specified", 'error')
|
||||
return self.redirect(self.get_index_url())
|
||||
|
||||
# start thread to actually do work / report progress
|
||||
key = '{}.download_results_rows'.format(route_prefix)
|
||||
progress = self.make_progress(key)
|
||||
results = self.get_effective_data()
|
||||
thread = Thread(target=self.download_results_rows_thread,
|
||||
args=(results, fmt, fields, user_uuid, progress))
|
||||
thread.start()
|
||||
|
||||
# show user the progress page
|
||||
return self.render_progress(progress, {
|
||||
'cancel_url': self.get_index_url(),
|
||||
'cancel_msg': "Download was canceled.",
|
||||
})
|
||||
|
||||
# not POST, so just download a file (if specified)
|
||||
filename = self.request.GET.get('filename')
|
||||
if not filename:
|
||||
return self.redirect(self.get_index_url())
|
||||
path = self.download_results_rows_path(user_uuid, filename)
|
||||
return self.file_response(path)
|
||||
|
||||
def download_results_rows_filename(self, fmt):
|
||||
"""
|
||||
Must return an appropriate "download results" filename for the given
|
||||
format. E.g. ``'products.csv'``
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
if fmt == 'csv':
|
||||
return '{}.rows.csv'.format(route_prefix)
|
||||
if fmt == 'xlsx':
|
||||
return '{}.rows.xlsx'.format(route_prefix)
|
||||
|
||||
def download_results_rows_path(self, user_uuid, filename=None,
|
||||
typ='results', makedirs=False):
|
||||
"""
|
||||
Returns an absolute path for the "results" data file, specific to the
|
||||
given user UUID.
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
path = os.path.join(self.rattail_config.datadir(), 'downloads',
|
||||
typ, route_prefix,
|
||||
user_uuid[:2], user_uuid[2:])
|
||||
if makedirs and not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
if filename:
|
||||
path = os.path.join(path, filename)
|
||||
return path
|
||||
|
||||
def download_results_rows_fields_available(self, **kwargs):
|
||||
"""
|
||||
Return the list of fields which are *available* to be written to
|
||||
download file. Default field list will be constructed from the
|
||||
underlying table columns.
|
||||
"""
|
||||
fields = []
|
||||
mapper = orm.class_mapper(self.model_class)
|
||||
for prop in mapper.iterate_properties:
|
||||
if isinstance(prop, orm.ColumnProperty):
|
||||
fields.append(prop.key)
|
||||
return fields
|
||||
|
||||
def download_results_rows_fields_default(self, fields, **kwargs):
|
||||
"""
|
||||
Return the default list of fields to be written to download file.
|
||||
Unless you override, all "available" fields will be included by
|
||||
default.
|
||||
"""
|
||||
return fields
|
||||
|
||||
def download_results_rows_thread(self, results, fmt, fields, user_uuid, progress):
|
||||
"""
|
||||
Thread target, which invokes :meth:`download_results_generate()` to
|
||||
officially generate the data file which is then to be downloaded.
|
||||
"""
|
||||
route_prefix = self.get_route_prefix()
|
||||
session = self.make_isolated_session()
|
||||
try:
|
||||
|
||||
# create folder(s) for output; make sure file doesn't exist
|
||||
filename = self.download_results_rows_filename(fmt)
|
||||
path = self.download_results_rows_path(user_uuid, filename, makedirs=True)
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
# generate file for download
|
||||
results = results.with_session(session).all()
|
||||
self.download_results_rows_setup(fields, progress=progress)
|
||||
self.download_results_rows_generate(session, results, path, fmt, fields,
|
||||
progress=progress)
|
||||
|
||||
session.commit()
|
||||
|
||||
except Exception as error:
|
||||
msg = "failed to generate results file for download!"
|
||||
log.warning(msg, exc_info=True)
|
||||
session.rollback()
|
||||
if progress:
|
||||
progress.session.load()
|
||||
progress.session['error'] = True
|
||||
progress.session['error_msg'] = "{}: {}".format(
|
||||
msg, simple_error(error))
|
||||
progress.session.save()
|
||||
return
|
||||
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if progress:
|
||||
progress.session.load()
|
||||
progress.session['complete'] = True
|
||||
progress.session['success_url'] = self.get_index_url()
|
||||
progress.session['extra_session_bits'] = {
|
||||
'{}.results_rows.generated'.format(route_prefix): path,
|
||||
}
|
||||
progress.session.save()
|
||||
|
||||
def download_results_rows_setup(self, fields, progress=None):
|
||||
"""
|
||||
Perform any up-front caching or other setup required, just prior to
|
||||
generating a new results data file for download.
|
||||
"""
|
||||
|
||||
def download_results_rows_generate(self, session, results, path, fmt, fields, progress=None):
|
||||
"""
|
||||
This method is responsible for actually generating the data file for a
|
||||
"download rows for results" operation, according to the given params.
|
||||
"""
|
||||
# we really are concerned with "rows of results" here, so let's just
|
||||
# replace the 'results' list with a list of rows
|
||||
original_results = results
|
||||
results = []
|
||||
|
||||
def collect(obj, i):
|
||||
results.extend(self.get_row_data(obj).all())
|
||||
|
||||
self.progress_loop(collect, original_results, progress,
|
||||
message="Collecting data for {}".format(self.get_row_model_title_plural()))
|
||||
|
||||
if fmt == 'csv':
|
||||
|
||||
if six.PY2:
|
||||
csv_file = open(path, 'wb')
|
||||
writer = UnicodeDictWriter(csv_file, fields, encoding='utf_8')
|
||||
else: # PY3
|
||||
csv_file = open(path, 'wt', encoding='utf_8')
|
||||
writer = csv.DictWriter(csv_file, fields)
|
||||
writer.writeheader()
|
||||
|
||||
def write(obj, i):
|
||||
data = self.download_results_rows_normalize(obj, fields, fmt=fmt)
|
||||
csvrow = self.download_results_rows_coerce_csv(data, fields)
|
||||
writer.writerow(csvrow)
|
||||
|
||||
self.progress_loop(write, results, progress,
|
||||
message="Writing data to CSV file")
|
||||
csv_file.close()
|
||||
|
||||
elif fmt == 'xlsx':
|
||||
|
||||
writer = ExcelWriter(path, fields,
|
||||
sheet_title=self.get_row_model_title_plural())
|
||||
writer.write_header()
|
||||
|
||||
xlrows = []
|
||||
def write(obj, i):
|
||||
data = self.download_results_rows_normalize(obj, fields, fmt=fmt)
|
||||
row = self.download_results_rows_coerce_xlsx(data, fields)
|
||||
xlrow = [row[field] for field in fields]
|
||||
xlrows.append(xlrow)
|
||||
|
||||
self.progress_loop(write, results, progress,
|
||||
message="Collecting data for Excel")
|
||||
|
||||
def finalize(x, i):
|
||||
writer.write_rows(xlrows)
|
||||
writer.auto_freeze()
|
||||
writer.auto_filter()
|
||||
writer.auto_resize()
|
||||
writer.save()
|
||||
|
||||
self.progress_loop(finalize, [1], progress,
|
||||
message="Writing Excel file to disk")
|
||||
|
||||
def download_results_rows_normalize(self, row, fields, **kwargs):
|
||||
"""
|
||||
Normalize the given row object into a data dict, for use when writing
|
||||
to the results file for download.
|
||||
"""
|
||||
data = {}
|
||||
for field in fields:
|
||||
value = getattr(row, field, None)
|
||||
|
||||
# make timestamps zone-aware
|
||||
if isinstance(value, datetime.datetime):
|
||||
value = localtime(self.rattail_config, value,
|
||||
from_utc=not self.has_local_times)
|
||||
|
||||
data[field] = value
|
||||
|
||||
return data
|
||||
|
||||
def download_results_rows_coerce_csv(self, data, fields, **kwargs):
|
||||
"""
|
||||
Coerce the given data dict record, to a "row" dict suitable for use
|
||||
when writing directly to CSV file. Each value in the dict should be a
|
||||
string type.
|
||||
"""
|
||||
csvrow = dict(data)
|
||||
for field in fields:
|
||||
value = csvrow.get(field)
|
||||
|
||||
if value is None:
|
||||
value = ''
|
||||
else:
|
||||
value = six.text_type(value)
|
||||
|
||||
csvrow[field] = value
|
||||
|
||||
return csvrow
|
||||
|
||||
def download_results_rows_coerce_xlsx(self, data, fields, **kwargs):
|
||||
"""
|
||||
Coerce the given data dict record, to a "row" dict suitable for use
|
||||
when writing directly to XLSX file.
|
||||
"""
|
||||
data = dict(data)
|
||||
for key in data:
|
||||
value = data[key]
|
||||
|
||||
# convert GPC to pretty string
|
||||
if isinstance(value, GPC):
|
||||
value = value.pretty()
|
||||
|
||||
# make timestamps local, "zone-naive"
|
||||
elif isinstance(value, datetime.datetime):
|
||||
value = localtime(self.rattail_config, value, tzinfo=False)
|
||||
|
||||
data[key] = value
|
||||
|
||||
return data
|
||||
|
||||
def row_results_xlsx(self):
|
||||
"""
|
||||
Download current *row* results as XLSX.
|
||||
|
|
@ -4152,6 +4452,14 @@ class MasterView(View):
|
|||
config.add_view(cls, attr='results_xlsx_download', route_name='{}.results_xlsx_download'.format(route_prefix),
|
||||
permission='{}.results_xlsx'.format(permission_prefix))
|
||||
|
||||
# download rows for results
|
||||
if cls.has_rows and cls.results_rows_downloadable:
|
||||
config.add_tailbone_permission(permission_prefix, '{}.download_results_rows'.format(permission_prefix),
|
||||
"Download *rows* for {} search results".format(model_title))
|
||||
config.add_route('{}.download_results_rows'.format(route_prefix), '{}/download-rows-for-results'.format(url_prefix))
|
||||
config.add_view(cls, attr='download_results_rows', route_name='{}.download_results_rows'.format(route_prefix),
|
||||
permission='{}.download_results_rows'.format(permission_prefix))
|
||||
|
||||
# quickie (search)
|
||||
if cls.supports_quickie_search:
|
||||
config.add_tailbone_permission(permission_prefix, '{}.quickie'.format(permission_prefix),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue