Add common configuration logic for "input file templates"
just used in one batch so far but should be useful for many more..once can get around to migrating them had to rework the configuration logic to use HTML form instead of JSON, to allow for the file uploads
This commit is contained in:
Lance Edgar
parent
099b6915f4
commit
30f95e2f08
13 changed files with 405 additions and 95 deletions
|
|
@ -83,6 +83,8 @@ class BatchMasterView(MasterView):
|
|||
has_worksheet = False
|
||||
has_worksheet_file = False
|
||||
|
||||
input_file_template_config_section = 'rattail.batch'
|
||||
|
||||
grid_columns = [
|
||||
'id',
|
||||
'description',
|
||||
|
|
@ -157,6 +159,10 @@ class BatchMasterView(MasterView):
|
|||
factory = self.get_handler_factory(self.rattail_config)
|
||||
return factory(self.rattail_config)
|
||||
|
||||
@property
|
||||
def input_file_template_config_prefix(self):
|
||||
return '{}.input_file_template'.format(self.batch_handler.batch_key)
|
||||
|
||||
def download_path(self, batch, filename):
|
||||
return self.rattail_config.batch_filepath(batch.batch_key, batch.uuid, filename)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ DataSync Views
|
|||
from __future__ import unicode_literals, absolute_import
|
||||
|
||||
import getpass
|
||||
import json
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
|
|
@ -132,7 +133,7 @@ class DataSyncThreadView(MasterView):
|
|||
settings = []
|
||||
watch = []
|
||||
|
||||
for profile in data['profiles']:
|
||||
for profile in json.loads(data['profiles']):
|
||||
pkey = profile['key']
|
||||
if profile['enabled']:
|
||||
watch.append(pkey)
|
||||
|
|
@ -181,12 +182,12 @@ class DataSyncThreadView(MasterView):
|
|||
'value': ', '.join(consumers)},
|
||||
])
|
||||
|
||||
settings.extend([
|
||||
{'name': 'rattail.datasync.watch',
|
||||
'value': ', '.join(watch)},
|
||||
{'name': 'tailbone.datasync.restart',
|
||||
'value': data['restart_command']},
|
||||
])
|
||||
if watch:
|
||||
settings.append({'name': 'rattail.datasync.watch',
|
||||
'value': ', '.join(watch)})
|
||||
|
||||
settings.append({'name': 'tailbone.datasync.restart',
|
||||
'value': data['restart_command']})
|
||||
|
||||
return settings
|
||||
|
||||
|
|
|
|||
|
|
@ -560,7 +560,7 @@ cd {prefix}
|
|||
def configure_gather_settings(self, data):
|
||||
settings = []
|
||||
|
||||
for handler in data['handlers']:
|
||||
for handler in json.loads(data['handlers']):
|
||||
key = handler['key']
|
||||
|
||||
settings.extend([
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from __future__ import unicode_literals, absolute_import
|
|||
import os
|
||||
import csv
|
||||
import datetime
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
|
||||
|
|
@ -36,11 +37,9 @@ import json
|
|||
import six
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import orm
|
||||
|
||||
import sqlalchemy_continuum as continuum
|
||||
from sqlalchemy_utils.functions import get_primary_keys, get_columns
|
||||
|
||||
|
||||
from rattail.db import model, Session as RattailSession
|
||||
from rattail.db.continuum import model_transaction_query
|
||||
from rattail.util import prettify, OrderedDict, simple_error
|
||||
|
|
@ -57,6 +56,7 @@ from pyramid import httpexceptions
|
|||
from pyramid.renderers import get_renderer, render_to_response, render
|
||||
from pyramid.response import FileResponse
|
||||
from webhelpers2.html import HTML, tags
|
||||
from webob.compat import cgi_FieldStorage
|
||||
|
||||
from tailbone import forms, grids, diffs
|
||||
from tailbone.views import View
|
||||
|
|
@ -114,6 +114,7 @@ class MasterView(View):
|
|||
execute_progress_initial_msg = None
|
||||
supports_prev_next = False
|
||||
supports_import_batch_from_file = False
|
||||
has_input_file_templates = False
|
||||
configurable = False
|
||||
|
||||
# set to True to add "View *global* Objects" permission, and
|
||||
|
|
@ -1467,6 +1468,26 @@ class MasterView(View):
|
|||
Return a content type for a file download, if known.
|
||||
"""
|
||||
|
||||
def download_input_file_template(self):
|
||||
"""
|
||||
View for downloading an input file template.
|
||||
"""
|
||||
key = self.request.GET['key']
|
||||
filespec = self.request.GET['file']
|
||||
|
||||
matches = [tmpl for tmpl in self.get_input_file_templates()
|
||||
if tmpl['key'] == key]
|
||||
if not matches:
|
||||
raise self.notfound()
|
||||
|
||||
template = matches[0]
|
||||
templatesdir = os.path.join(self.rattail_config.datadir(),
|
||||
'templates', 'input_files',
|
||||
self.get_route_prefix())
|
||||
basedir = os.path.join(templatesdir, template['key'])
|
||||
path = os.path.join(basedir, filespec)
|
||||
return self.file_response(path)
|
||||
|
||||
def edit(self):
|
||||
"""
|
||||
View for editing an existing model record.
|
||||
|
|
@ -2230,6 +2251,90 @@ class MasterView(View):
|
|||
kwargs['db_picker_options'] = [tags.Option(k) for k in engines]
|
||||
kwargs['db_picker_selected'] = selected
|
||||
|
||||
# add info for downloadable input file templates, if any
|
||||
if self.has_input_file_templates:
|
||||
templates = self.normalize_input_file_templates()
|
||||
kwargs['input_file_templates'] = OrderedDict([(tmpl['key'], tmpl)
|
||||
for tmpl in templates])
|
||||
|
||||
return kwargs
|
||||
|
||||
def get_input_file_templates(self):
|
||||
return []
|
||||
|
||||
def normalize_input_file_templates(self, templates=None,
|
||||
include_file_options=False):
|
||||
if templates is None:
|
||||
templates = self.get_input_file_templates()
|
||||
|
||||
route_prefix = self.get_route_prefix()
|
||||
|
||||
if include_file_options:
|
||||
templatesdir = os.path.join(self.rattail_config.datadir(),
|
||||
'templates', 'input_files',
|
||||
route_prefix)
|
||||
|
||||
for template in templates:
|
||||
|
||||
if 'config_section' not in template:
|
||||
template['config_section'] = self.input_file_template_config_section
|
||||
section = template['config_section']
|
||||
|
||||
if 'config_prefix' not in template:
|
||||
template['config_prefix'] = '{}.{}'.format(
|
||||
self.input_file_template_config_prefix,
|
||||
template['key'])
|
||||
prefix = template['config_prefix']
|
||||
|
||||
for key in ('mode', 'file', 'url'):
|
||||
|
||||
if 'option_{}'.format(key) not in template:
|
||||
template['option_{}'.format(key)] = '{}.{}'.format(prefix, key)
|
||||
|
||||
if 'setting_{}'.format(key) not in template:
|
||||
template['setting_{}'.format(key)] = '{}.{}'.format(
|
||||
section,
|
||||
template['option_{}'.format(key)])
|
||||
|
||||
if key not in template:
|
||||
value = self.rattail_config.get(
|
||||
section,
|
||||
template['option_{}'.format(key)])
|
||||
if value is not None:
|
||||
template[key] = value
|
||||
|
||||
template.setdefault('mode', 'default')
|
||||
template.setdefault('file', None)
|
||||
template.setdefault('url', template['default_url'])
|
||||
|
||||
if include_file_options:
|
||||
options = []
|
||||
basedir = os.path.join(templatesdir, template['key'])
|
||||
if os.path.exists(basedir):
|
||||
for name in sorted(os.listdir(basedir)):
|
||||
if len(name) == 4 and name.isdigit():
|
||||
files = os.listdir(os.path.join(basedir, name))
|
||||
if len(files) == 1:
|
||||
options.append(os.path.join(name, files[0]))
|
||||
template['file_options'] = options
|
||||
template['file_options_dir'] = basedir
|
||||
|
||||
if template['mode'] == 'external':
|
||||
template['effective_url'] = template['url']
|
||||
elif template['mode'] == 'hosted':
|
||||
template['effective_url'] = self.request.route_url(
|
||||
'{}.download_input_file_template'.format(route_prefix),
|
||||
_query={'key': template['key'],
|
||||
'file': template['file']})
|
||||
else:
|
||||
template['effective_url'] = template['default_url']
|
||||
|
||||
return templates
|
||||
|
||||
def template_kwargs_index(self, **kwargs):
|
||||
"""
|
||||
Method stub, so subclass can always invoke super() for it.
|
||||
"""
|
||||
return kwargs
|
||||
|
||||
def template_kwargs_create(self, **kwargs):
|
||||
|
|
@ -4043,16 +4148,71 @@ class MasterView(View):
|
|||
self.request.session.flash("Settings have been removed.")
|
||||
return self.redirect(self.request.current_route_url())
|
||||
else:
|
||||
data = self.request.json_body
|
||||
data = self.request.POST
|
||||
|
||||
# collect any uploaded files
|
||||
uploads = {}
|
||||
for key, value in six.iteritems(data):
|
||||
if isinstance(value, cgi_FieldStorage):
|
||||
tempdir = tempfile.mkdtemp()
|
||||
filename = os.path.basename(value.filename)
|
||||
filepath = os.path.join(tempdir, filename)
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(value.file.read())
|
||||
uploads[key] = {
|
||||
'filedir': tempdir,
|
||||
'filename': filename,
|
||||
'filepath': filepath,
|
||||
}
|
||||
|
||||
# process any uploads first
|
||||
if uploads:
|
||||
self.configure_process_uploads(uploads, data)
|
||||
|
||||
# then gather/save settings
|
||||
settings = self.configure_gather_settings(data)
|
||||
self.configure_remove_settings()
|
||||
self.configure_save_settings(settings)
|
||||
self.request.session.flash("Settings have been saved.")
|
||||
return self.json_response({'success': True})
|
||||
return self.redirect(self.request.current_route_url())
|
||||
|
||||
context = self.configure_get_context()
|
||||
return self.render_to_response('configure', context)
|
||||
|
||||
def configure_process_uploads(self, uploads, data):
|
||||
if self.has_input_file_templates:
|
||||
templatesdir = os.path.join(self.rattail_config.datadir(),
|
||||
'templates', 'input_files',
|
||||
self.get_route_prefix())
|
||||
|
||||
def get_next_filedir(basedir):
|
||||
nextid = 1
|
||||
while True:
|
||||
path = os.path.join(basedir, '{:04d}'.format(nextid))
|
||||
if not os.path.exists(path):
|
||||
# this should fail if there happens to be a race
|
||||
# condition and someone else got to this id first
|
||||
os.mkdir(path)
|
||||
return path
|
||||
nextid += 1
|
||||
|
||||
for template in self.normalize_input_file_templates():
|
||||
key = '{}.upload'.format(template['setting_file'])
|
||||
if key in uploads:
|
||||
assert self.request.POST[template['setting_mode']] == 'hosted'
|
||||
assert not self.request.POST[template['setting_file']]
|
||||
info = uploads[key]
|
||||
basedir = os.path.join(templatesdir, template['key'])
|
||||
if not os.path.exists(basedir):
|
||||
os.makedirs(basedir)
|
||||
filedir = get_next_filedir(basedir)
|
||||
filepath = os.path.join(filedir, info['filename'])
|
||||
shutil.copyfile(info['filepath'], filepath)
|
||||
shutil.rmtree(info['filedir'])
|
||||
numdir = os.path.basename(filedir)
|
||||
data[template['setting_file']] = os.path.join(numdir,
|
||||
info['filename'])
|
||||
|
||||
def configure_get_simple_settings(self):
|
||||
"""
|
||||
If you have some "simple" settings, each of which basically
|
||||
|
|
@ -4120,22 +4280,34 @@ class MasterView(View):
|
|||
|
||||
context['simple_settings'] = settings
|
||||
|
||||
# add settings for downloadable input file templates, if any
|
||||
if self.has_input_file_templates:
|
||||
settings = {}
|
||||
file_options = {}
|
||||
file_option_dirs = {}
|
||||
for template in self.normalize_input_file_templates(
|
||||
include_file_options=True):
|
||||
settings[template['setting_mode']] = template['mode']
|
||||
settings[template['setting_file']] = template['file']
|
||||
settings[template['setting_url']] = template['url']
|
||||
file_options[template['key']] = template['file_options']
|
||||
file_option_dirs[template['key']] = template['file_options_dir']
|
||||
context['input_file_template_settings'] = settings
|
||||
context['input_file_options'] = file_options
|
||||
context['input_file_option_dirs'] = file_option_dirs
|
||||
|
||||
return context
|
||||
|
||||
def configure_gather_settings(self, data):
|
||||
settings = []
|
||||
|
||||
# maybe collect "simple" settings
|
||||
simple_settings = self.configure_get_simple_settings()
|
||||
if simple_settings and 'simple_settings' in data:
|
||||
|
||||
data_settings = data['simple_settings']
|
||||
if simple_settings:
|
||||
|
||||
for simple in simple_settings:
|
||||
name = self.configure_get_name_for_simple_setting(simple)
|
||||
value = None
|
||||
|
||||
if name in data_settings:
|
||||
value = data_settings[name]
|
||||
value = data.get(name)
|
||||
|
||||
if simple.get('type') is bool:
|
||||
value = six.text_type(bool(value)).lower()
|
||||
|
|
@ -4145,14 +4317,45 @@ class MasterView(View):
|
|||
settings.append({'name': name,
|
||||
'value': value})
|
||||
|
||||
# maybe also collect input file template settings
|
||||
if self.has_input_file_templates:
|
||||
for template in self.normalize_input_file_templates():
|
||||
|
||||
# mode
|
||||
settings.append({'name': template['setting_mode'],
|
||||
'value': data.get(template['setting_mode'])})
|
||||
|
||||
# file
|
||||
value = data.get(template['setting_file'])
|
||||
if value:
|
||||
# nb. avoid saving if empty, so can remain "null"
|
||||
settings.append({'name': template['setting_file'],
|
||||
'value': value})
|
||||
|
||||
# url
|
||||
settings.append({'name': template['setting_url'],
|
||||
'value': data.get(template['setting_url'])})
|
||||
|
||||
return settings
|
||||
|
||||
def configure_remove_settings(self):
|
||||
model = self.model
|
||||
names = []
|
||||
|
||||
simple_settings = self.configure_get_simple_settings()
|
||||
if simple_settings:
|
||||
model = self.model
|
||||
names = [self.configure_get_name_for_simple_setting(simple)
|
||||
for simple in simple_settings]
|
||||
names.extend([self.configure_get_name_for_simple_setting(simple)
|
||||
for simple in simple_settings])
|
||||
|
||||
if self.has_input_file_templates:
|
||||
for template in self.normalize_input_file_templates():
|
||||
names.extend([
|
||||
template['setting_mode'],
|
||||
template['setting_file'],
|
||||
template['setting_url'],
|
||||
])
|
||||
|
||||
if names:
|
||||
self.Session.query(model.Setting)\
|
||||
.filter(model.Setting.name.in_(names))\
|
||||
.delete(synchronize_session=False)
|
||||
|
|
@ -4365,6 +4568,14 @@ class MasterView(View):
|
|||
config.add_tailbone_permission(permission_prefix, '{}.merge'.format(permission_prefix),
|
||||
"Merge 2 {}".format(model_title_plural))
|
||||
|
||||
# download input file template
|
||||
if cls.has_input_file_templates and cls.creatable:
|
||||
config.add_route('{}.download_input_file_template'.format(route_prefix),
|
||||
'{}/download-input-file-template'.format(url_prefix))
|
||||
config.add_view(cls, attr='download_input_file_template',
|
||||
route_name='{}.download_input_file_template'.format(route_prefix),
|
||||
permission='{}.create'.format(permission_prefix))
|
||||
|
||||
# view
|
||||
if cls.viewable:
|
||||
config.add_tailbone_permission(permission_prefix, '{}.view'.format(permission_prefix),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue