Improve data file handling for file batches.

Leverages a FormAlchemy "extension" of sorts.

tailbone/forms/renderers/batch.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2015 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU Affero General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option)
+#  any later version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Batch Field Renderers
+"""
+
+from __future__ import unicode_literals
+
+import os
+import stat
+import random
+
+from formalchemy.ext import fsblob
+
+
+class FileFieldRenderer(fsblob.FileFieldRenderer):
+    """
+    Custom file field renderer for batches based on a single source data file.
+    In edit mode, shows a file upload field.  In readonly mode, shows the
+    filename and its size.
+    """
+
+    @classmethod
+    def new(cls, view):
+        name = 'Configured%s_%s' % (cls.__name__, str(random.random())[2:])
+        return type(str(name), (cls,), dict(view=view))
+
+    @property
+    def storage_path(self):
+        return self.view.upload_dir
+
+    def get_size(self):
+        size = super(FileFieldRenderer, self).get_size()
+        if size:
+            return size
+        batch = self.field.parent.model
+        path = os.path.join(self.view.handler.datadir(batch), self.field.value)
+        if os.path.isfile(path):
+            return os.stat(path)[stat.ST_SIZE]
+        return 0
+
+    def get_url(self, filename):
+        batch = self.field.parent.model
+        return self.view.request.route_url('{0}.download'.format(self.view.route_prefix), uuid=batch.uuid)

tailbone/views/batch.py

@@ -46,6 +46,7 @@ from rattail.threads import Thread
 from tailbone.db import Session
 from tailbone.views import SearchableAlchemyGridView, CrudView
 from tailbone.forms import DateTimeFieldRenderer, UserFieldRenderer, EnumFieldRenderer
+from tailbone.forms.renderers.batch import FileFieldRenderer
 from tailbone.grids.search import BooleanSearchFilter, EnumSearchFilter
 from tailbone.progress import SessionProgress
 
@@ -436,6 +437,7 @@ class BatchCrud(BaseCrud):
         Redirect to view batch after creating a batch.
         """
         batch = form.fieldset.model
+        Session.flush()
         return self.view_url(batch.uuid)
 
     def post_update_url(self, form):
@@ -561,28 +563,6 @@ class BatchCrud(BaseCrud):
         return HTTPFound(location=self.view_url(batch.uuid))
 
 
-class DownloadLinkRenderer(formalchemy.FieldRenderer):
-    """
-    Field renderer for batch filenames, shows a link to download the file.
-    """
-
-    def __init__(self, route_prefix):
-        self.route_prefix = route_prefix
-
-    def __call__(self, field):
-        super(DownloadLinkRenderer, self).__init__(field)
-        return self
-
-    def render_readonly(self, **kwargs):
-        filename = self.value
-        if not filename:
-            return ''
-        batch = self.field.parent.model
-        return link_to(filename, self.request.route_url(
-                '{0}.download'.format(self.route_prefix),
-                uuid=batch.uuid))
-
-
 class FileBatchCrud(BatchCrud):
     """
     Base CRUD view for batches which involve a file upload as the first step.
@@ -597,6 +577,21 @@ class FileBatchCrud(BatchCrud):
             return HTTPFound(location=self.request.route_url(
                     '{0}.refresh'.format(self.route_prefix), uuid=batch.uuid))
 
+    @property
+    def upload_dir(self):
+        """
+        The path to the root upload folder, to be used as the ``storage_path``
+        argument for the file field renderer.
+        """
+        uploads = os.path.join(
+            self.request.rattail_config.require('rattail', 'batch.files'),
+            'uploads')
+        uploads = self.request.rattail_config.get(
+            'tailbone', 'batch.uploads', default=uploads)
+        if not os.path.exists(uploads):
+            os.makedirs(uploads)
+        return uploads
+
     def fieldset(self, model):
         """
         Creates the fieldset for the view.  Derived classes should *not*
@@ -609,14 +604,11 @@ class FileBatchCrud(BatchCrud):
         fs.cognized_by.set(label="Cognized by", renderer=UserFieldRenderer)
         fs.executed.set(renderer=DateTimeFieldRenderer(self.request.rattail_config))
         fs.executed_by.set(label="Executed by", renderer=UserFieldRenderer)
-        fs.append(formalchemy.Field('data_file'))
+        fs.filename.set(renderer=FileFieldRenderer.new(self), label="Data File")
-        fs.data_file.set(renderer=formalchemy.fields.FileFieldRenderer)
-        fs.filename.set(renderer=DownloadLinkRenderer(self.route_prefix), readonly=True)
         self.configure_fieldset(fs)
         if self.creating:
             del fs.created
             del fs.created_by
-            del fs.filename
             if 'cognized' in fs.render_fields:
                 del fs.cognized
             if 'cognized_by' in fs.render_fields:
@@ -628,8 +620,8 @@ class FileBatchCrud(BatchCrud):
             if 'data_rows' in fs.render_fields:
                 del fs.data_rows
         else:
-            if 'data_file' in fs.render_fields:
+            if self.updating and 'filename' in fs.render_fields:
-                del fs.data_file
+                fs.filename.set(readonly=True)
             batch = fs.model
             if not batch.executed:
                 if 'executed' in fs.render_fields:
@@ -648,7 +640,6 @@ class FileBatchCrud(BatchCrud):
             include=[
                 fs.created,
                 fs.created_by,
-                fs.data_file,
                 fs.filename,
                 # fs.cognized,
                 # fs.cognized_by,
@@ -669,13 +660,23 @@ class FileBatchCrud(BatchCrud):
         # For new batches, assign current user as creator, save file etc.
         if self.creating:
             batch.created_by = self.request.user
-            batch.filename = form.fieldset.data_file.renderer._filename
+
-            # Expunge batch from session to prevent it from being flushed.
+            # Expunge batch from session to prevent it from being flushed
+            # during init.  This is done as a convenience to views which
+            # provide an init method.  Some batches may have required fields
+            # which aren't filled in yet, but the view may need to query the
+            # database to obtain the values.  This will cause a session flush,
+            # and the missing fields will trigger data integrity errors.
             Session.expunge(batch)
             self.batch_inited = self.init_batch(batch)
             if self.batch_inited:
                 Session.add(batch)
-                batch.write_file(self.request.rattail_config, form.fieldset.data_file.value)
+                Session.flush()
+
+                # Handler saves a copy of the file and updates the batch filename.
+                path = os.path.join(self.upload_dir, batch.filename)
+                self.handler.set_data_file(batch, path)
+                os.remove(path)
 
     def init_batch(self, batch):
         """
@@ -716,12 +717,11 @@ class FileBatchCrud(BatchCrud):
         batch = self.current_batch()
         if not batch:
             return HTTPNotFound()
-        config = self.request.rattail_config
+        path = self.handler.data_path(batch)
-        path = batch.filepath(config)
         response = FileResponse(path, request=self.request)
-        response.headers[b'Content-Length'] = str(batch.filesize(config))
+        response.headers[b'Content-Length'] = str(os.path.getsize(path))
-        response.headers[b'Content-Disposition'] = b'attachment; filename="{0}"'.format(
+        filename = os.path.basename(batch.filename).encode('ascii', 'replace')
-            batch.filename.encode('ascii', 'replace'))
+        response.headers[b'Content-Disposition'] = b'attachment; filename="{0}"'.format(filename)
         return response
 
 

tailbone/views/vendors/catalogs.py

@@ -114,10 +114,9 @@ class VendorCatalogCrud(FileBatchCrud):
         fs.configure(
             include=[
                 fs.vendor,
-                fs.data_file.label("Catalog File"),
+                fs.filename.label("Catalog File"),
                 fs.parser_key.label("File Type"),
                 fs.effective,
-                fs.filename,
                 fs.created,
                 fs.created_by,
                 fs.executed,

tailbone/views/vendors/invoices.py

@@ -114,9 +114,8 @@ class VendorInvoiceCrud(FileBatchCrud):
         fs.configure(
             include=[
                 fs.vendor.readonly(),
-                fs.data_file.label("Invoice File"),
+                fs.filename.label("Invoice File"),
                 fs.parser_key.label("File Type"),
-                fs.filename,
                 fs.purchase_order_number.label(self.handler.po_number_title),
                 fs.invoice_date.readonly(),
                 fs.created,