[gen] Binary files stored in fields appy.fields.File are now stored outside the ZODB, on the filesystem; Ref fields can now also be rendered as dropdown menus: every menu represents a coherent group of link

ed objects. The main menu entry can be textual or an icon; computed fields are by default rendered in view and cell layouts.

fields/__init__.py

@@ -40,6 +40,7 @@ class Field:
     cssFiles = {}
     jsFiles = {}
     dLayouts = 'lrv-d-f'
+    wLayouts = Table('lrv-f')
 
     # Render a field. Optiona vars:
     # * fieldName   can be given as different as field.name for fields included

fields/computed.py

@@ -43,13 +43,14 @@ class Computed(Field):
     </x>''')
 
     def __init__(self, validator=None, multiplicity=(0,1), default=None,
-                 show='view', page='main', group=None, layouts=None, move=0,
+                 show=('view', 'result'), page='main', group=None,
-                 indexed=False, searchable=False, specificReadPermission=False,
+                 layouts=None, move=0, indexed=False, searchable=False,
-                 specificWritePermission=False, width=None, height=None,
+                 specificReadPermission=False, specificWritePermission=False,
-                 maxChars=None, colspan=1, method=None, plainText=False,
+                 width=None, height=None, maxChars=None, colspan=1, method=None,
-                 master=None, masterValue=None, focus=False, historized=False,
+                 plainText=False, master=None, masterValue=None, focus=False,
-                 sync=True, mapping=None, label=None, sdefault='', scolspan=1,
+                 historized=False, sync=True, mapping=None, label=None,
-                 swidth=None, sheight=None, context=None):
+                 sdefault='', scolspan=1, swidth=None, sheight=None,
+                 context=None):
         # The Python method used for computing the field value, or a PX.
         self.method = method
         # Does field computation produce plain text or XHTML?

fields/file.py

@@ -16,36 +16,189 @@
 
 # ------------------------------------------------------------------------------
 import time, os.path, mimetypes
+from DateTime import DateTime
 from appy import Object
 from appy.fields import Field
 from appy.px import Px
 from appy.shared import utils as sutils
+from appy.shared import UnmarshalledFile, mimeTypesExts
+
+# ------------------------------------------------------------------------------
+WRONG_FILE_TUPLE = 'This is not the way to set a file. You can specify a ' \
+    '2-tuple (fileName, fileContent) or a 3-tuple (fileName, fileContent, ' \
+    'mimeType).'
+
+# ------------------------------------------------------------------------------
+class FileInfo:
+    '''For a "file" field, its binary content is stored on the filesystem.
+       Within the database, we store a FileInfo instance that only stores some
+       metadata.'''
+    BYTES = 5000
+    def __init__(self, fsPath):
+        # The path on disk (from the root DB folder) where the file will be
+        # stored.
+        self.fsPath = fsPath
+        self.fsName = None # The name of the file in fsPath
+        self.uploadName = None # The name of the uploaded file
+        self.size = 0 # Its size, in bytes
+        self.mimeType = None # Its MIME type
+        self.modified = None # The last modification date for this file.
+
+    def removeFile(self, dbFolder, removeEmptyFolders=False):
+        '''Removes the file from the filesystem.'''
+        try:
+            os.remove(os.path.join(dbFolder, self.fsPath, self.fsName))
+        except Exception, e:
+            # If the current ZODB transaction is re-triggered, the file may
+            # already have been deleted.
+            pass
+        # Don't leave empty folders on disk. So delete folder and parent folders
+        # if this removal leaves them empty (unless p_removeEmptyFolders is
+        # False).
+        if removeEmptyFolders:
+            sutils.FolderDeleter.deleteEmpty(os.path.join(dbFolder,self.fsPath))
+
+    def normalizeFileName(self, name):
+        '''Normalizes file p_name.'''
+        return name[max(name.rfind('/'), name.rfind('\\'), name.rfind(':'))+1:]
+
+    def getShownSize(self):
+        '''Displays this file's size in the user interface.'''
+        if self.size < 1024:
+            # Display the size in bytes
+            return '%d byte(s)' % self.size
+        else:
+            # Display the size in Kb
+            return '%d Kb' % (self.size / 1024)
+
+    def replicateFile(self, src, dest):
+        '''p_src and p_dest are open file handlers. This method copies content
+           of p_src to p_dest and returns the file size.'''
+        size = 0
+        while True:
+            chunk = src.read(self.BYTES)
+            if not chunk: break
+            size += len(chunk)
+            dest.write(chunk)
+        return size
+
+    def writeFile(self, fieldName, fileObj, dbFolder):
+        '''Writes to the filesystem the p_fileObj file, that can be:
+           - a Zope FileUpload (coming from a HTTP post);
+           - a OFS.Image.File object (legacy within-ZODB file object);
+           - a tuple (fileName, fileContent, mimeType)
+             (see doc in method File.store below).'''
+        # Determine p_fileObj's type.
+        fileType = fileObj.__class__.__name__
+        # Set MIME type.
+        if fileType == 'FileUpload':
+            mimeType = fileObj.headers.get('content-type')
+        elif fileType == 'File':
+            mimeType = fileObj.content_type
+        else:
+            mimeType = fileObj[2]
+        self.mimeType = mimeType or File.defaultMimeType
+        # Determine the original name of the file to store.
+        fileName= fileType.startswith('File') and fileObj.filename or fileObj[0]
+        if not fileName:
+            # Name it according to field name. Deduce file extension from the
+            # MIME type.
+            ext = (self.mimeType in mimeTypesExts) and \
+                  mimeTypesExts[self.mimeType] or 'bin'
+            fileName = '%s.%s' % (fieldName, ext)
+        # As a preamble, extract file metadata from p_fileObj and store it in
+        # this FileInfo instance.
+        name = self.normalizeFileName(fileName)
+        self.uploadName = name
+        self.fsName = '%s%s' % (fieldName, os.path.splitext(name)[1].lower())
+        # Write the file on disk (and compute/get its size in bytes)
+        fsName = os.path.join(dbFolder, self.fsPath, self.fsName)
+        f = file(fsName, 'wb')
+        if fileType == 'FileUpload':
+            # Write the FileUpload instance on disk.
+            self.size = self.replicateFile(fileObj, f)
+        elif fileType == 'File':
+            # Write the File instance on disk.
+            if fileObj.data.__class__.__name__ == 'Pdata':
+                # The file content is splitted in several chunks.
+                f.write(fileObj.data.data)
+                nextPart = fileObj.data.next
+                while nextPart:
+                    f.write(nextPart.data)
+                    nextPart = nextPart.next
+            else:
+                # Only one chunk
+                f.write(fileObj.data)
+            self.size = fileObj.size
+        else:
+            # Write fileObj[1] on disk.
+            if fileObj[1].__class__.__name__ == 'file':
+                # It is an open file handler.
+                self.size = self.replicateFile(fileObj[1], f)
+            else:
+                # We have file content directly in fileObj[1]
+                self.size = len(fileObj[1])
+                f.write(fileObj[1])
+        f.close()
+        self.modified = DateTime()
+
+    def copyFile(self, fieldName, filePath, dbFolder):
+        '''Copies the "external" file stored at _filePath in the db-controlled
+           file system, for storing a value for p_fieldName.'''
+        # Set names for the file
+        name = self.normalizeFileName(filePath)
+        self.uploadName = name
+        self.fsName = '%s%s' % (fieldName, os.path.splitext(name)[1])
+        # Set mimeType
+        self.mimeType= mimetypes.guess_type(filePath)[0] or File.defaultMimeType
+        # Copy the file
+        shutil.copyfile(filePath, self.fsName)
+        self.modified = DateTime()
+        self.size = os.stat(self.fsName).st_size
+
+    def writeResponse(self, response, dbFolder):
+        '''Writes this file in the HTTP p_response object.'''
+        # As a preamble, initialise response headers.
+        header = response.setHeader
+        header('Content-Disposition', 'inline;filename="%s"' % self.uploadName)
+        header('Content-Type', self.mimeType)
+        header('Content-Length', self.size)
+        header('Accept-Ranges', 'bytes')
+        header('Last-Modified', self.modified.rfc822())
+        #sh('Cachecontrol', 'no-cache')
+        #sh('Expires', 'Thu, 11 Dec 1975 12:05:05 GMT')
+        # Write the file in the response
+        fsName = os.path.join(dbFolder, self.fsPath, self.fsName)
+        f = file(fsName, 'rb')
+        while True:
+            chunk = f.read(self.BYTES)
+            if not chunk: break
+            response.write(chunk)
+        f.close()
 
 # ------------------------------------------------------------------------------
 class File(Field):
 
     pxView = pxCell = Px('''
-     <x var="info=field.getFileInfo(value);
+     <x var="downloadUrl='%s/download?name=%s' % (zobj.absolute_url(), name);
-             empty=not info.size;
+             shownSize=value.getShownSize()">
-             imgSrc='%s/download?name=%s' % (zobj.absolute_url(), name)">
+      <x if="value and not field.isImage">
-      <x if="not empty and not field.isImage">
+       <a href=":downloadUrl">:value.uploadName</a>&nbsp;&nbsp;-
-       <a href=":imgSrc">:info.filename</a>&nbsp;&nbsp;-
+       <i class="discreet">:shownSize</i>
-       <i class="discreet">:'%sKb' % (info.size / 1024)</i>
       </x>
-      <x if="not empty and field.isImage"><img src=":imgSrc"/></x>
+      <x if="value and field.isImage">
-      <x if="empty">-</x>
+       <img src=":downloadUrl"
+            title=":'%s, %s' % (value.uploadName, shownSize)"/></x>
+      <x if="not value">-</x>
      </x>''')
 
     pxEdit = Px('''
-     <x var="info=field.getFileInfo(value);
+     <x var="fName=q('%s_file' % name)">
-             empty= not info.size;
+      <x if="value">:field.pxView</x><br if="value"/>
-             fName=q('%s_file' % name)">
+      <x if="value">
-
-      <x if="not empty">:field.pxView</x><br/>
-      <x if="not empty">
        <!-- Keep the file unchanged. -->
        <input type="radio" value="nochange"
-              checked=":(info.size != 0) and 'checked' or None"
+              checked=":value and 'checked' or None"
               name=":'%s_delete' % name" id=":'%s_nochange' % name"
               onclick=":'document.getElementById(%s).disabled=true' % fName"/>
        <label lfor=":'%s_nochange' % name">Keep the file unchanged</label><br/>
@@ -58,7 +211,7 @@ class File(Field):
        </x>
        <!-- Replace with a new file. -->
        <input type="radio" value=""
-              checked=":(info.size == 0) and 'checked' or None"
+              checked=":not value and 'checked' or None"
               name=":'%s_delete' % name" id=":'%s_upload' % name"
               onclick=":'document.getElementById(%s).disabled=false' % fName"/>
        <label lfor=":'%s_upload' % name">Replace it with a new file</label><br/>
@@ -66,7 +219,7 @@ class File(Field):
       <!-- The upload field. -->
       <input type="file" name=":'%s_file' % name" id=":'%s_file' % name"
              size=":field.width"/>
-      <script var="isDisabled=empty and 'false' or 'true'"
+      <script var="isDisabled=not value and 'false' or 'true'"
              type="text/javascript">:'document.getElementById(%s).disabled=%s'%\
                                      (q(fName), q(isDisabled))</script></x>''')
 
@@ -93,7 +246,7 @@ class File(Field):
         '''Returns a File instance as can be stored in the database or
            manipulated in code, filled with content from a file on disk,
            located at p_filePath. If you want to give it a name that is more
-           sexy than the actual basename of filePath, specify it in
+           sexy than the actual basename of p_filePath, specify it in
            p_fileName.
 
            If p_zope is True, it will be the raw Zope object = an instance of
@@ -110,15 +263,6 @@ class File(Field):
         if not zope: res = sutils.FileWrapper(res)
         return res
 
-    def getValue(self, obj):
-        value = Field.getValue(self, obj)
-        if value: value = sutils.FileWrapper(value)
-        return value
-
-    def getFormattedValue(self, obj, value, showChanges=False):
-        if not value: return value
-        return value._zopeFile
-
     def getRequestValue(self, request, requestName=None):
         name = requestName or self.name
         return request.get('%s_file' % name)
@@ -151,80 +295,69 @@ class File(Field):
     defaultMimeType = 'application/octet-stream'
     def store(self, obj, value):
         '''Stores the p_value that represents some file. p_value can be:
-           * an instance of Zope class ZPublisher.HTTPRequest.FileUpload. In
+           a. an instance of Zope class ZPublisher.HTTPRequest.FileUpload. In
-             this case, it is file content coming from a HTTP POST;
+              this case, it is file content coming from a HTTP POST;
-           * an instance of Zope class OFS.Image.File;
+           b. an instance of Zope class OFS.Image.File (legacy within-ZODB file
-           * an instance of appy.shared.utils.FileWrapper, which wraps an
+              object);
-             instance of OFS.Image.File and adds useful methods for manipulating
+           c. an instance of appy.shared.UnmarshalledFile. In this case, the
-             it;
+              file comes from a peer Appy site, unmarshalled from XML content
-           * a string. In this case, the string represents the path of a file
+              sent via an HTTP request;
-             on disk;
+           d. a string. In this case, the string represents the path of a file
-           * a 2-tuple (fileName, fileContent) where:
+              on disk;
-             - fileName is the name of the file (ie "myFile.odt")
+           e. a 2-tuple (fileName, fileContent) where:
-             - fileContent is the binary or textual content of the file or an
+              - fileName is the name of the file (ie "myFile.odt")
-                           open file handler.
+              - fileContent is the binary or textual content of the file or an
-           * a 3-tuple (fileName, fileContent, mimeType) where
+                open file handler.
-             - fileName and fileContent have the same meaning than above;
+           f. a 3-tuple (fileName, fileContent, mimeType) where
-             - mimeType is the MIME type of the file.
+              - fileName and fileContent have the same meaning than above;
+              - mimeType is the MIME type of the file.
         '''
+        zobj = obj.o
         if value:
-            ZFileUpload = obj.o.getProductConfig().FileUpload
+            # There is a new value to store. Get the folder on disk where to
-            OFSImageFile = obj.o.getProductConfig().File
+            # store the new file.
-            if isinstance(value, ZFileUpload):
+            dbFolder, folder = zobj.getFsFolder(create=True)
-                # The file content comes from a HTTP POST.
+            # Remove the previous file if it existed.
-                # Retrieve the existing value, or create one if None
+            info = getattr(obj.aq_base, self.name, None)
-                existingValue = getattr(obj.aq_base, self.name, None)
+            if info: info.removeFile(dbFolder)
-                if not existingValue:
+            # Store the new file. As a preamble, create a FileInfo instance.
-                    existingValue = OFSImageFile(self.name, '', '')
+            info = FileInfo(folder)
-                # Set mimetype
+            cfg = zobj.getProductConfig()
-                if value.headers.has_key('content-type'):
+            if isinstance(value, cfg.FileUpload) or isinstance(value, cfg.File):
-                    mimeType = value.headers['content-type']
+                # Cases a, b
-                else:
+                info.writeFile(self.name, value, dbFolder)
-                    mimeType = File.defaultMimeType
+            elif isinstance(value, UnmarshalledFile):
-                existingValue.content_type = mimeType
+                # Case c
-                # Set filename
+                fileInfo = (value.name, value.content, value.mimeType)
-                fileName = value.filename
+                info.writeFile(self.name, fileInfo, dbFolder)
-                filename= fileName[max(fileName.rfind('/'),fileName.rfind('\\'),
-                                       fileName.rfind(':'))+1:]
-                existingValue.filename = fileName
-                # Set content
-                existingValue.manage_upload(value)
-                setattr(obj, self.name, existingValue)
-            elif isinstance(value, OFSImageFile):
-                setattr(obj, self.name, value)
-            elif isinstance(value, sutils.FileWrapper):
-                setattr(obj, self.name, value._zopeFile)
             elif isinstance(value, basestring):
-                setattr(obj, self.name, File.getFileObject(value, zope=True))
+                # Case d
-            elif type(value) in sutils.sequenceTypes:
+                info.copyFile(self.name, value, dbFolder)
-                # It should be a 2-tuple or 3-tuple
+            else:
+                # Cases e, f. Extract file name, content and MIME type.
                 fileName = None
-                mimeType = None
                 if len(value) == 2:
                     fileName, fileContent = value
                 elif len(value) == 3:
                     fileName, fileContent, mimeType = value
-                else:
+                if not fileName:
-                    raise WRONG_FILE_TUPLE
+                    raise Exception(WRONG_FILE_TUPLE)
-                if fileName:
+                mimeType = mimeType or mimetypes.guess_type(fileName)[0]
-                    fileId = 'file.%f' % time.time()
+                info.writeFile(self.name, (fileName, fileContent, mimeType),
-                    zopeFile = OFSImageFile(fileId, fileName, fileContent)
+                               dbFolder)
-                    zopeFile.filename = fileName
+            # Store the FileInfo instance in the database.
-                    if not mimeType:
+            setattr(obj, self.name, info)
-                        mimeType = mimetypes.guess_type(fileName)[0]
-                    zopeFile.content_type = mimeType
-                    setattr(obj, self.name, zopeFile)
         else:
             # I store value "None", excepted if I find in the request the desire
             # to keep the file unchanged.
             action = None
-            rq = getattr(obj, 'REQUEST', None)
+            rq = getattr(zobj, 'REQUEST', None)
             if rq: action = rq.get('%s_delete' % self.name, None)
-            if action == 'nochange': pass
+            if action != 'nochange':
-            else: setattr(obj, self.name, None)
+                # Delete the file on disk
-
+                info = getattr(zobj.aq_base, self.name)
-    def getFileInfo(self, fileObject):
+                if info:
-        '''Returns filename and size of p_fileObject.'''
+                    info.removeFile(zobj.getDbFolder(), removeEmptyFolders=True)
-        if not fileObject: return Object(filename='', size=0)
+                # Delete the FileInfo in the DB
-        return Object(filename=fileObject.filename, size=fileObject.size)
+                setattr(zobj, self.name, None)
 # ------------------------------------------------------------------------------

fields/ref.py

@@ -16,6 +16,7 @@
 
 # ------------------------------------------------------------------------------
 import sys, re
+from appy import Object
 from appy.fields import Field
 from appy.px import Px
 from appy.gen.layout import Table
@@ -127,11 +128,127 @@ class Ref(Field):
            onclick=":ajaxBaseCall.replace('**v**', 'True')"/>
      </x>''')
 
+    # PX that displays referred objects as a list.
+    pxViewList = Px('''<x>
+     <!-- Display a simplified widget if at most 1 referenced object. -->
+     <table if="atMostOneRef">
+      <tr valign="top">
+       <!-- If there is no object -->
+       <x if="not zobjects">
+        <td class="discreet">:_('no_ref')</td>
+        <td>:field.pxAdd</td>
+       </x>
+       <!-- If there is an object -->
+       <x if="zobjects">
+        <td for="ztied in zobjects"
+            var2="includeShownInfo=True">:field.pxObjectTitle</td>
+       </x>
+      </tr>
+     </table>
+
+     <!-- Display a table in all other cases -->
+     <x if="not atMostOneRef">
+      <div if="not innerRef or showPlusIcon" style="margin-bottom: 4px">
+       (<x>:totalNumber</x>)
+       <x>:field.pxAdd</x>
+       <!-- The search button if field is queryable -->
+       <input if="zobjects and field.queryable" type="button" class="button"
+              style=":url('buttonSearch', bg=True)" value=":_('search_title')"
+              onclick=":'goto(%s)' % \
+                q('%s/search?className=%s&amp;ref=%s:%s' % \
+                (ztool.absolute_url(), tiedClassName, zobj.UID(), \
+                 field.name))"/>
+      </div>
+
+      <!-- (Top) navigation -->
+      <x>:tool.pxNavigate</x>
+
+      <!-- No object is present -->
+      <p class="discreet" if="not zobjects">:_('no_ref')</p>
+
+      <table if="zobjects" class=":innerRef and 'innerAppyTable' or ''"
+             width="100%">
+       <tr valign="bottom">
+        <td>
+         <!-- Show forward or backward reference(s) -->
+         <table class=":not innerRef and 'list' or ''"
+                width=":innerRef and '100%' or field.layouts['view'].width"
+                var="columns=ztool.getColumnsSpecifiers(tiedClassName, \
+                       field.shownInfo, dir)">
+          <tr if="field.showHeaders">
+           <th for="column in columns" width=":column.width"
+               align="column.align" var2="refField=column.field">
+            <span>:_(refField.labelId)</span>
+            <x>:field.pxSortIcons</x>
+            <x var="className=tiedClassName">:tool.pxShowDetails</x>
+           </th>
+          </tr>
+          <tr for="ztied in zobjects" valign="top"
+              class=":loop.ztied.odd and 'even' or 'odd'">
+           <td for="column in columns"
+               width=":column.width" align=":column.align"
+               var2="refField=column.field">
+            <!-- The "title" field -->
+            <x if="refField.name == 'title'">
+             <x>:field.pxObjectTitle</x>
+             <div if="ztied.mayAct()">:field.pxObjectActions</div>
+            </x>
+            <!-- Any other field -->
+            <x if="refField.name != 'title'">
+             <x var="zobj=ztied; obj=ztied.appy(); layoutType='cell';
+                     innerRef=True; field=refField"
+                if="zobj.showField(field.name, \
+                                   layoutType='result')">:field.pxRender</x>
+            </x>
+           </td>
+          </tr>
+         </table>
+        </td>
+       </tr>
+      </table>
+
+      <!-- (Bottom) navigation -->
+      <x>:tool.pxNavigate</x>
+     </x></x>''')
+
+    # PX that displays referred objects as menus.
+    pxViewMenus = Px('''
+     <table><tr style="font-size: 93%">
+      <td for="menu in field.getLinkedObjectsByMenu(obj, zobjects)">
+
+       <!-- A single object in the menu: show a clickable icon to get it -->
+       <a if="len(menu.zobjects)==1" var2="ztied=menu.zobjects[0]"
+          class="dropdownMenu" href=":field.getMenuUrl(zobj, ztied)"
+          title=":ztied.title">
+        <img if="menu.icon" src=":menu.icon"/><x
+             if="not menu.icon">:menu.text</x> 1</a>
+
+       <!-- Several objects: put them in a dropdown menu -->
+       <div if="len(menu.zobjects) &gt; 1" class="dropdownMenu"
+            var2="dropdownId='%s_%d' % (zobj.UID(), loop.menu.nb)"
+            onmouseover=":'toggleDropdown(%s)' % q(dropdownId)"
+            onmouseout=":'toggleDropdown(%s,%s)' % (q(dropdownId), q('none'))">
+        <img if="menu.icon" src=":menu.icon" title=":menu.text"/><x
+             if="not menu.icon">:menu.text</x>
+        <!-- Display the number of objects in the menu (if more than one) -->
+        <x if="len(menu.zobjects) &gt; 1">:len(menu.zobjects)</x>
+        <!-- The dropdown menu containing annexes -->
+        <div class="dropdown" id=":dropdownId">
+         <div for="ztied in menu.zobjects"
+              var2="ztiedUrl=field.getMenuUrl(zobj, ztied)">
+          <a href=":ztiedUrl">:ztied.title</a>
+         </div>
+        </div>
+       </div>
+      </td>
+     </tr></table>''')
+
     # PX that displays referred objects through this field.
-    pxView = pxCell = Px('''
+    pxView = Px('''
      <div var="innerRef=req.get('innerRef', False) == 'True';
                ajaxHookId=zobj.UID() + field.name;
-               startNumber=int(req.get('%s_startNumber' % ajaxHookId, 0));
+               render=render|'list';
+               startNumber=field.getStartNumber(render, req, ajaxHookId);
                info=field.getLinkedObjects(zobj, startNumber);
                zobjects=info.objects;
                totalNumber=info.totalNumber;
@@ -151,93 +268,18 @@ class Ref(Field):
                changeOrder=field.changeOrderEnabled(zobj);
                showSubTitles=req.get('showSubTitles', 'true') == 'true'"
           id=":ajaxHookId">
-
       <!-- The definition of "atMostOneRef" above may sound strange: we
            shouldn't check the actual number of referenced objects. But for
            back references people often forget to specify multiplicities. So
            concretely, multiplicities (0,None) are coded as (0,1). -->
-      <!-- Display a simplified widget if at most 1 referenced object. -->
-      <table if="atMostOneRef">
-       <tr valign="top">
-        <!-- If there is no object -->
-        <x if="not zobjects">
-         <td class="discreet">:_('no_ref')</td>
-         <td>:field.pxAdd</td>
-        </x>
-        <!-- If there is an object -->
-        <x if="zobjects">
-         <td for="ztied in zobjects"
-             var2="includeShownInfo=True">:field.pxObjectTitle</td>
-        </x>
-       </tr>
-      </table>
 
-      <!-- Display a table in all other cases -->
+      <x if="render == 'list'">:field.pxViewList</x>
-      <x if="not atMostOneRef">
+      <x if="render == 'menus'">:field.pxViewMenus</x>
-       <div if="not innerRef or showPlusIcon" style="margin-bottom: 4px">
-        (<x>:totalNumber</x>)
-        <x>:field.pxAdd</x>
-        <!-- The search button if field is queryable -->
-        <input if="zobjects and field.queryable" type="button" class="button"
-               style=":url('buttonSearch', bg=True)" value=":_('search_title')"
-               onclick=":'goto(%s)' % \
-                 q('%s/search?className=%s&amp;ref=%s:%s' % \
-                 (ztool.absolute_url(), tiedClassName, zobj.UID(), \
-                  field.name))"/>
-       </div>
-
-       <!-- (Top) navigation -->
-       <x>:tool.pxNavigate</x>
-
-       <!-- No object is present -->
-       <p class="discreet" if="not zobjects">:_('no_ref')</p>
-
-       <table if="zobjects" class=":innerRef and 'innerAppyTable' or ''"
-              width="100%">
-        <tr valign="bottom">
-         <td>
-          <!-- Show forward or backward reference(s) -->
-          <table class=":not innerRef and 'list' or ''"
-                 width=":innerRef and '100%' or field.layouts['view'].width"
-                 var="columns=ztool.getColumnsSpecifiers(tiedClassName, \
-                        field.shownInfo, dir)">
-           <tr if="field.showHeaders">
-            <th for="column in columns" width=":column.width"
-                align="column.align" var2="refField=column.field">
-             <span>:_(refField.labelId)</span>
-             <x>:field.pxSortIcons</x>
-             <x var="className=tiedClassName">:tool.pxShowDetails</x>
-            </th>
-           </tr>
-           <tr for="ztied in zobjects" valign="top"
-               class=":loop.ztied.odd and 'even' or 'odd'">
-            <td for="column in columns"
-                width=":column.width" align=":column.align"
-                var2="refField=column.field">
-             <!-- The "title" field -->
-             <x if="refField.name == 'title'">
-              <x>:field.pxObjectTitle</x>
-              <div if="ztied.mayAct()">:field.pxObjectActions</div>
-             </x>
-             <!-- Any other field -->
-             <x if="refField.name != 'title'">
-              <x var="zobj=ztied; obj=ztied.appy(); layoutType='cell';
-                      innerRef=True; field=refField"
-                 if="zobj.showField(field.name, \
-                                    layoutType='result')">:field.pxRender</x>
-             </x>
-            </td>
-           </tr>
-          </table>
-         </td>
-        </tr>
-       </table>
-
-       <!-- (Bottom) navigation -->
-       <x>:tool.pxNavigate</x>
-      </x>
      </div>''')
 
+    # The "menus" render mode is only applicable in "cell", not in "view".
+    pxCell = Px('''<x var="render=field.render">:field.pxView</x>''')
+
     pxEdit = Px('''
      <select if="field.link"
              var2="requestValue=req.get(name, []);
@@ -287,7 +329,9 @@ class Ref(Field):
                  masterValue=None, focus=False, historized=False, mapping=None,
                  label=None, queryable=False, queryFields=None, queryNbCols=1,
                  navigable=False, searchSelect=None, changeOrder=True,
-                 sdefault='', scolspan=1, swidth=None, sheight=None):
+                 sdefault='', scolspan=1, swidth=None, sheight=None,
+                 render='list', menuIdMethod=None, menuInfoMethod=None,
+                 menuUrlMethod=None):
         self.klass = klass
         self.attribute = attribute
         # May the user add new objects through this ref ?
@@ -361,6 +405,29 @@ class Ref(Field):
         # If changeOrder is False, it even if the user has the right to modify
         # the field, it will not be possible to move objects or sort them.
         self.changeOrder = changeOrder
+        # There are different ways to render a bunch of linked objects:
+        # - "list" (the default) renders them as a list (=a XHTML table);
+        # - "menus" renders them as a series of popup menus, grouped by type.
+        # Note that render mode "menus" will only be applied in "cell" layouts.
+        # Indeed, we need to keep the "list" rendering in the "view" layout
+        # because the "menus" rendering is minimalist and does not allow to
+        # perform all operations on Ref objects (add, move, delete, edit...).
+        self.render = render
+        # If render is 'menus', 2 methods must be provided.
+        # "menuIdMethod" will be called, with every linked object as single arg,
+        # and must return an ID that identifies the menu into which the object
+        # will be inserted.
+        self.menuIdMethod = menuIdMethod
+        # "menuInfoMethod" will be called with every collected menu ID (from
+        # calls to the previous method) to get info about this menu. This info
+        # must be a tuple (text, icon):
+        # - "text" is the menu name;
+        # - "icon" (can be None) gives the URL of an icon, if you want to render
+        #   the menu as an icon instead of a text.
+        self.menuInfoMethod = menuInfoMethod
+        # "menuUrlMethod" is an optional method that allows to compute an
+        # alternative URL for the tied object that is shown within the menu.
+        self.menuUrlMethod = menuUrlMethod
         Field.__init__(self, validator, multiplicity, default, show, page,
                        group, layouts, move, indexed, False,
                        specificReadPermission, specificWritePermission, width,
@@ -448,6 +515,54 @@ class Ref(Field):
         return self.getValue(obj, type='zobjects', someObjects=True,
                              startNumber=startNumber)
 
+    def getLinkedObjectsByMenu(self, obj, zobjects):
+        '''This method groups p_zobjects into sub-lists of objects, grouped by
+           menu (happens when self.render == 'menus').'''
+        res = []
+        # We store in "menuIds" the already encountered menus:
+        # ~{s_menuId : i_indexInRes}~
+        menuIds = {}
+        # Browse every object from p_zobjects and put them in their menu
+        # (within "res").
+        for zobj in zobjects:
+            menuId = self.menuIdMethod(obj, zobj.appy())
+            if menuId in menuIds:
+                # We have already encountered this menu.
+                menuIndex = menuIds[menuId]
+                res[menuIndex].zobjects.append(zobj)
+            else:
+                # A new menu.
+                menu = Object(id=menuId, zobjects=[zobj])
+                res.append(menu)
+                menuIds[menuId] = len(res) - 1
+        # Complete information about every menu by calling self.menuInfoMethod
+        for menu in res:
+            text, icon = self.menuInfoMethod(obj, menu.id)
+            menu.text = text
+            menu.icon = icon
+        return res
+
+    def getMenuUrl(self, zobj, ztied):
+        '''We must provide the URL of the tied object p_ztied, when shown in a
+           Ref field in render mode 'menus'. If self.menuUrlMethod is specified,
+           use it. Else, returns the "normal" URL of the view page for the tied
+           object, but without any navigation information, because in this
+           render mode, tied object's order is lost and navigation is
+           impossible.'''
+        if self.menuUrlMethod:
+            return self.menuUrlMethod(zobj.appy(), ztied.appy())
+        return ztied.getUrl(nav='')
+
+    def getStartNumber(self, render, req, ajaxHookId):
+        '''This method returns the index of the first linked object that must be
+           shown, or None if all linked objects must be shown at once (it
+           happens when p_render is "menus").'''
+        # When using 'menus' render mode, all linked objects must be shown.
+        if render == 'menus': return
+        # When using 'list' (=default) render mode, the index of the first
+        # object to show is in the request.
+        return int(req.get('%s_startNumber' % ajaxHookId, 0))
+
     def getFormattedValue(self, obj, value, showChanges=False):
         return value
 

gen/installer.py

@@ -169,6 +169,9 @@ class ZopeInstaller:
         appyTool = tool.appy()
         appyTool.log('Appy version is "%s".' % appy.version.short)
 
+        # Execute custom pre-installation code if any.
+        if hasattr(appyTool, 'beforeInstall'): appyTool.beforeInstall()
+
         # Create the default users if they do not exist.
         for login, roles in self.defaultUsers.iteritems():
             if not appyTool.count('User', noSecurity=True, login=login):

gen/migrator.py

@@ -9,123 +9,18 @@ class Migrator:
         self.installer = installer
         self.logger = installer.logger
         self.app = installer.app
+        self.tool = self.app.config.appy()
 
-    bypassRoles = ('Authenticated', 'Member')
+    def migrateTo_0_9_0(self):
-    bypassGroups = ('Administrators', 'Reviewers')
+        '''Migrates this DB to Appy 0.9.x.'''
-    def migrateUsers(self, ploneSite):
+        pass
-        '''Migrate users from Plone's acl_users to Zope acl_users with
-           corresponding Appy objects.'''
-        # First of all, remove the Plone-patched root acl_users by a standard
-        # (hum, Appy-patched) Zope UserFolder.
-        tool = self.app.config.appy()
-        from AccessControl.User import manage_addUserFolder
-        self.app.manage_delObjects(ids=['acl_users'])
-        manage_addUserFolder(self.app)
-        # Put an admin user into it
-        newUsersDb = self.app.acl_users
-        newUsersDb._doAddUser('admin', 'admin', ['Manager'], ())
-        # Copy users from Plone acl_users to Zope acl_users
-        for user in ploneSite.acl_users.getUsers():
-            id = user.getId()
-            userRoles = user.getRoles()
-            for br in self.bypassRoles:
-                if br in userRoles: userRoles.remove(br)
-            userInfo = ploneSite.portal_membership.getMemberById(id)
-            userName = userInfo.getProperty('fullname') or id
-            userEmail = userInfo.getProperty('email') or ''
-            appyUser = tool.create('users', login=id,
-                password1='fake', password2='fake', roles=userRoles,
-                name=userName, firstName=' ', email=userEmail)
-            appyUser.title = appyUser.title.strip()
-            # Set the correct password
-            password = ploneSite.acl_users.source_users._user_passwords[id]
-            newUsersDb.data[id].__ = password
-            # Manage groups. Exclude not-used default Plone groups.
-            for groupId in user.getGroups():
-                if groupId in self.bypassGroups: continue
-                if tool.count('Group', noSecurity=True, login=groupId):
-                    # The Appy group already exists, get it
-                    appyGroup = tool.search('Group', noSecurity=True,
-                                            login=groupId)[0]
-                else:
-                    # Create the group. Todo: get Plone group roles and title
-                    appyGroup = tool.create('groups', login=groupId,
-                                            title=groupId)
-                appyGroup.addUser(appyUser)
-
-    def reindexObject(self, obj):
-        obj.reindex()
-        i = 1
-        for subObj in obj.objectValues():
-            i += self.reindexObject(subObj)
-        return i # The number of reindexed (sub-)object(s)
-
-    def migrateTo_0_8_0(self):
-        '''Migrates a Plone-based (<= 0.7.1) Appy app to a Ploneless (0.8.0)
-           Appy app.'''
-        self.logger.info('Migrating to Appy 0.8.0...')
-        # Find the Plone site. It must be at the root of the Zope tree.
-        ploneSite = None
-        for obj in self.app.objectValues():
-            if obj.__class__.__name__ == 'PloneSite':
-                ploneSite = obj
-                break
-        # As a preamble: delete translation objects from self.app.config: they
-        # will be copied from the old tool.
-        self.app.config.manage_delObjects(ids=self.app.config.objectIds())
-        # Migrate data objects:
-        # - from oldDataFolder to self.app.data
-        # - from oldTool       to self.app.config (excepted translation
-        #                         objects that were re-created from i18n files).
-        appName = self.app.config.getAppName()
-        for oldFolderName in (appName, 'portal_%s' % appName.lower()):
-            oldFolder = getattr(ploneSite, oldFolderName)
-            objectIds = [id for id in oldFolder.objectIds()]
-            cutted = oldFolder.manage_cutObjects(ids=objectIds)
-            if oldFolderName == appName:
-                destFolder = self.app.data
-            else:
-                destFolder = self.app.config
-            destFolder.manage_pasteObjects(cutted)
-            i = 0
-            for obj in destFolder.objectValues():
-                i += self.reindexObject(obj)
-            self.logger.info('%d objects imported into %s.' % \
-                             (i, destFolder.getId()))
-            if oldFolderName != appName:
-                # Re-link objects copied into the self.app.config with the Tool
-                # through Ref fields.
-                tool = self.app.config.appy()
-                pList = tool.o.getProductConfig().PersistentList
-                for field in tool.fields:
-                    if field.type != 'Ref': continue
-                    n = field.name
-                    if n in ('users', 'groups'): continue
-                    uids = getattr(oldFolder, n)
-                    if uids:
-                        # Update the forward reference
-                        setattr(tool.o, n, pList(uids))
-                        # Update the back reference
-                        for obj in getattr(tool, n):
-                            backList = getattr(obj.o, field.back.name)
-                            backList.remove(oldFolder._at_uid)
-                            backList.append(tool.uid)
-                        self.logger.info('config.%s: linked %d object(s)' % \
-                                         (n, len(uids)))
-                    else:
-                        self.logger.info('config.%s: no object to link.' % n)
-        self.migrateUsers(ploneSite)
-        self.logger.info('Migration done.')
 
     def run(self):
-        if self.app.acl_users.__class__.__name__ == 'UserFolder':
+        appyVersion = self.tool.appyVersion
-            return # Already Ploneless
+        if not appyVersion or (appyVersion < '0.9.0'):
-        tool = self.app.config.appy()
-        appyVersion = tool.appyVersion
-        if not appyVersion or (appyVersion < '0.8.0'):
             # Migration is required.
             startTime = time.time()
-            self.migrateTo_0_8_0()
+            self.migrateTo_0_9_0()
             stopTime = time.time()
             elapsed = (stopTime-startTime) / 60.0
             self.logger.info('Migration done in %d minute(s).' % elapsed)

gen/mixins/__init__.py

@@ -3,7 +3,7 @@
    - mixins/ToolMixin is mixed in with the generated application Tool class.'''
 
 # ------------------------------------------------------------------------------
-import os, os.path, sys, types, urllib, cgi
+import os, os.path, re, sys, types, urllib, cgi
 from appy import Object
 from appy.px import Px
 from appy.fields.workflow import UiTransition
@@ -11,11 +11,14 @@ import appy.gen as gen
 from appy.gen.utils import *
 from appy.gen.layout import Table, defaultPageLayouts
 from appy.gen.descriptors import WorkflowDescriptor, ClassDescriptor
-from appy.shared.utils import sequenceTypes,normalizeText,Traceback,getMimeType
+from appy.shared import utils as sutils
 from appy.shared.data import rtlLanguages
 from appy.shared.xml_parser import XmlMarshaller
 from appy.shared.diff import HtmlDiff
 
+# ------------------------------------------------------------------------------
+NUMBERED_ID = re.compile('.+\d{4}$')
+
 # ------------------------------------------------------------------------------
 class BaseMixin:
     '''Every Zope class generated by appy.gen inherits from this class or a
@@ -135,6 +138,11 @@ class BaseMixin:
                 field.back.unlinkObject(obj, self, back=True)
         # Uncatalog the object
         self.reindex(unindex=True)
+        # Delete the filesystem folder corresponding to this object
+        folder = os.path.join(*self.getFsFolder())
+        if os.path.exists(folder):
+            sutils.FolderDeleter.delete(folder)
+            sutils.FolderDeleter.deleteEmpty(os.path.dirname(folder))
         # Delete the object
         self.getParentNode().manage_delObjects([self.id])
 
@@ -210,6 +218,39 @@ class BaseMixin:
         obj = createObject(tool.getPath('/temp_folder'), id, className, appName)
         return self.goto(obj.getUrl(**urlParams))
 
+    def getDbFolder(self):
+        '''Gets the folder, on the filesystem, where the database (Data.fs and
+           sub-folders) lies.'''
+        return os.path.dirname(self.getTool().getApp()._p_jar.db().getName())
+        
+    def getFsFolder(self, create=False):
+        '''Gets the folder where binary files tied to this object will be stored
+           on the filesystem. If p_create is True and the folder does not exist,
+           it is created (together with potentially missing parent folders).
+           This folder is returned as a tuple (s_baseDbFolder, s_subPath).'''
+        objId = self.id
+        # Get the root folder where Data.fs lies.
+        dbFolder = self.getDbFolder()
+        # Build the list of path elements within this db folder.
+        path = []
+        inConfig = False
+        for elem in self.getPhysicalPath():
+            if not elem: continue
+            if elem == 'data': continue
+            if elem == 'config': inConfig = True
+            if not path or ((len(path) == 1) and inConfig):
+                # This object is at the root of the filesystem.
+                if NUMBERED_ID.match(elem):
+                    path.append(elem[-4:])
+            path.append(elem)
+            # We are done if elem corresponds to the object id.
+            if elem == objId: break
+        path = os.sep.join(path)
+        if create:
+            fullPath = os.path.join(dbFolder, path)
+            if not os.path.exists(fullPath): os.makedirs(fullPath)
+        return dbFolder, path
+
     def view(self):
         '''Returns the view PX.'''
         obj = self.appy()
@@ -504,7 +545,7 @@ class BaseMixin:
                 else:
                     res = XmlMarshaller().marshall(methodRes, objectType='appy')
             except Exception, e:
-                tb = Traceback.get()
+                tb = sutils.Traceback.get()
                 res = XmlMarshaller().marshall(tb, objectType='appy')
         return res
 
@@ -1132,7 +1173,7 @@ class BaseMixin:
         elif resultType.startswith('file'):
             # msg does not contain a message, but a file instance.
             response = self.REQUEST.RESPONSE
-            response.setHeader('Content-Type', getMimeType(msg.name))
+            response.setHeader('Content-Type', sutils.getMimeType(msg.name))
             response.setHeader('Content-Disposition', 'inline;filename="%s"' %\
                                os.path.basename(msg.name))
             response.write(msg.read())
@@ -1218,7 +1259,7 @@ class BaseMixin:
 
     def SortableTitle(self):
         '''Returns the title as must be stored in index "SortableTitle".'''
-        return normalizeText(self.Title())
+        return sutils.normalizeText(self.Title())
 
     def SearchableText(self):
         '''This method concatenates the content of every field with
@@ -1522,17 +1563,10 @@ class BaseMixin:
            (not appyType.isShowable(self, 'result')):
             from zExceptions import NotFound
             raise NotFound()
-        theFile = getattr(self.aq_base, name, None)
+        info = getattr(self.aq_base, name, None)
-        if theFile:
+        if info:
-            response = self.REQUEST.RESPONSE
+            # Write the file in the HTTP response.
-            response.setHeader('Content-Disposition', 'inline;filename="%s"' % \
+            info.writeResponse(self.REQUEST.RESPONSE, self.getDbFolder())
-                               theFile.filename)
-            # Define content type
-            if theFile.content_type:
-                response.setHeader('Content-Type', theFile.content_type)
-            response.setHeader('Cachecontrol', 'no-cache')
-            response.setHeader('Expires', 'Thu, 11 Dec 1975 12:05:05 GMT')
-            return theFile.index_html(self.REQUEST, self.REQUEST.RESPONSE)
 
     def upload(self):
         '''Receives an image uploaded by the user via ckeditor and stores it in

gen/ui/appy.css

@@ -100,6 +100,11 @@ img { border: 0; vertical-align: middle }
 .popup { display: none; position: absolute; top: 30%; left: 35%;
          width: 350px; z-index : 100; background: white; padding: 8px;
          border: 1px solid grey }
+.dropdown { display:none; position: absolute; border: 1px solid #cccccc;
+            background-color: white; padding-top: 4px }
+.dropdownMenu { cursor: pointer; padding-right: 4px }
+.dropdown a { padding: 0 0.5em }
+.dropdown a:hover { text-decoration: underline }
 .list { margin-bottom: 3px }
 .list td, .list th { border: 1px solid grey;
                      padding-left: 5px; padding-right: 5px; padding-top: 3px }

gen/ui/appy.js

@@ -244,6 +244,18 @@ function toggleCheckbox(visibleCheckbox, hiddenBoolean) {
   else hidden.value = 'False';
 }
 
+// Shows/hides a dropdown menu
+function toggleDropdown(dropdownId, forcedValue){
+  var dropdown = document.getElementById(dropdownId);
+  // Force to p_forcedValue if specified
+  if (forcedValue) {dropdown.style.display = forcedValue}
+  else {
+    var displayValue = dropdown.style.display;
+    if (displayValue == 'block') dropdown.style.display = 'none';
+    else dropdown.style.display = 'block';
+  }
+}
+
 // Function that sets a value for showing/hiding sub-titles.
 function setSubTitles(value, tag) {
   createCookie('showSubTitles', value);

gen/wrappers/__init__.py

@@ -14,9 +14,6 @@ from appy.shared.xml_parser import XmlMarshaller
 from appy.shared.csv_parser import CsvMarshaller
 
 # Some error messages ----------------------------------------------------------
-WRONG_FILE_TUPLE = 'This is not the way to set a file. You can specify a ' \
-    '2-tuple (fileName, fileContent) or a 3-tuple (fileName, fileContent, ' \
-    'mimeType).'
 FREEZE_ERROR = 'Error while trying to freeze a "%s" file in POD field ' \
     '"%s" (%s).'
 FREEZE_FATAL_ERROR = 'A server error occurred. Please contact the system ' \

shared/__init__.py

@@ -36,7 +36,7 @@ class UnmarshalledFile:
     def __init__(self):
         self.name = '' # The name of the file on disk
         self.mimeType = None # The MIME type of the file
-        self.content = '' # The binary content of the file of a file object
+        self.content = '' # The binary content of the file or a file object
         self.size = 0 # The length of the file in bytes.
 
 class UnicodeBuffer:

shared/utils.py

@@ -23,6 +23,7 @@ sequenceTypes = (list, tuple)
 
 # ------------------------------------------------------------------------------
 class FolderDeleter:
+    @staticmethod
     def delete(dirName):
         '''Recursively deletes p_dirName.'''
         dirName = os.path.abspath(dirName)
@@ -32,7 +33,19 @@ class FolderDeleter:
             for name in dirs:
                 os.rmdir(os.path.join(root, name))
         os.rmdir(dirName)
-    delete = staticmethod(delete)
+
+    @staticmethod
+    def deleteEmpty(dirName):
+        '''Deletes p_dirName and its parent dirs if they are empty.'''
+        while True:
+            try:
+                if not os.listdir(dirName):
+                    os.rmdir(dirName)
+                    dirName = os.path.dirname(dirName)
+                else:
+                    break
+            except OSError, oe:
+                break
 
 # ------------------------------------------------------------------------------
 extsToClean = ('.pyc', '.pyo', '.fsz', '.deltafsz', '.dat', '.log')