New test system based on doctest and unittest and many more.

gen/__init__.py

@@ -127,7 +127,7 @@ class Integer(Type):
                  specificWritePermission=False, width=None, height=None,
                  master=None, masterValue=None, focus=False):
         Type.__init__(self, validator, multiplicity, index, default, optional,
-                      editDefault, show, page, group, move, indexed, False,
+                      editDefault, show, page, group, move, indexed, searchable,
                       specificReadPermission, specificWritePermission, width,
                       height, master, masterValue, focus)
         self.pythonType = long
@@ -549,4 +549,9 @@ class Config:
         # If you don't need the portlet that appy.gen has generated for your
         # application, set the following parameter to False.
         self.showPortlet = True
+        # Default number of flavours. It will be used for generating i18n labels
+        # for classes in every flavour. Indeed, every flavour can name its
+        # concepts differently. For example, class Thing in flavour 2 may have
+        # i18n label "MyProject_Thing_2".
+        self.numberOfFlavours = 2
 # ------------------------------------------------------------------------------

gen/generator.py

@@ -150,6 +150,7 @@ class Generator:
         self.workflows = []
         self.initialize()
         self.config = Config.getDefault()
+        self.modulesWithTests = set()
 
     def determineAppyType(self, klass):
         '''Is p_klass an Appy class ? An Appy workflow? None of this ?
@@ -171,6 +172,12 @@ class Generator:
                     break
         return res
 
+    def containsTests(self, moduleOrClass):
+        '''Does p_moduleOrClass contain doctests?'''
+        if moduleOrClass.__doc__ and (moduleOrClass.__doc__.find('>>>') != -1):
+            return True
+        return False
+
     IMPORT_ERROR = 'Warning: error while importing module %s (%s)'
     SYNTAX_ERROR = 'Warning: error while parsing module %s (%s)'
     def walkModule(self, moduleName):
@@ -190,6 +197,8 @@ class Generator:
         except SyntaxError, se:
             print self.SYNTAX_ERROR % (moduleName, str(se))
             return
+        if self.containsTests(moduleObj):
+            self.modulesWithTests.add(moduleObj.__name__)
         classType = type(Generator)
         # Find all classes in this module
         for moduleElemName in moduleObj.__dict__.keys():
@@ -215,10 +224,14 @@ class Generator:
                             descrClass = self.classDescriptor
                             self.classes.append(
                                 descrClass(moduleElem, attrs, self))
+                        if self.containsTests(moduleElem):
+                            self.modulesWithTests.add(moduleObj.__name__)
                     elif appyType == 'workflow':
                         descrClass = self.workflowDescriptor
                         self.workflows.append(
                             descrClass(moduleElem, attrs, self))
+                        if self.containsTests(moduleElem):
+                            self.modulesWithTests.add(moduleObj.__name__)
             elif isinstance(moduleElem, Config):
                 self.config = moduleElem
 

gen/plone25/generator.py

@@ -132,6 +132,7 @@ class Generator(AbstractGenerator):
         self.generateInstall()
         self.generateWorkflows()
         self.generateWrappers()
+        self.generateTests()
         if self.config.frontPage == True:
             self.labels.append(msg('front_page_text', '', msg.FRONT_PAGE_TEXT))
             self.copyFile('frontPage.pt', self.repls,
@@ -155,8 +156,8 @@ class Generator(AbstractGenerator):
         f = open(os.path.join(self.outputFolder, 'version.txt'), 'w')
         f.write(self.version)
         f.close()
-        # Make Extensions a Python package
+        # Make Extensions and tests Python packages
-        for moduleFolder in ('Extensions',):
+        for moduleFolder in ('Extensions', 'tests'):
             initFile = '%s/%s/__init__.py' % (self.outputFolder, moduleFolder)
             if not os.path.isfile(initFile):
                 f = open(initFile, 'w')
@@ -539,6 +540,14 @@ class Generator(AbstractGenerator):
         repls['podTemplateBody'] = PodTemplate._appy_getBody()
         self.copyFile('appyWrappers.py', repls, destFolder='Extensions')
 
+    def generateTests(self):
+        '''Generates the file needed for executing tests.'''
+        repls = self.repls.copy()
+        modules = self.modulesWithTests
+        repls['imports'] = '\n'.join(['import %s' % m for m in modules])
+        repls['modulesWithTests'] = ','.join(modules)
+        self.copyFile('testAll.py', repls, destFolder='tests')
+
     def generateTool(self):
         '''Generates the Plone tool that corresponds to this application.'''
         # Generate the tool class in itself and related i18n messages
@@ -672,7 +681,7 @@ class Generator(AbstractGenerator):
         poMsgPl.produceNiceDefault()
         self.labels.append(poMsgPl)
         # Create i18n labels for flavoured variants
-        for i in range(2,10):
+        for i in range(2, self.config.numberOfFlavours+1):
             poMsg = PoMessage('%s_%d' % (classDescr.name, i), '',
                               classDescr.klass.__name__)
             poMsg.produceNiceDefault()

gen/plone25/installer.py

@@ -104,6 +104,7 @@ class PloneInstaller:
         if not hasattr(site.portal_types, self.appyFolderType):
             self.registerAppyFolderType()
         # Create the folder
+        
         if not hasattr(site.aq_base, self.productName):
             # Temporarily allow me to create Appy large plone folders
             getattr(site.portal_types, self.appyFolderType).global_allow = 1
@@ -114,6 +115,7 @@ class PloneInstaller:
                                title=self.productName)
             getattr(site.portal_types, self.appyFolderType).global_allow = 0
         appFolder = getattr(site, self.productName)
+        
         # All roles defined as creators should be able to create the
         # corresponding root content types in this folder.
         i = -1

gen/plone25/mixins/TestMixin.py

@@ -0,0 +1,27 @@
+# ------------------------------------------------------------------------------
+class TestMixin:
+    '''This class is mixed in with any PloneTestCase.'''
+    def createUser(self, userId, roles):
+        '''Creates a user p_name p_with some p_roles.'''
+        pms = self.portal.portal_membership
+        pms.addMember(userId, 'password', [], [])
+        self.setRoles(roles, name=userId)
+
+    def changeUser(self, userId):
+        '''Logs out currently logged user and logs in p_loginName.'''
+        self.logout()
+        self.login(userId)
+
+# Functions executed before and after every test -------------------------------
+def beforeTest(test):
+    g = test.globs
+    g['tool'] = test.app.plone.get('portal_%s' % g['appName'].lower()).appy()
+    g['appFolder'] = g['tool'].o.getProductConfig().diskFolder
+    moduleOrClassName = g['test'].name # Not used yet.
+    # Initialize the test
+    test.createUser('admin', ('Member','Manager'))
+    test.login('admin')
+    g['t'] = g['test']
+
+def afterTest(test): pass
+# ------------------------------------------------------------------------------

gen/plone25/mixins/__init__.py

@@ -1024,7 +1024,8 @@ class AbstractMixin:
         self._appy_manageRefsFromRequest()
         # If the creation was initiated by another object, update the
         # reference.
-        if created:
+        if created and hasattr(self.REQUEST, 'SESSION'):
+            # When used by the test system, no SESSION object is created.
             session = self.REQUEST.SESSION
             initiatorUid = session.get('initiator', None)
             initiator = None

gen/plone25/skin/macros.pt

@@ -200,7 +200,7 @@
 <metal:group define-macro="showGroup">
   <fieldset class="appyGroup">
     <legend><i tal:define="groupDescription python:contextObj.translate('%s_group_%s' % (contextObj.meta_type, widgetDescr['name']))"
-               tal:content="groupDescription"></i></legend>
+               tal:content="structure groupDescription"></i></legend>
     <table tal:define="global fieldNb python:-1" width="100%">
       <tr valign="top" tal:repeat="rowNb python:range(widgetDescr['rows'])">
         <td tal:repeat="colNb python:range(widgetDescr['cols'])"
@@ -754,8 +754,8 @@
     var state = readCookie(groupId);
     if ((state != 'collapsed') && (state != 'expanded')) {
       // No cookie yet, create it.
-      createCookie(groupId, 'expanded');
+      createCookie(groupId, 'collapsed');
-      state = 'expanded';
+      state = 'collapsed';
     }
     var group = document.getElementById(groupId);
     var displayValue = 'none';
@@ -829,7 +829,7 @@
     <tal:searchOrGroup repeat="searchOrGroup python: tool.getSearches(rootClass)">
       <tal:group condition="searchOrGroup/isGroup">
       <tal:expanded define="group searchOrGroup;
-                            expanded python: tool.getCookieValue(group['labelId']) == 'expanded'">
+                            expanded python: tool.getCookieValue(group['labelId'], default='collapsed') == 'expanded'">
         <tal:comment replace="nothing">Group name</tal:comment>
         <dt class="portletAppyItem portletGroup">
           <img align="left" style="cursor:pointer"
@@ -839,15 +839,15 @@
           <span tal:replace="group/label"/>
         </dt>
         <tal:comment replace="nothing">Group searches</tal:comment>
-        <div tal:attributes="id group/labelId;
+        <span tal:attributes="id group/labelId;
                              style python:test(expanded, 'display:block', 'display:none')">
-          <dt class="portletAppyItem portletSearch" tal:repeat="search group/searches">
+          <dt class="portletAppyItem portletSearch portletGroupItem" tal:repeat="search group/searches">
             <a tal:attributes="href python: '%s?type_name=%s&flavourNumber=%s&search=%s' % (queryUrl, rootClass, flavourNumber, search['name']);
                          title search/descr;
                          class python: test(search['name'] == currentSearch, 'portletCurrent', '');"
                tal:content="structure search/label"></a>
           </dt>
-        </div>
+        </span>
       </tal:expanded>
       </tal:group>
       <dt tal:define="search searchOrGroup" tal:condition="not: searchOrGroup/isGroup"
@@ -855,8 +855,7 @@
 
         <a tal:attributes="href python: '%s?type_name=%s&flavourNumber=%s&search=%s' % (queryUrl, rootClass, flavourNumber, search['name']);
                      title search/descr;
-                     class python: test(search['name'] == currentSearch, 'portletCurrent', '');
+                     class python: test(search['name'] == currentSearch, 'portletCurrent', '');"
-                     id search/group"
            tal:content="structure search/label"></a>
       </dt>
     </tal:searchOrGroup>

gen/plone25/templates/ArchetypesTemplate.py

@@ -1,5 +1,6 @@
 <!codeHeader!>
 from AccessControl import ClassSecurityInfo
+from DateTime import DateTime
 from Products.Archetypes.atapi import *
 import Products.<!applicationName!>.config
 from Extensions.appyWrappers import <!genClassName!>_Wrapper

gen/plone25/templates/FlavourTemplate.py

@@ -1,5 +1,6 @@
 <!codeHeader!>
 from AccessControl import ClassSecurityInfo
+from DateTime import DateTime
 from Products.Archetypes.atapi import *
 import Products.<!applicationName!>.config
 from appy.gen.plone25.mixins.FlavourMixin import FlavourMixin

gen/plone25/templates/IEFixes.css.dtml

@@ -1,6 +1,12 @@
 /* Appy-specific IE-fixes */
 .portletSearch {
   font-size: 85%;
+  border-left: 1px solid #8cacbb;
+  border-right: 1px solid #8cacbb;
+}
+.portletGroup {
+  font-size: 85%;
+  padding-left: 0.7em;
 }
 
 /* Stylesheet with Internet Explorer-specific workarounds. */

gen/plone25/templates/PodTemplate.py

@@ -1,5 +1,6 @@
 <!codeHeader!>
 from AccessControl import ClassSecurityInfo
+from DateTime import DateTime
 from Products.Archetypes.atapi import *
 import Products.<!applicationName!>.config
 from appy.gen.plone25.mixins.PodTemplateMixin import PodTemplateMixin

gen/plone25/templates/Styles.css.dtml

@@ -229,7 +229,7 @@ fieldset {
 }
 .portletSearch {
   padding: 0 0 0 0.6em;
-  font-style: italic;
+  font-style: normal;
   font-size: 95%;
 }
 .portletGroup {
@@ -237,6 +237,10 @@ fieldset {
   font-weight: bold;
   font-style: normal;
 }
+.portletGroupItem {
+  padding-left: 0.8em;
+  font-style: italic;
+}
 .portletCurrent {
   font-weight: bold;
 }

gen/plone25/templates/ToolTemplate.py

@@ -1,5 +1,6 @@
 <!codeHeader!>
 from AccessControl import ClassSecurityInfo
+from DateTime import DateTime
 from Products.Archetypes.atapi import *
 from Products.CMFCore.utils import UniqueObject
 import Products.<!applicationName!>.config

gen/plone25/templates/config.py

@@ -1,5 +1,5 @@
 <!codeHeader!>
-import sys
+import os, os.path, sys
 try: # New CMF
     from Products.CMFCore.permissions import setDefaultRoles
 except ImportError: # Old CMF
@@ -23,6 +23,7 @@ logger = logging.getLogger('<!applicationName!>')
 
 # Some global variables --------------------------------------------------------
 PROJECTNAME = '<!applicationName!>'
+diskFolder = os.path.dirname(<!applicationName!>.__file__)
 defaultAddRoles = [<!defaultAddRoles!>]
 DEFAULT_ADD_CONTENT_PERMISSION = "Add portal content"
 ADD_CONTENT_PERMISSIONS = {

gen/plone25/templates/testAll.py

@@ -0,0 +1,25 @@
+<!codeHeader!>
+
+from unittest import TestSuite
+from Testing import ZopeTestCase
+from Testing.ZopeTestCase import ZopeDocTestSuite
+from Products.PloneTestCase import PloneTestCase
+from appy.gen.plone25.mixins.TestMixin import TestMixin, beforeTest, afterTest
+<!imports!>
+
+# Initialize Zope & Plone test systems -----------------------------------------
+ZopeTestCase.installProduct('<!applicationName!>')
+PloneTestCase.setupPloneSite(products=['<!applicationName!>'])
+
+class Test(PloneTestCase.PloneTestCase, TestMixin):
+    '''Base test class for <!applicationName!> test cases.'''
+
+# Data needed for defining the tests -------------------------------------------
+data = {'test_class': Test, 'setUp': beforeTest, 'tearDown': afterTest,
+        'globs': {'appName': '<!applicationName!>'}}
+modulesWithTests = [<!modulesWithTests!>]
+
+# ------------------------------------------------------------------------------
+def test_suite():
+    return TestSuite([ZopeDocTestSuite(m, **data) for m in modulesWithTests])
+# ------------------------------------------------------------------------------

gen/plone25/utils.py

@@ -7,6 +7,8 @@ def stringify(value):
         for v in value:
             res += '%s,' % stringify(v)
         res += ')'
+    elif value.__class__.__name__ == 'DateTime':
+        res = 'DateTime("%s")' % value.strftime('%Y/%m/%d %H:%M')
     else:
         res = str(value)
         if isinstance(value, basestring):

gen/plone25/wrappers/__init__.py

@@ -6,6 +6,7 @@ import time, os.path, mimetypes, unicodedata
 from appy.gen import Search
 from appy.gen.utils import sequenceTypes
 from appy.shared.utils import getOsTempFolder
+from appy.shared.xml_parser import XmlMarshaller
 
 # Some error messages ----------------------------------------------------------
 WRONG_FILE_TUPLE = 'This is not the way to set a file. You can specify a ' \
@@ -68,6 +69,8 @@ class AbstractWrapper:
             self._set_file_attribute(name, v)
         else:
             exec "self.o.set%s%s(v)" % (name[0].upper(), name[1:])
+    def __repr__(self):
+        return '<%s wrapper at %s>' % (self.klass.__name__, id(self))
     def __cmp__(self, other):
         if other: return cmp(self.o, other.o)
         else:     return 1
@@ -258,6 +261,26 @@ class AbstractWrapper:
            method in those cases.'''
         self.o.reindexObject()
 
+    def export(self, at='string'):
+        '''Creates an "exportable", XML version of this object. If p_at is
+           "string", this method returns the XML version. Else, (a) if not p_at,
+           the XML will be exported on disk, in the OS temp folder, with an
+           ugly name; (b) else, it will be exported at path p_at.'''
+        # Determine where to put the result
+        toDisk = (at != 'string')
+        if toDisk and not at:
+            at = getOsTempFolder() + '/' + self.o.UID() + '.xml'
+        # Create the XML version of the object
+        xml = XmlMarshaller().marshall(self.o, objectType='archetype')
+        # Produce the desired result
+        if toDisk:
+            f = file(at, 'w')
+            f.write(xml)
+            f.close()
+            return at
+        else:
+            return xml
+
 # ------------------------------------------------------------------------------
 class FileWrapper:
     '''When you get, from an appy object, the value of a File attribute, you

pod/converter.py

@@ -17,7 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,USA.
 
 # ------------------------------------------------------------------------------
-import sys, os, os.path, time, signal
+import sys, os, os.path, time, signal, unicodedata
 from optparse import OptionParser
 
 ODT_FILE_TYPES = {'doc': 'MS Word 97', # Could be 'MS Word 2003 XML'
@@ -55,6 +55,8 @@ class Converter:
     def __init__(self, docPath, resultType, port=DEFAULT_PORT):
         self.port = port
         self.docUrl = self.getDocUrl(docPath)
+        self.docUrlStr = unicodedata.normalize('NFKD', self.docUrl).encode(
+            "ascii", "ignore")
         self.resultFilter = self.getResultFilter(resultType)
         self.resultUrl = self.getResultUrl(resultType)
         self.ooContext = None
@@ -74,7 +76,7 @@ class Converter:
                                                     ODT_FILE_TYPES.keys()))
         return res
     def getResultUrl(self, resultType):
-        baseName = os.path.splitext(self.docUrl)[0]
+        baseName = os.path.splitext(self.docUrlStr)[0]
         if resultType != 'odt':
             res = '%s.%s' % (baseName, resultType)
         else:
@@ -164,7 +166,7 @@ class Converter:
                     except IndexOutOfBoundsException:
                         pass
         except IllegalArgumentException, iae:
-            raise ConverterError(URL_NOT_FOUND % (self.docUrl, iae))
+            raise ConverterError(URL_NOT_FOUND % (self.docUrlStr, iae))
     def convertDocument(self):
         if self.resultFilter != 'ODT':
             # I must really perform a conversion

shared/__init__.py

@@ -13,6 +13,9 @@ mimeTypes = {'odt': 'application/vnd.oasis.opendocument.text',
 class UnmarshalledObject:
     '''Used for producing objects from a marshalled Python object (in some files
        like a CSV file or an XML file).'''
+    def __init__(self, **fields):
+        for k, v in fields.iteritems():
+            setattr(self, k, v)
     def __repr__(self):
         res = u'<PythonObject '
         for attrName, attrValue in self.__dict__.iteritems():

shared/test.py

@@ -17,12 +17,12 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,USA.
 
 # ------------------------------------------------------------------------------
-import os, os.path, sys, difflib, time, xml.sax
+import os, os.path, sys, time
-from xml.sax.handler import ContentHandler
 from optparse import OptionParser
 from appy.shared.utils import FolderDeleter, Traceback
 from appy.shared.errors import InternalError
 from appy.shared.rtf import RtfTablesParser
+from appy.shared.xml_parser import XmlComparator
 
 # ------------------------------------------------------------------------------
 class TesterError(Exception): pass
@@ -59,78 +59,6 @@ TEST_REPORT_SINGLETON_ERROR = 'You can only use the TestReport constructor ' \
                               'TestReport instance via the TestReport.' \
                               'instance static member.'
 
-# ------------------------------------------------------------------------------
-class XmlHandler(ContentHandler):
-    '''This handler is used for producing a readable XML (with carriage returns)
-       and for removing some tags that always change (like dates) from a file
-       that need to be compared to another file.'''
-    def __init__(self, xmlTagsToIgnore, xmlAttrsToIgnore):
-        ContentHandler.__init__(self)
-        self.res = u'<?xml version="1.0" encoding="UTF-8"?>'
-        self.namespaces = {} # ~{s_namespaceUri:s_namespaceName}~
-        self.indentLevel = -1
-        self.tabWidth = 3
-        self.tagsToIgnore = xmlTagsToIgnore
-        self.attrsToIgnore = xmlAttrsToIgnore
-        self.ignoring = False # Some content must be ignored, and not dumped
-        # into the result.
-    def isIgnorable(self, elem):
-        '''Is p_elem an ignorable element ?'''
-        res = False
-        for nsUri, elemName in self.tagsToIgnore:
-            elemFullName = ''
-            try:
-                nsName = self.ns(nsUri)
-                elemFullName = '%s:%s' % (nsName, elemName)
-            except KeyError:
-                pass
-            if elemFullName == elem:
-                res = True
-                break
-        return res
-    def setDocumentLocator(self, locator):
-        self.locator = locator
-    def endDocument(self):
-        pass
-    def dumpSpaces(self):
-        self.res += '\n' + (' ' * self.indentLevel * self.tabWidth)
-    def manageNamespaces(self, attrs):
-        '''Manage namespaces definitions encountered in attrs'''
-        for attrName, attrValue in attrs.items():
-            if attrName.startswith('xmlns:'):
-                self.namespaces[attrValue] = attrName[6:]
-    def ns(self, nsUri):
-        return self.namespaces[nsUri]
-    def startElement(self, elem, attrs):
-        self.manageNamespaces(attrs)
-        # Do we enter into a ignorable element ?
-        if self.isIgnorable(elem):
-            self.ignoring = True
-        else:
-            if not self.ignoring:
-                self.indentLevel += 1
-                self.dumpSpaces()
-                self.res += '<%s' % elem
-                attrsNames = attrs.keys()
-                attrsNames.sort()
-                for attrToIgnore in self.attrsToIgnore:
-                    if attrToIgnore in attrsNames:
-                        attrsNames.remove(attrToIgnore)
-                for attrName in attrsNames:
-                    self.res += ' %s="%s"' % (attrName, attrs[attrName])
-                self.res += '>'
-    def endElement(self, elem):
-        if self.isIgnorable(elem):
-            self.ignoring = False
-        else:
-            if not self.ignoring:
-                self.dumpSpaces()
-                self.indentLevel -= 1
-                self.res += '</%s>' % elem
-    def characters(self, content):
-        if not self.ignoring:
-            self.res += content.replace('\n', '')
-
 # ------------------------------------------------------------------------------
 class TestReport:
     instance = None
@@ -175,51 +103,11 @@ class Test:
             expectedFlavourSpecific = '%s.%s' % (expected, self.flavour)
             if os.path.exists(expectedFlavourSpecific):
                 expected = expectedFlavourSpecific
-        differ = difflib.Differ()
+        # Perform the comparison
-        if areXml:
+        comparator = XmlComparator(expected, actual, areXml, xmlTagsToIgnore,
-            f = file(expected)
+            xmlAttrsToIgnore)
-            contentA = f.read()
+        return not comparator.filesAreIdentical(
-            f.close()
+            report=self.report, encoding=encoding)
-            # Actual result
-            f = file(actual)
-            contentB = f.read()
-            f.close()
-            xmlHandler = XmlHandler(xmlTagsToIgnore, xmlAttrsToIgnore)
-            xml.sax.parseString(contentA, xmlHandler)
-            contentA = xmlHandler.res.split('\n')
-            xmlHandler = XmlHandler(xmlTagsToIgnore, xmlAttrsToIgnore)
-            xml.sax.parseString(contentB, xmlHandler)
-            contentB = xmlHandler.res.split('\n')
-        else:
-            f = file(expected)
-            contentA = f.readlines()
-            f.close()
-            # Actual result
-            f = file(actual)
-            contentB = f.readlines()
-            f.close()
-        diffResult = list(differ.compare(contentA, contentB))
-        atLeastOneDiff = False
-        lastLinePrinted = False
-        i = -1
-        for line in diffResult:
-            i += 1
-            if line and (line[0] != ' '):
-                if not atLeastOneDiff:
-                    self.report.say('Difference(s) detected between files ' \
-                                    '%s and %s:' % (expected, actual),
-                                    encoding='utf-8')
-                    atLeastOneDiff = True
-                if not lastLinePrinted:
-                    self.report.say('...')
-                if areXml:
-                    self.report.say(line, encoding=encoding)
-                else:
-                    self.report.say(line[:-1], encoding=encoding)
-                lastLinePrinted = True
-            else:
-                lastLinePrinted = False
-        return atLeastOneDiff
     def run(self):
         self.report.say('-' * 79)
         self.report.say('- Test %s.' % self.data['Name'])

shared/xml_parser.py

@@ -17,7 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,USA.
 
 # ------------------------------------------------------------------------------
-import xml.sax
+import xml.sax, difflib
 from xml.sax.handler import ContentHandler, ErrorHandler
 from xml.sax.xmlreader import InputSource
 from StringIO import StringIO
@@ -188,6 +188,12 @@ class XmlUnmarshaller(XmlParser):
         # for example convert strings that have specific values (in this case,
         # knowing that the value is a 'string' is not sufficient).
 
+    def convertAttrs(self, attrs):
+        '''Converts XML attrs to a dict.'''
+        res = {}
+        for k, v in attrs.items(): res[str(k)] = v
+        return res
+
     def startDocument(self):
         self.res = None # The resulting web of Python objects
         # (UnmarshalledObject instances).
@@ -210,7 +216,8 @@ class XmlUnmarshaller(XmlParser):
             elemType = self.tagTypes[elem]
         if elemType in self.containerTags:
             # I must create a new container object.
-            if elemType == 'object': newObject = UnmarshalledObject()
+            if elemType == 'object':
+                newObject = UnmarshalledObject(**self.convertAttrs(attrs))
             elif elemType == 'tuple': newObject = [] # Tuples become lists
             elif elemType == 'list': newObject = []
             elif elemType == 'file':
@@ -219,7 +226,7 @@ class XmlUnmarshaller(XmlParser):
                     newObject.name = attrs['name']
                 if attrs.has_key('mimeType'):
                     newObject.mimeType = attrs['mimeType']
-            else: newObject = UnmarshalledObject()
+            else: newObject = UnmarshalledObject(**self.convertAttrs(attrs))
             # Store the value on the last container, or on the root object.
             self.storeValue(elem, newObject)
             # Push the new object on the container stack
@@ -424,10 +431,12 @@ class XmlMarshaller:
            (Zope/Plone), specify 'archetype' for p_objectType.'''
         res = StringIO()
         # Dump the XML prologue and root element
+        if objectType == 'archetype': objectId = instance.UID() # ID in DB
+        else: objectId = str(id(instance)) # ID in RAM
         res.write(self.xmlPrologue)
         res.write('<'); res.write(self.rootElementName)
-        res.write(' type="object">')
+        res.write(' type="object" id="'); res.write(objectId); res.write('">')
-        # Dump the value of the fields that must be dumped
+        # Dump the object ID and the value of the fields that must be dumped
         if objectType == 'popo':
             for fieldName, fieldValue in instance.__dict__.iteritems():
                 mustDump = False
@@ -479,4 +488,149 @@ class XmlMarshaller:
            result. p_res is the StringIO buffer where the result of the
            marshalling process is currently dumped; p_instance is the instance
            currently marshalled.'''
+
+# ------------------------------------------------------------------------------
+class XmlHandler(ContentHandler):
+    '''This handler is used for producing, in self.res, a readable XML
+       (with carriage returns) and for removing some tags that always change
+       (like dates) from a file that need to be compared to another file.'''
+    def __init__(self, xmlTagsToIgnore, xmlAttrsToIgnore):
+        ContentHandler.__init__(self)
+        self.res = u'<?xml version="1.0" encoding="UTF-8"?>'
+        self.namespaces = {} # ~{s_namespaceUri:s_namespaceName}~
+        self.indentLevel = -1
+        self.tabWidth = 3
+        self.tagsToIgnore = xmlTagsToIgnore
+        self.attrsToIgnore = xmlAttrsToIgnore
+        self.ignoring = False # Some content must be ignored, and not dumped
+        # into the result.
+    def isIgnorable(self, elem):
+        '''Is p_elem an ignorable element ?'''
+        res = False
+        for tagName in self.tagsToIgnore:
+            if isinstance(tagName, list) or isinstance(tagName, tuple):
+                # We have a namespace
+                nsUri, elemName = tagName
+                try:
+                    nsName = self.ns(nsUri)
+                    elemFullName = '%s:%s' % (nsName, elemName)
+                except KeyError:
+                    elemFullName = ''
+            else:
+                # No namespace
+                elemFullName = tagName
+            if elemFullName == elem:
+                res = True
+                break
+        return res
+    def setDocumentLocator(self, locator):
+        self.locator = locator
+    def endDocument(self):
+        pass
+    def dumpSpaces(self):
+        self.res += '\n' + (' ' * self.indentLevel * self.tabWidth)
+    def manageNamespaces(self, attrs):
+        '''Manage namespaces definitions encountered in attrs'''
+        for attrName, attrValue in attrs.items():
+            if attrName.startswith('xmlns:'):
+                self.namespaces[attrValue] = attrName[6:]
+    def ns(self, nsUri):
+        return self.namespaces[nsUri]
+    def startElement(self, elem, attrs):
+        self.manageNamespaces(attrs)
+        # Do we enter into a ignorable element ?
+        if self.isIgnorable(elem):
+            self.ignoring = True
+        else:
+            if not self.ignoring:
+                self.indentLevel += 1
+                self.dumpSpaces()
+                self.res += '<%s' % elem
+                attrsNames = attrs.keys()
+                attrsNames.sort()
+                for attrToIgnore in self.attrsToIgnore:
+                    if attrToIgnore in attrsNames:
+                        attrsNames.remove(attrToIgnore)
+                for attrName in attrsNames:
+                    self.res += ' %s="%s"' % (attrName, attrs[attrName])
+                self.res += '>'
+    def endElement(self, elem):
+        if self.isIgnorable(elem):
+            self.ignoring = False
+        else:
+            if not self.ignoring:
+                self.dumpSpaces()
+                self.indentLevel -= 1
+                self.res += '</%s>' % elem
+    def characters(self, content):
+        if not self.ignoring:
+            self.res += content.replace('\n', '')
+
+# ------------------------------------------------------------------------------
+class XmlComparator:
+    '''Compares 2 XML files and produces a diff.'''
+    def __init__(self, fileNameA, fileNameB, areXml=True, xmlTagsToIgnore=(),
+        xmlAttrsToIgnore=()):
+        self.fileNameA = fileNameA
+        self.fileNameB = fileNameB
+        self.areXml = areXml # Can also diff non-XML files.
+        self.xmlTagsToIgnore = xmlTagsToIgnore
+        self.xmlAttrsToIgnore = xmlAttrsToIgnore
+
+    def filesAreIdentical(self, report=None, encoding=None):
+        '''Compares the 2 files and returns True if they are identical (if we
+           ignore xmlTagsToIgnore and xmlAttrsToIgnore).
+           If p_report is specified, it must be an instance of
+           appy.shared.test.TestReport; the diffs will be dumped in it.'''
+        # Perform the comparison
+        differ = difflib.Differ()
+        if self.areXml:
+            f = file(self.fileNameA)
+            contentA = f.read()
+            f.close()
+            f = file(self.fileNameB)
+            contentB = f.read()
+            f.close()
+            xmlHandler = XmlHandler(self.xmlTagsToIgnore, self.xmlAttrsToIgnore)
+            xml.sax.parseString(contentA, xmlHandler)
+            contentA = xmlHandler.res.split('\n')
+            xmlHandler = XmlHandler(self.xmlTagsToIgnore, self.xmlAttrsToIgnore)
+            xml.sax.parseString(contentB, xmlHandler)
+            contentB = xmlHandler.res.split('\n')
+        else:
+            f = file(self.fileNameA)
+            contentA = f.readlines()
+            f.close()
+            f = file(self.fileNameB)
+            contentB = f.readlines()
+            f.close()
+        diffResult = list(differ.compare(contentA, contentB))
+        # Analyse, format and report the result.
+        atLeastOneDiff = False
+        lastLinePrinted = False
+        i = -1
+        for line in diffResult:
+            i += 1
+            if line and (line[0] != ' '):
+                if not atLeastOneDiff:
+                    if report:
+                        report.say('Difference(s) detected between files '\
+                            '%s and %s:' % (self.fileNameA, self.fileNameB),
+                            encoding='utf-8')
+                    else:
+                        print 'Differences:'
+                    atLeastOneDiff = True
+                if not lastLinePrinted:
+                    if report: report.say('...')
+                    else: print '...'
+                if self.areXml:
+                    if report: report.say(line, encoding=encoding)
+                    else: print line
+                else:
+                    if report: report.say(line[:-1], encoding=encoding)
+                    else: print line[:-1]
+                lastLinePrinted = True
+            else:
+                lastLinePrinted = False
+        return not atLeastOneDiff
 # ------------------------------------------------------------------------------