202 lines
9 KiB
Python
202 lines
9 KiB
Python
|
# ------------------------------------------------------------------------------
|
||
|
|
# Appy is a framework for building applications in the Python language.
|
||
|
|
# Copyright (C) 2007 Gaetan Delannay
|
||
|
|
|
||
|
|
# This program is free software; you can redistribute it and/or
|
||
|
|
# modify it under the terms of the GNU General Public License
|
||
|
|
# as published by the Free Software Foundation; either version 2
|
||
|
|
# of the License, or (at your option) any later version.
|
||
|
|
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
|
||
|
|
# You should have received a copy of the GNU General Public License
|
||
|
|
# along with this program; if not, write to the Free Software
|
||
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA.
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------------------
|
||
|
|
from appy.shared import UnmarshalledObject
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------------------
|
||
|
|
WRONG_LINE = 'Line number %d in file %s does not have the right number of ' \
|
||
|
|
'fields.'
|
||
|
|
|
||
|
|
class CsvParser:
|
||
|
|
'''This class reads a CSV file and creates a list of Python objects from it.
|
||
|
|
The first line of the CSV file must declare the format of the following
|
||
|
|
lines, which are 'data' lines. For example, if the first line of the file
|
||
|
|
is
|
||
|
|
|
||
|
|
id,roles*,password
|
||
|
|
|
||
|
|
Then subsequent lines in the CSV need to conform to this syntax. Field
|
||
|
|
separator will be the comma. Result of method 'parse' will be a list of
|
||
|
|
Python objects, each one having attributes id, roles and password.
|
||
|
|
Attributes declared with a star (like 'roles') are lists. An empty value
|
||
|
|
will produce an empty list in the resulting object; several values need
|
||
|
|
to be separated with the '+' sign. Here are some examples of valid 'data'
|
||
|
|
lines for the first line above:
|
||
|
|
|
||
|
|
gdy,,
|
||
|
|
gdy,MeetingManager,abc
|
||
|
|
gdy,MeetingManager+MeetingMember,abc
|
||
|
|
|
||
|
|
In the first (and subsequent) line(s), you may choose among the following
|
||
|
|
separators: , : ; |
|
||
|
|
'''
|
||
|
|
separators = [',', ':', ';', '|']
|
||
|
|
typeLetters = {'i': int, 'f': float, 's': str, 'b': bool}
|
||
|
|
def __init__(self, fileName, references={}, klass=None):
|
||
|
|
self.fileName = fileName
|
||
|
|
self.res = [] # The resulting list of Python objects.
|
||
|
|
self.sep = None
|
||
|
|
self.attributes = None # The list of attributes corresponding to
|
||
|
|
# CSV columns.
|
||
|
|
self.attributesFlags = None # Here we now if every attribute is a list
|
||
|
|
# (True) of not (False).
|
||
|
|
self.attributesTypes = None # Here we now the type of the attribute (if
|
||
|
|
# the attribute is a list it denotes the type of every item in the
|
||
|
|
# list): string, integer, float, boolean.
|
||
|
|
self.references = references
|
||
|
|
self.klass = klass # If a klass is given here, instead of creating
|
||
|
|
# UnmarshalledObject instances we will create instances of this class.
|
||
|
|
# But be careful: we will not call the constructor of this class. We
|
||
|
|
# will simply create instances of UnmarshalledObject and dynamically
|
||
|
|
# change the class of created instances to this class.
|
||
|
|
|
||
|
|
def identifySeparator(self, line):
|
||
|
|
'''What is the separator used in this file?'''
|
||
|
|
maxLength = 0
|
||
|
|
res = None
|
||
|
|
for sep in self.separators:
|
||
|
|
newLength = len(line.split(sep))
|
||
|
|
if newLength > maxLength:
|
||
|
|
maxLength = newLength
|
||
|
|
res = sep
|
||
|
|
self.sep = res
|
||
|
|
|
||
|
|
def identifyAttributes(self, line):
|
||
|
|
self.attributes = line.split(self.sep)
|
||
|
|
self.attributesFlags = [False] * len(self.attributes)
|
||
|
|
self.attributesTypes = [str] * len(self.attributes)
|
||
|
|
i = -1
|
||
|
|
for attr in self.attributes:
|
||
|
|
i += 1
|
||
|
|
# Is this attribute mono- or multi-valued?
|
||
|
|
if attr.endswith('*'):
|
||
|
|
self.attributesFlags[i] = True
|
||
|
|
attrNoFlag = attr.strip('*')
|
||
|
|
attrInfo = attrNoFlag.split('-')
|
||
|
|
# What is the type of value(s) for this attribute ?
|
||
|
|
if (len(attrInfo) == 2) and (attrInfo[1] in self.typeLetters):
|
||
|
|
self.attributesTypes[i] = self.typeLetters[attrInfo[1]]
|
||
|
|
# Remove trailing stars
|
||
|
|
self.attributes = [a.strip('*').split('-')[0] for a in self.attributes]
|
||
|
|
|
||
|
|
def resolveReference(self, attrName, refId):
|
||
|
|
'''Finds, in self.reference, the object having p_refId.'''
|
||
|
|
refObjects, refAttrName = self.references[attrName]
|
||
|
|
res = None
|
||
|
|
for refObject in refObjects:
|
||
|
|
if getattr(refObject, refAttrName) == refId:
|
||
|
|
res = refObject
|
||
|
|
break
|
||
|
|
return res
|
||
|
|
|
||
|
|
def convertValue(self, value, basicType):
|
||
|
|
'''Converts the atomic p_value which is a string into some other atomic
|
||
|
|
Python type specified in p_basicType (int, float, ...).'''
|
||
|
|
if (basicType != str) and (basicType != unicode):
|
||
|
|
try:
|
||
|
|
exec 'res = %s' % str(value)
|
||
|
|
except SyntaxError, se:
|
||
|
|
res = None
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
exec 'res = """%s"""' % str(value)
|
||
|
|
except SyntaxError, se:
|
||
|
|
try:
|
||
|
|
exec "res = '''%s'''" % str(value)
|
||
|
|
except SyntaxError, se:
|
||
|
|
res = None
|
||
|
|
return res
|
||
|
|
|
||
|
|
def parse(self):
|
||
|
|
'''Parses the CSV file named self.fileName and creates a list of
|
||
|
|
corresponding Python objects (UnmarshalledObject instances). Among
|
||
|
|
object fields, some may be references. If it is the case, you may
|
||
|
|
specify in p_references a dict of referred objects. The parser will
|
||
|
|
then replace string values of some fields (which are supposed to be
|
||
|
|
ids of referred objects) with corresponding objects in p_references.
|
||
|
|
|
||
|
|
How does this work? p_references must be a dictionary:
|
||
|
|
- keys correspond to field names of the current object;
|
||
|
|
- values are 2-tuples:
|
||
|
|
* 1st value is the list of available referred objects;
|
||
|
|
* 2nd value is the name of the attribute on those objects that
|
||
|
|
stores their ID.
|
||
|
|
'''
|
||
|
|
# The first pass parses the file and creates the Python object
|
||
|
|
f = file(self.fileName)
|
||
|
|
firstLine = True
|
||
|
|
lineNb = 0
|
||
|
|
for line in f:
|
||
|
|
lineNb += 1
|
||
|
|
line = line.strip()
|
||
|
|
if not line: continue
|
||
|
|
if firstLine:
|
||
|
|
# The first line declares the structure of the following 'data'
|
||
|
|
# lines.
|
||
|
|
self.identifySeparator(line)
|
||
|
|
self.identifyAttributes(line)
|
||
|
|
firstLine = False
|
||
|
|
else:
|
||
|
|
# Add an object corresponding to this line.
|
||
|
|
lineObject = UnmarshalledObject()
|
||
|
|
if self.klass:
|
||
|
|
lineObject.__class__ = self.klass
|
||
|
|
i = -1
|
||
|
|
# Do we get the right number of field values on this line ?
|
||
|
|
attrValues = line.split(self.sep)
|
||
|
|
if len(attrValues) != len(self.attributes):
|
||
|
|
raise WRONG_LINE % (lineNb, self.fileName)
|
||
|
|
for attrValue in line.split(self.sep):
|
||
|
|
i += 1
|
||
|
|
theValue = attrValue
|
||
|
|
vType = self.attributesTypes[i]
|
||
|
|
if self.attributesFlags[i]:
|
||
|
|
# The attribute is multi-valued
|
||
|
|
if not attrValue:
|
||
|
|
theValue = []
|
||
|
|
elif '+' in theValue:
|
||
|
|
theValue = [self.convertValue(v, vType) \
|
||
|
|
for v in attrValue.split('+')]
|
||
|
|
else:
|
||
|
|
theValue = [self.convertValue(theValue, vType)]
|
||
|
|
else:
|
||
|
|
# The attribute is mono-valued
|
||
|
|
theValue = self.convertValue(theValue, vType)
|
||
|
|
setattr(lineObject, self.attributes[i], theValue)
|
||
|
|
self.res.append(lineObject)
|
||
|
|
f.close()
|
||
|
|
# The second pass resolves the p_references if any
|
||
|
|
for attrName, refInfo in self.references.iteritems():
|
||
|
|
if attrName in self.attributes:
|
||
|
|
# Replace ID with real object from p_references
|
||
|
|
for obj in self.res:
|
||
|
|
attrValue = getattr(obj, attrName)
|
||
|
|
if isinstance(attrValue, list) or \
|
||
|
|
isinstance(attrValue, tuple):
|
||
|
|
# Multiple values to resolve
|
||
|
|
newValue = []
|
||
|
|
for v in attrValue:
|
||
|
|
newValue.append(self.resolveReference(attrName,v))
|
||
|
|
else:
|
||
|
|
# Only one value to resolve
|
||
|
|
newValue = self.resolveReference(attrName, attrValue)
|
||
|
|
setattr(obj, attrName, newValue)
|
||
|
|
return self.res
|
||
|
|
# ------------------------------------------------------------------------------
|