501 lines
21 KiB
Python
501 lines
21 KiB
Python
# -*- coding: utf-8 -*-
|
|
# ------------------------------------------------------------------------------
|
|
# Appy is a framework for building applications in the Python language.
|
|
# Copyright (C) 2007 Gaetan Delannay
|
|
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA.
|
|
|
|
# ------------------------------------------------------------------------------
|
|
import os, os.path, re, time, sys, traceback, unicodedata, shutil
|
|
sequenceTypes = (list, tuple)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
class FolderDeleter:
|
|
def delete(dirName):
|
|
'''Recursively deletes p_dirName.'''
|
|
dirName = os.path.abspath(dirName)
|
|
for root, dirs, files in os.walk(dirName, topdown=False):
|
|
for name in files:
|
|
os.remove(os.path.join(root, name))
|
|
for name in dirs:
|
|
os.rmdir(os.path.join(root, name))
|
|
os.rmdir(dirName)
|
|
delete = staticmethod(delete)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
extsToClean = ('.pyc', '.pyo', '.fsz', '.deltafsz', '.dat', '.log')
|
|
def cleanFolder(folder, exts=extsToClean, verbose=False):
|
|
'''This function allows to remove, in p_folder and subfolders, any file
|
|
whose extension is in p_exts.'''
|
|
if verbose: print 'Cleaning folder', folder, '...'
|
|
# Remove files with an extension listed in exts
|
|
for root, dirs, files in os.walk(folder):
|
|
for fileName in files:
|
|
ext = os.path.splitext(fileName)[1]
|
|
if (ext in exts) or ext.endswith('~'):
|
|
fileToRemove = os.path.join(root, fileName)
|
|
if verbose: print 'Removing %s...' % fileToRemove
|
|
os.remove(fileToRemove)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
def copyFolder(source, dest, cleanDest=False):
|
|
'''Copies the content of folder p_source to folder p_dest. p_dest is
|
|
created, with intermediary subfolders if required. If p_cleanDest is
|
|
True, it removes completely p_dest if it existed. Else, content of
|
|
p_source will be added to possibly existing content in p_dest, excepted
|
|
if file names corresponds. In this case, file in p_source will overwrite
|
|
file in p_dest.'''
|
|
dest = os.path.abspath(dest)
|
|
# Delete the dest folder if required
|
|
if os.path.exists(dest) and cleanDest:
|
|
FolderDeleter.delete(dest)
|
|
# Create the dest folder if it does not exist
|
|
if not os.path.exists(dest):
|
|
os.makedirs(dest)
|
|
# Copy the content of p_source to p_dest.
|
|
for name in os.listdir(source):
|
|
sourceName = os.path.join(source, name)
|
|
destName = os.path.join(dest, name)
|
|
if os.path.isfile(sourceName):
|
|
# Copy a single file
|
|
shutil.copy(sourceName, destName)
|
|
elif os.path.isdir(sourceName):
|
|
# Copy a subfolder (recursively)
|
|
copyFolder(sourceName, destName)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
def encodeData(data, encoding=None):
|
|
'''Applies some p_encoding to string p_data, but only if an p_encoding is
|
|
specified.'''
|
|
if not encoding: return data
|
|
return data.encode(encoding)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
def copyData(data, target, targetMethod, type='string', encoding=None,
|
|
chunkSize=1024):
|
|
'''Copies p_data to a p_target, using p_targetMethod. For example, it copies
|
|
p_data which is a string containing the binary content of a file, to
|
|
p_target, which can be a HTTP connection or a file object.
|
|
|
|
p_targetMethod can be "write" (files) or "send" (HTTP connections) or ...
|
|
p_type can be "string", "file" or "zope". In the latter case it is an
|
|
instance of OFS.Image.File. If p_type is "file", one may, in p_chunkSize,
|
|
specify the amount of bytes transmitted at a time.
|
|
|
|
If an p_encoding is specified, it is applied on p_data before copying.
|
|
|
|
Note that if the p_target is a Python file, it must be opened in a way
|
|
that is compatible with the content of p_data, ie file('myFile.doc','wb')
|
|
if content is binary.'''
|
|
dump = getattr(target, targetMethod)
|
|
if not type or (type == 'string'): dump(encodeData(data, encoding))
|
|
elif type == 'file':
|
|
while True:
|
|
chunk = data.read(chunkSize)
|
|
if not chunk: break
|
|
dump(encodeData(chunk, encoding))
|
|
elif type == 'zope':
|
|
# A OFS.Image.File instance can be split into several chunks
|
|
if isinstance(data.data, basestring): # One chunk
|
|
dump(encodeData(data.data, encoding))
|
|
else:
|
|
# Several chunks
|
|
data = data.data
|
|
while data is not None:
|
|
dump(encodeData(data.data, encoding))
|
|
data = data.next
|
|
|
|
# ------------------------------------------------------------------------------
|
|
class Traceback:
|
|
'''Dumps the last traceback into a string.'''
|
|
def get():
|
|
res = ''
|
|
excType, excValue, tb = sys.exc_info()
|
|
tbLines = traceback.format_tb(tb)
|
|
for tbLine in tbLines:
|
|
res += ' %s' % tbLine
|
|
res += ' %s: %s' % (str(excType), str(excValue))
|
|
return res
|
|
get = staticmethod(get)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
def getOsTempFolder():
|
|
tmp = '/tmp'
|
|
if os.path.exists(tmp) and os.path.isdir(tmp):
|
|
res = tmp
|
|
elif os.environ.has_key('TMP'):
|
|
res = os.environ['TMP']
|
|
elif os.environ.has_key('TEMP'):
|
|
res = os.environ['TEMP']
|
|
else:
|
|
raise "Sorry, I can't find a temp folder on your machine."
|
|
return res
|
|
|
|
def getTempFileName(prefix='', extension=''):
|
|
'''Returns the absolute path to a unique file name in the OS temp folder.
|
|
The caller will then be able to create a file with this name.
|
|
|
|
A p_prefix to this file can be provided. If an p_extension is provided,
|
|
it will be appended to the name. Both dotted and not dotted versions
|
|
of p_extension are allowed (ie, ".pdf" or "pdf").'''
|
|
res = '%s/%s_%f' % (getOsTempFolder(), prefix, time.time())
|
|
if extension:
|
|
if extension.startswith('.'): res += extension
|
|
else: res += '.' + extension
|
|
return res
|
|
|
|
# ------------------------------------------------------------------------------
|
|
def executeCommand(cmd):
|
|
'''Executes command p_cmd and returns the content of its stderr.'''
|
|
childStdIn, childStdOut, childStdErr = os.popen3(cmd)
|
|
res = childStdErr.read()
|
|
childStdIn.close(); childStdOut.close(); childStdErr.close()
|
|
return res
|
|
|
|
# ------------------------------------------------------------------------------
|
|
unwantedChars = ('\\', '/', ':', '*', '?', '"', '<', '>', '|', ' ', '\t')
|
|
alphaRex = re.compile('[a-zA-Z]')
|
|
alphanumRex = re.compile('[a-zA-Z0-9]')
|
|
def normalizeString(s, usage='fileName'):
|
|
'''Returns a version of string p_s whose special chars (like accents) have
|
|
been replaced with normal chars. Moreover, if p_usage is:
|
|
* fileName: it removes any char that can't be part of a file name;
|
|
* alphanum: it removes any non-alphanumeric char;
|
|
* alpha: it removes any non-letter char.
|
|
'''
|
|
# We work in unicode. Convert p_s to unicode if not unicode.
|
|
if isinstance(s, str): s = s.decode('utf-8')
|
|
elif not isinstance(s, unicode): s = unicode(s)
|
|
# Remove any special char like accents.
|
|
s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
|
|
# Remove any other char, depending on p_usage.
|
|
if usage == 'fileName':
|
|
# Remove any char that can't be found within a file name under
|
|
# Windows or that could lead to problems with OpenOffice.
|
|
res = ''
|
|
for char in s:
|
|
if char not in unwantedChars: res += char
|
|
elif usage.startswith('alpha'):
|
|
exec 'rex = %sRex' % usage
|
|
res = ''
|
|
for char in s:
|
|
if rex.match(char): res += char
|
|
else:
|
|
res = s
|
|
return res
|
|
|
|
def formatNumber(n, sep=',', precision=2, tsep=' '):
|
|
'''Returns a string representation of number p_n, which can be a float
|
|
or integer. p_sep is the decimal separator to use. p_precision is the
|
|
number of digits to keep in the decimal part for producing a nice rounded
|
|
string representation. p_tsep is the "thousands" separator.'''
|
|
if n == None: return ''
|
|
# Manage precision
|
|
if precision == None:
|
|
res = str(n)
|
|
else:
|
|
format = '%%.%df' % precision
|
|
res = format % n
|
|
# Use the correct decimal separator
|
|
res = res.replace('.', sep)
|
|
# Insert p_tsep every 3 chars in the integer part of the number
|
|
splitted = res.split(sep)
|
|
res = ''
|
|
if len(splitted[0]) < 4: res = splitted[0]
|
|
else:
|
|
i = len(splitted[0])-1
|
|
j = 0
|
|
while i >= 0:
|
|
j += 1
|
|
res = splitted[0][i] + res
|
|
if (j % 3) == 0:
|
|
res = tsep + res
|
|
i -= 1
|
|
# Add the decimal part if not 0
|
|
if len(splitted) > 1:
|
|
try:
|
|
decPart = int(splitted[1])
|
|
if decPart != 0:
|
|
res += sep + str(decPart)
|
|
except ValueError:
|
|
# This exception may occur when the float value has an "exp"
|
|
# part, like in this example: 4.345e-05
|
|
res += sep + splitted[1]
|
|
return res
|
|
|
|
# ------------------------------------------------------------------------------
|
|
toLower = {'Ç':'ç','Ù':'ù','Û':'û','Ü':'ü','Î':'î','Ï':'ï','Ô':'ô','Ö':'ö',
|
|
'É':'é','È':'è','Ê':'ê','Ë':'ë','À':'à','Â':'â','Ä':'ä'}
|
|
toUpper = {'ç':'Ç','ù':'Ù','û':'Û','ü':'Ü','î':'Î','ï':'Ï','ô':'Ô','ö':'Ö',
|
|
'é':'É','è':'È','ê':'Ê','ë':'Ë','à':'À','â':'Â','ä':'Ä'}
|
|
|
|
def lower(s):
|
|
'''French-accents-aware variant of string.lower.'''
|
|
res = s.lower()
|
|
for upp, low in toLower.iteritems():
|
|
if upp in res: res = res.replace(upp, low)
|
|
return res
|
|
|
|
def upper(s):
|
|
'''French-accents-aware variant of string.upper.'''
|
|
res = s.upper()
|
|
for low, upp in toUpper.iteritems():
|
|
if low in res: res = res.replace(low, upp)
|
|
return res
|
|
|
|
# ------------------------------------------------------------------------------
|
|
typeLetters = {'b': bool, 'i': int, 'j': long, 'f':float, 's':str, 'u':unicode,
|
|
'l': list, 'd': dict}
|
|
exts = {'py': ('.py', '.vpy', '.cpy'), 'pt': ('.pt', '.cpt')}
|
|
|
|
class CodeAnalysis:
|
|
'''This class holds information about some code analysis (line counts) that
|
|
spans some folder hierarchy.'''
|
|
def __init__(self, name):
|
|
self.name = name # Let's give a name for the analysis
|
|
self.numberOfFiles = 0 # The total number of analysed files
|
|
self.emptyLines = 0 # The number of empty lines within those files
|
|
self.commentLines = 0 # The number of comment lines
|
|
# A code line is defined as anything that is not an empty or comment
|
|
# line.
|
|
self.codeLines = 0
|
|
|
|
def numberOfLines(self):
|
|
'''Computes the total number of lines within analysed files.'''
|
|
return self.emptyLines + self.commentLines + self.codeLines
|
|
|
|
def analyseZptFile(self, theFile):
|
|
'''Analyses the ZPT file named p_fileName.'''
|
|
inDoc = False
|
|
for line in theFile:
|
|
stripped = line.strip()
|
|
# Manage a comment
|
|
if not inDoc and (line.find('<tal:comment ') != -1):
|
|
inDoc = True
|
|
if inDoc:
|
|
self.commentLines += 1
|
|
if line.find('</tal:comment>') != -1:
|
|
inDoc = False
|
|
continue
|
|
# Manage an empty line
|
|
if not stripped:
|
|
self.emptyLines += 1
|
|
else:
|
|
self.codeLines += 1
|
|
|
|
docSeps = ('"""', "'''")
|
|
def isPythonDoc(self, line, start, isStart=False):
|
|
'''Returns True if we find, in p_line, the start of a docstring (if
|
|
p_start is True) or the end of a docstring (if p_start is False).
|
|
p_isStart indicates if p_line is the start of the docstring.'''
|
|
if start:
|
|
res = line.startswith(self.docSeps[0]) or \
|
|
line.startswith(self.docSeps[1])
|
|
else:
|
|
sepOnly = (line == self.docSeps[0]) or (line == self.docSeps[1])
|
|
if sepOnly:
|
|
# If the line contains the separator only, is this the start or
|
|
# the end of the docstring?
|
|
if isStart: res = False
|
|
else: res = True
|
|
else:
|
|
res = line.endswith(self.docSeps[0]) or \
|
|
line.endswith(self.docSeps[1])
|
|
return res
|
|
|
|
def analysePythonFile(self, theFile):
|
|
'''Analyses the Python file named p_fileName.'''
|
|
# Are we in a docstring ?
|
|
inDoc = False
|
|
for line in theFile:
|
|
stripped = line.strip()
|
|
# Manage a line that is within a docstring
|
|
inDocStart = False
|
|
if not inDoc and self.isPythonDoc(stripped, start=True):
|
|
inDoc = True
|
|
inDocStart = True
|
|
if inDoc:
|
|
self.commentLines += 1
|
|
if self.isPythonDoc(stripped, start=False, isStart=inDocStart):
|
|
inDoc = False
|
|
continue
|
|
# Manage an empty line
|
|
if not stripped:
|
|
self.emptyLines += 1
|
|
continue
|
|
# Manage a comment line
|
|
if line.startswith('#'):
|
|
self.commentLines += 1
|
|
continue
|
|
# If we are here, we have a code line.
|
|
self.codeLines += 1
|
|
|
|
def analyseFile(self, fileName):
|
|
'''Analyses file named p_fileName.'''
|
|
self.numberOfFiles += 1
|
|
theFile = file(fileName)
|
|
ext = os.path.splitext(fileName)[1]
|
|
if ext in exts['py']: self.analysePythonFile(theFile)
|
|
elif ext in exts['pt']: self.analyseZptFile(theFile)
|
|
theFile.close()
|
|
|
|
def printReport(self):
|
|
'''Returns the analysis report as a string, only if there is at least
|
|
one analysed line.'''
|
|
lines = self.numberOfLines()
|
|
if not lines: return
|
|
commentRate = (self.commentLines / float(lines)) * 100.0
|
|
blankRate = (self.emptyLines / float(lines)) * 100.0
|
|
print '%s: %d files, %d lines (%.0f%% comments, %.0f%% blank)' % \
|
|
(self.name, self.numberOfFiles, lines, commentRate, blankRate)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
class LinesCounter:
|
|
'''Counts and classifies the lines of code within a folder hierarchy.'''
|
|
defaultExcludes = ('%s.svn' % os.sep, '%s.bzr' % os.sep, '%stmp' % os.sep,
|
|
'%stemp' % os.sep)
|
|
|
|
def __init__(self, folderOrModule, excludes=None):
|
|
if isinstance(folderOrModule, basestring):
|
|
# It is the path of some folder
|
|
self.folder = folderOrModule
|
|
else:
|
|
# It is a Python module
|
|
self.folder = os.path.dirname(folderOrModule.__file__)
|
|
# These dicts will hold information about analysed files
|
|
self.python = {False: CodeAnalysis('Python'),
|
|
True: CodeAnalysis('Python (test)')}
|
|
self.zpt = {False: CodeAnalysis('ZPT'),
|
|
True: CodeAnalysis('ZPT (test)')}
|
|
# Are we currently analysing real or test code?
|
|
self.inTest = False
|
|
# Which paths to exclude from the analysis?
|
|
self.excludes = list(self.defaultExcludes)
|
|
if excludes: self.excludes += excludes
|
|
|
|
def printReport(self):
|
|
'''Displays on stdout a small analysis report about self.folder.'''
|
|
for zone in (False, True): self.python[zone].printReport()
|
|
for zone in (False, True): self.zpt[zone].printReport()
|
|
|
|
def isExcluded(self, path):
|
|
'''Must p_path be excluded from the analysis?'''
|
|
for excl in self.excludes:
|
|
if excl in path: return True
|
|
|
|
def run(self):
|
|
'''Let's start the analysis of self.folder.'''
|
|
# The test markers will allow us to know if we are analysing test code
|
|
# or real code within a given part of self.folder code hierarchy.
|
|
testMarker1 = '%stest%s' % (os.sep, os.sep)
|
|
testMarker2 = '%stest' % os.sep
|
|
testMarker3 = '%stests%s' % (os.sep, os.sep)
|
|
testMarker4 = '%stests' % os.sep
|
|
j = os.path.join
|
|
for root, folders, files in os.walk(self.folder):
|
|
if self.isExcluded(root): continue
|
|
# Are we in real code or in test code ?
|
|
self.inTest = False
|
|
if root.endswith(testMarker2) or (root.find(testMarker1) != -1) or \
|
|
root.endswith(testMarker4) or (root.find(testMarker3) != -1):
|
|
self.inTest = True
|
|
# Scan the files in this folder
|
|
for fileName in files:
|
|
ext = os.path.splitext(fileName)[1]
|
|
if ext in exts['py']:
|
|
self.python[self.inTest].analyseFile(j(root, fileName))
|
|
elif ext in exts['pt']:
|
|
self.zpt[self.inTest].analyseFile(j(root, fileName))
|
|
self.printReport()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
CONVERSION_ERROR = 'An error occurred while executing command "%s". %s'
|
|
class FileWrapper:
|
|
'''When you get, from an appy object, the value of a File attribute, you
|
|
get an instance of this class.'''
|
|
def __init__(self, zopeFile):
|
|
'''This constructor is only used by Appy to create a nice File instance
|
|
from a Zope corresponding instance (p_zopeFile). If you need to
|
|
create a new file and assign it to a File attribute, use the
|
|
attribute setter, do not create yourself an instance of this
|
|
class.'''
|
|
d = self.__dict__
|
|
d['_zopeFile'] = zopeFile # Not for you!
|
|
d['name'] = zopeFile.filename
|
|
d['content'] = zopeFile.data
|
|
d['mimeType'] = zopeFile.content_type
|
|
d['size'] = zopeFile.size # In bytes
|
|
|
|
def __setattr__(self, name, v):
|
|
d = self.__dict__
|
|
if name == 'name':
|
|
self._zopeFile.filename = v
|
|
d['name'] = v
|
|
elif name == 'content':
|
|
self._zopeFile.update_data(v, self.mimeType, len(v))
|
|
d['content'] = v
|
|
d['size'] = len(v)
|
|
elif name == 'mimeType':
|
|
self._zopeFile.content_type = self.mimeType = v
|
|
else:
|
|
raise 'Impossible to set attribute %s. "Settable" attributes ' \
|
|
'are "name", "content" and "mimeType".' % name
|
|
|
|
def dump(self, filePath=None, format=None, tool=None):
|
|
'''Writes the file on disk. If p_filePath is specified, it is the
|
|
path name where the file will be dumped; folders mentioned in it
|
|
must exist. If not, the file will be dumped in the OS temp folder.
|
|
The absolute path name of the dumped file is returned.
|
|
If an error occurs, the method returns None. If p_format is
|
|
specified, OpenOffice will be called for converting the dumped file
|
|
to the desired format. In this case, p_tool, a Appy tool, must be
|
|
provided. Indeed, any Appy tool contains parameters for contacting
|
|
OpenOffice in server mode.'''
|
|
if not filePath:
|
|
filePath = '%s/file%f.%s' % (getOsTempFolder(), time.time(),
|
|
normalizeString(self.name))
|
|
f = file(filePath, 'w')
|
|
if self.content.__class__.__name__ == 'Pdata':
|
|
# The file content is splitted in several chunks.
|
|
f.write(self.content.data)
|
|
nextPart = self.content.next
|
|
while nextPart:
|
|
f.write(nextPart.data)
|
|
nextPart = nextPart.next
|
|
else:
|
|
# Only one chunk
|
|
f.write(self.content)
|
|
f.close()
|
|
if format:
|
|
if not tool: return
|
|
# Convert the dumped file using OpenOffice
|
|
errorMessage = tool.convert(filePath, format)
|
|
# Even if we have an "error" message, it could be a simple warning.
|
|
# So we will continue here and, as a subsequent check for knowing if
|
|
# an error occurred or not, we will test the existence of the
|
|
# converted file (see below).
|
|
os.remove(filePath)
|
|
# Return the name of the converted file.
|
|
baseName, ext = os.path.splitext(filePath)
|
|
if (ext == '.%s' % format):
|
|
filePath = '%s.res.%s' % (baseName, format)
|
|
else:
|
|
filePath = '%s.%s' % (baseName, format)
|
|
if not os.path.exists(filePath):
|
|
tool.log(CONVERSION_ERROR % (cmd, errorMessage), type='error')
|
|
return
|
|
return filePath
|
|
# ------------------------------------------------------------------------------
|