# -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # Appy is a framework for building applications in the Python language. # Copyright (C) 2007 Gaetan Delannay # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA. # ------------------------------------------------------------------------------ import os, os.path, re, time, sys, traceback, unicodedata, shutil sequenceTypes = (list, tuple) # ------------------------------------------------------------------------------ class FolderDeleter: def delete(dirName): '''Recursively deletes p_dirName.''' dirName = os.path.abspath(dirName) for root, dirs, files in os.walk(dirName, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(dirName) delete = staticmethod(delete) # ------------------------------------------------------------------------------ extsToClean = ('.pyc', '.pyo', '.fsz', '.deltafsz', '.dat', '.log') def cleanFolder(folder, exts=extsToClean, folders=(), verbose=False): '''This function allows to remove, in p_folder and subfolders, any file whose extension is in p_exts, and any folder whose name is in p_folders.''' if verbose: print 'Cleaning folder', folder, '...' # Remove files with an extension listed in p_exts if exts: for root, dirs, files in os.walk(folder): for fileName in files: ext = os.path.splitext(fileName)[1] if (ext in exts) or ext.endswith('~'): fileToRemove = os.path.join(root, fileName) if verbose: print 'Removing file %s...' % fileToRemove os.remove(fileToRemove) # Remove folders whose names are in p_folders. if folders: for root, dirs, files in os.walk(folder): for folderName in dirs: if folderName in folders: toDelete = os.path.join(root, folderName) if verbose: print 'Removing folder %s...' % toDelete FolderDeleter.delete(toDelete) # ------------------------------------------------------------------------------ def copyFolder(source, dest, cleanDest=False): '''Copies the content of folder p_source to folder p_dest. p_dest is created, with intermediary subfolders if required. If p_cleanDest is True, it removes completely p_dest if it existed. Else, content of p_source will be added to possibly existing content in p_dest, excepted if file names corresponds. In this case, file in p_source will overwrite file in p_dest.''' dest = os.path.abspath(dest) # Delete the dest folder if required if os.path.exists(dest) and cleanDest: FolderDeleter.delete(dest) # Create the dest folder if it does not exist if not os.path.exists(dest): os.makedirs(dest) # Copy the content of p_source to p_dest. for name in os.listdir(source): sourceName = os.path.join(source, name) destName = os.path.join(dest, name) if os.path.isfile(sourceName): # Copy a single file shutil.copy(sourceName, destName) elif os.path.isdir(sourceName): # Copy a subfolder (recursively) copyFolder(sourceName, destName) # ------------------------------------------------------------------------------ def encodeData(data, encoding=None): '''Applies some p_encoding to string p_data, but only if an p_encoding is specified.''' if not encoding: return data return data.encode(encoding) # ------------------------------------------------------------------------------ def copyData(data, target, targetMethod, type='string', encoding=None, chunkSize=1024): '''Copies p_data to a p_target, using p_targetMethod. For example, it copies p_data which is a string containing the binary content of a file, to p_target, which can be a HTTP connection or a file object. p_targetMethod can be "write" (files) or "send" (HTTP connections) or ... p_type can be "string", "file" or "zope". In the latter case it is an instance of OFS.Image.File. If p_type is "file", one may, in p_chunkSize, specify the amount of bytes transmitted at a time. If an p_encoding is specified, it is applied on p_data before copying. Note that if the p_target is a Python file, it must be opened in a way that is compatible with the content of p_data, ie file('myFile.doc','wb') if content is binary.''' dump = getattr(target, targetMethod) if not type or (type == 'string'): dump(encodeData(data, encoding)) elif type == 'file': while True: chunk = data.read(chunkSize) if not chunk: break dump(encodeData(chunk, encoding)) elif type == 'zope': # A OFS.Image.File instance can be split into several chunks if isinstance(data.data, basestring): # One chunk dump(encodeData(data.data, encoding)) else: # Several chunks data = data.data while data is not None: dump(encodeData(data.data, encoding)) data = data.next # ------------------------------------------------------------------------------ class Traceback: '''Dumps the last traceback into a string.''' def get(): res = '' excType, excValue, tb = sys.exc_info() tbLines = traceback.format_tb(tb) for tbLine in tbLines: res += ' %s' % tbLine res += ' %s: %s' % (str(excType), str(excValue)) return res get = staticmethod(get) # ------------------------------------------------------------------------------ def getOsTempFolder(): tmp = '/tmp' if os.path.exists(tmp) and os.path.isdir(tmp): res = tmp elif os.environ.has_key('TMP'): res = os.environ['TMP'] elif os.environ.has_key('TEMP'): res = os.environ['TEMP'] else: raise "Sorry, I can't find a temp folder on your machine." return res def getTempFileName(prefix='', extension=''): '''Returns the absolute path to a unique file name in the OS temp folder. The caller will then be able to create a file with this name. A p_prefix to this file can be provided. If an p_extension is provided, it will be appended to the name. Both dotted and not dotted versions of p_extension are allowed (ie, ".pdf" or "pdf").''' res = '%s/%s_%f' % (getOsTempFolder(), prefix, time.time()) if extension: if extension.startswith('.'): res += extension else: res += '.' + extension return res # ------------------------------------------------------------------------------ def executeCommand(cmd): '''Executes command p_cmd and returns the content of its stderr.''' childStdIn, childStdOut, childStdErr = os.popen3(cmd) res = childStdErr.read() childStdIn.close(); childStdOut.close(); childStdErr.close() return res # ------------------------------------------------------------------------------ unwantedChars = ('\\', '/', ':', '*', '?', '"', '<', '>', '|', ' ', '\t') alphaRex = re.compile('[a-zA-Z]') alphanumRex = re.compile('[a-zA-Z0-9]') def normalizeString(s, usage='fileName'): '''Returns a version of string p_s whose special chars (like accents) have been replaced with normal chars. Moreover, if p_usage is: * fileName: it removes any char that can't be part of a file name; * alphanum: it removes any non-alphanumeric char; * alpha: it removes any non-letter char. ''' # We work in unicode. Convert p_s to unicode if not unicode. if isinstance(s, str): s = s.decode('utf-8') elif not isinstance(s, unicode): s = unicode(s) # Remove any special char like accents. s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') # Remove any other char, depending on p_usage. if usage == 'fileName': # Remove any char that can't be found within a file name under # Windows or that could lead to problems with OpenOffice. res = '' for char in s: if char not in unwantedChars: res += char elif usage.startswith('alpha'): exec 'rex = %sRex' % usage res = '' for char in s: if rex.match(char): res += char else: res = s return res def formatNumber(n, sep=',', precision=2, tsep=' '): '''Returns a string representation of number p_n, which can be a float or integer. p_sep is the decimal separator to use. p_precision is the number of digits to keep in the decimal part for producing a nice rounded string representation. p_tsep is the "thousands" separator.''' if n == None: return '' # Manage precision if precision == None: res = str(n) else: format = '%%.%df' % precision res = format % n # Use the correct decimal separator res = res.replace('.', sep) # Insert p_tsep every 3 chars in the integer part of the number splitted = res.split(sep) res = '' if len(splitted[0]) < 4: res = splitted[0] else: i = len(splitted[0])-1 j = 0 while i >= 0: j += 1 res = splitted[0][i] + res if (j % 3) == 0: res = tsep + res i -= 1 # Add the decimal part if not 0 if len(splitted) > 1: try: decPart = int(splitted[1]) if decPart != 0: res += sep + str(decPart) except ValueError: # This exception may occur when the float value has an "exp" # part, like in this example: 4.345e-05 res += sep + splitted[1] return res # ------------------------------------------------------------------------------ toLower = {'Ç':'ç','Ù':'ù','Û':'û','Ü':'ü','Î':'î','Ï':'ï','Ô':'ô','Ö':'ö', 'É':'é','È':'è','Ê':'ê','Ë':'ë','À':'à','Â':'â','Ä':'ä'} toUpper = {'ç':'Ç','ù':'Ù','û':'Û','ü':'Ü','î':'Î','ï':'Ï','ô':'Ô','ö':'Ö', 'é':'É','è':'È','ê':'Ê','ë':'Ë','à':'À','â':'Â','ä':'Ä'} def lower(s): '''French-accents-aware variant of string.lower.''' res = s.lower() for upp, low in toLower.iteritems(): if upp in res: res = res.replace(upp, low) return res def upper(s): '''French-accents-aware variant of string.upper.''' res = s.upper() for low, upp in toUpper.iteritems(): if low in res: res = res.replace(low, upp) return res # ------------------------------------------------------------------------------ typeLetters = {'b': bool, 'i': int, 'j': long, 'f':float, 's':str, 'u':unicode, 'l': list, 'd': dict} exts = {'py': ('.py', '.vpy', '.cpy'), 'pt': ('.pt', '.cpt')} class CodeAnalysis: '''This class holds information about some code analysis (line counts) that spans some folder hierarchy.''' def __init__(self, name): self.name = name # Let's give a name for the analysis self.numberOfFiles = 0 # The total number of analysed files self.emptyLines = 0 # The number of empty lines within those files self.commentLines = 0 # The number of comment lines # A code line is defined as anything that is not an empty or comment # line. self.codeLines = 0 def numberOfLines(self): '''Computes the total number of lines within analysed files.''' return self.emptyLines + self.commentLines + self.codeLines def analyseZptFile(self, theFile): '''Analyses the ZPT file named p_fileName.''' inDoc = False for line in theFile: stripped = line.strip() # Manage a comment if not inDoc and (line.find('') != -1: inDoc = False continue # Manage an empty line if not stripped: self.emptyLines += 1 else: self.codeLines += 1 docSeps = ('"""', "'''") def isPythonDoc(self, line, start, isStart=False): '''Returns True if we find, in p_line, the start of a docstring (if p_start is True) or the end of a docstring (if p_start is False). p_isStart indicates if p_line is the start of the docstring.''' if start: res = line.startswith(self.docSeps[0]) or \ line.startswith(self.docSeps[1]) else: sepOnly = (line == self.docSeps[0]) or (line == self.docSeps[1]) if sepOnly: # If the line contains the separator only, is this the start or # the end of the docstring? if isStart: res = False else: res = True else: res = line.endswith(self.docSeps[0]) or \ line.endswith(self.docSeps[1]) return res def analysePythonFile(self, theFile): '''Analyses the Python file named p_fileName.''' # Are we in a docstring ? inDoc = False for line in theFile: stripped = line.strip() # Manage a line that is within a docstring inDocStart = False if not inDoc and self.isPythonDoc(stripped, start=True): inDoc = True inDocStart = True if inDoc: self.commentLines += 1 if self.isPythonDoc(stripped, start=False, isStart=inDocStart): inDoc = False continue # Manage an empty line if not stripped: self.emptyLines += 1 continue # Manage a comment line if line.startswith('#'): self.commentLines += 1 continue # If we are here, we have a code line. self.codeLines += 1 def analyseFile(self, fileName): '''Analyses file named p_fileName.''' self.numberOfFiles += 1 theFile = file(fileName) ext = os.path.splitext(fileName)[1] if ext in exts['py']: self.analysePythonFile(theFile) elif ext in exts['pt']: self.analyseZptFile(theFile) theFile.close() def printReport(self): '''Returns the analysis report as a string, only if there is at least one analysed line.''' lines = self.numberOfLines() if not lines: return commentRate = (self.commentLines / float(lines)) * 100.0 blankRate = (self.emptyLines / float(lines)) * 100.0 print '%s: %d files, %d lines (%.0f%% comments, %.0f%% blank)' % \ (self.name, self.numberOfFiles, lines, commentRate, blankRate) # ------------------------------------------------------------------------------ class LinesCounter: '''Counts and classifies the lines of code within a folder hierarchy.''' defaultExcludes = ('%s.svn' % os.sep, '%s.bzr' % os.sep, '%stmp' % os.sep, '%stemp' % os.sep) def __init__(self, folderOrModule, excludes=None): if isinstance(folderOrModule, basestring): # It is the path of some folder self.folder = folderOrModule else: # It is a Python module self.folder = os.path.dirname(folderOrModule.__file__) # These dicts will hold information about analysed files self.python = {False: CodeAnalysis('Python'), True: CodeAnalysis('Python (test)')} self.zpt = {False: CodeAnalysis('ZPT'), True: CodeAnalysis('ZPT (test)')} # Are we currently analysing real or test code? self.inTest = False # Which paths to exclude from the analysis? self.excludes = list(self.defaultExcludes) if excludes: self.excludes += excludes def printReport(self): '''Displays on stdout a small analysis report about self.folder.''' for zone in (False, True): self.python[zone].printReport() for zone in (False, True): self.zpt[zone].printReport() def isExcluded(self, path): '''Must p_path be excluded from the analysis?''' for excl in self.excludes: if excl in path: return True def run(self): '''Let's start the analysis of self.folder.''' # The test markers will allow us to know if we are analysing test code # or real code within a given part of self.folder code hierarchy. testMarker1 = '%stest%s' % (os.sep, os.sep) testMarker2 = '%stest' % os.sep testMarker3 = '%stests%s' % (os.sep, os.sep) testMarker4 = '%stests' % os.sep j = os.path.join for root, folders, files in os.walk(self.folder): if self.isExcluded(root): continue # Are we in real code or in test code ? self.inTest = False if root.endswith(testMarker2) or (root.find(testMarker1) != -1) or \ root.endswith(testMarker4) or (root.find(testMarker3) != -1): self.inTest = True # Scan the files in this folder for fileName in files: ext = os.path.splitext(fileName)[1] if ext in exts['py']: self.python[self.inTest].analyseFile(j(root, fileName)) elif ext in exts['pt']: self.zpt[self.inTest].analyseFile(j(root, fileName)) self.printReport() # ------------------------------------------------------------------------------ CONVERSION_ERROR = 'An error occurred while executing command "%s". %s' class FileWrapper: '''When you get, from an appy object, the value of a File attribute, you get an instance of this class.''' def __init__(self, zopeFile): '''This constructor is only used by Appy to create a nice File instance from a Zope corresponding instance (p_zopeFile). If you need to create a new file and assign it to a File attribute, use the attribute setter, do not create yourself an instance of this class.''' d = self.__dict__ d['_zopeFile'] = zopeFile # Not for you! d['name'] = zopeFile.filename d['content'] = zopeFile.data d['mimeType'] = zopeFile.content_type d['size'] = zopeFile.size # In bytes def __setattr__(self, name, v): d = self.__dict__ if name == 'name': self._zopeFile.filename = v d['name'] = v elif name == 'content': self._zopeFile.update_data(v, self.mimeType, len(v)) d['content'] = v d['size'] = len(v) elif name == 'mimeType': self._zopeFile.content_type = self.mimeType = v else: raise 'Impossible to set attribute %s. "Settable" attributes ' \ 'are "name", "content" and "mimeType".' % name def dump(self, filePath=None, format=None, tool=None): '''Writes the file on disk. If p_filePath is specified, it is the path name where the file will be dumped; folders mentioned in it must exist. If not, the file will be dumped in the OS temp folder. The absolute path name of the dumped file is returned. If an error occurs, the method returns None. If p_format is specified, OpenOffice will be called for converting the dumped file to the desired format. In this case, p_tool, a Appy tool, must be provided. Indeed, any Appy tool contains parameters for contacting OpenOffice in server mode.''' if not filePath: filePath = '%s/file%f.%s' % (getOsTempFolder(), time.time(), normalizeString(self.name)) f = file(filePath, 'w') if self.content.__class__.__name__ == 'Pdata': # The file content is splitted in several chunks. f.write(self.content.data) nextPart = self.content.next while nextPart: f.write(nextPart.data) nextPart = nextPart.next else: # Only one chunk f.write(self.content) f.close() if format: if not tool: return # Convert the dumped file using OpenOffice errorMessage = tool.convert(filePath, format) # Even if we have an "error" message, it could be a simple warning. # So we will continue here and, as a subsequent check for knowing if # an error occurred or not, we will test the existence of the # converted file (see below). os.remove(filePath) # Return the name of the converted file. baseName, ext = os.path.splitext(filePath) if (ext == '.%s' % format): filePath = '%s.res.%s' % (baseName, format) else: filePath = '%s.%s' % (baseName, format) if not os.path.exists(filePath): tool.log(CONVERSION_ERROR % (cmd, errorMessage), type='error') return return filePath # ------------------------------------------------------------------------------