From 13443ea79e6cde1ce1b7e0ff1a11c04d66c4b3d1 Mon Sep 17 00:00:00 2001 From: Gaetan Delannay Date: Wed, 18 Jan 2012 14:27:24 +0100 Subject: [PATCH] appy.bin: generate.py: one less arg: outputFolder has been removed (the script now generates the Zope product in /zope); generate.py: new option '-d', for generating a Debian package from the Python (Appy) app. --- bin/generate.py | 47 +++++++++-------- bin/publish.py | 97 ++---------------------------------- gen/generator.py | 11 ++-- shared/packaging.py | 119 ++++++++++++++++++++++++++++++++++++++++++++ shared/utils.py | 18 +++++-- 5 files changed, 171 insertions(+), 121 deletions(-) create mode 100644 shared/packaging.py diff --git a/bin/generate.py b/bin/generate.py index c92faf2..a5e126a 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -1,17 +1,17 @@ -'''This script allows to generate a product from a Appy application.''' +'''This script allows to generate a Zope product from a Appy application.''' # ------------------------------------------------------------------------------ import sys, os.path from optparse import OptionParser from appy.gen.generator import GeneratorError, ZopeGenerator from appy.shared.utils import LinesCounter +from appy.shared.packaging import Debianizer import appy.version # ------------------------------------------------------------------------------ ERROR_CODE = 1 APP_NOT_FOUND = 'Application not found at %s.' WRONG_NG_OF_ARGS = 'Wrong number of arguments.' -WRONG_OUTPUT_FOLDER = 'Output folder not found. Please create it first.' C_OPTION = 'Removes from i18n files all labels that are not automatically ' \ 'generated from your gen-application. It can be useful during ' \ 'development, when you do lots of name changes (classes, ' \ @@ -32,26 +32,24 @@ S_OPTION = 'Sorts all i18n labels. If you use this option, among the ' \ 'i18n file. When the development is finished, it may be a good ' \ 'idea to sort the labels to get a clean and logically ordered ' \ 'set of translation files.' +D_OPTION = 'Generates a Debian package for this app. The Debian package will ' \ + 'be generated at the same level as the root application folder.' class GeneratorScript: - '''usage: %prog [options] app outputFolder + '''usage: %prog [options] app "app" is the path to your Appy application, which must be a Python package (= a folder containing a file named __init__.py). Your app may reside anywhere, but needs to be accessible by Zope. Typically, it may be or symlinked - in /lib/python, but not within the - generated product, stored or symlinked in - /Products. + in /lib/python. - "outputFolder" is the folder where the Zope product will be generated. - For example, if you develop your application in - /home/gdy/MyProject/MyProject, you typically specify - "/home/gdy/MyProject/zope" as outputFolder. + This command generates a Zope product in /zope, which must be + or symlinked in /Products. ''' def manageArgs(self, parser, options, args): # Check number of args - if len(args) != 2: + if len(args) != 1: print WRONG_NG_OF_ARGS parser.print_help() sys.exit(ERROR_CODE) @@ -59,13 +57,8 @@ class GeneratorScript: if not os.path.exists(args[0]): print APP_NOT_FOUND % args[0] sys.exit(ERROR_CODE) - # Check existence of outputFolder - if not os.path.exists(args[1]): - print WRONG_OUTPUT_FOLDER - sys.exit(ERROR_CODE) - # Convert all paths in absolute paths - for i in (0,1): - args[i] = os.path.abspath(args[i]) + # Convert app path to an absolute path + args[0] = os.path.abspath(args[0]) def run(self): optParser = OptionParser(usage=GeneratorScript.__doc__) @@ -73,14 +66,26 @@ class GeneratorScript: dest='i18nClean', default=False, help=C_OPTION) optParser.add_option("-s", "--i18n-sort", action='store_true', dest='i18nSort', default=False, help=S_OPTION) + optParser.add_option("-d", "--debian", action='store_true', + dest='debian', default=False, help=D_OPTION) (options, args) = optParser.parse_args() try: self.manageArgs(optParser, options, args) print 'Appy version:', appy.version.verbose - print 'Generating Zope product in %s...' % args[1] - ZopeGenerator(args[0], args[1], options).run() + print 'Generating Zope product in %s/zope...' % args[0] + ZopeGenerator(args[0], options).run() # Give the user some statistics about its code - LinesCounter(args[0]).run() + LinesCounter(args[0], excludes=['%szope' % os.sep]).run() + # Generates a Debian package for this app if required + if options.debian: + app = args[0] + appDir = os.path.dirname(app) + # Get the app version from zope/version.txt + f = file(os.path.join(app, 'zope', 'version.txt')) + version = f.read() + f.close() + version = version[:version.find('build')-1] + Debianizer(app, appDir, appVersion=version).run() except GeneratorError, ge: sys.stderr.write(str(ge)) sys.stderr.write('\n') diff --git a/bin/publish.py b/bin/publish.py index b737458..bdb06b2 100644 --- a/bin/publish.py +++ b/bin/publish.py @@ -1,9 +1,10 @@ #!/usr/bin/python # Imports ---------------------------------------------------------------------- -import os, os.path, sys, shutil, re, zipfile, sys, ftplib, time, subprocess, md5 +import os, os.path, sys, shutil, re, zipfile, sys, ftplib, time import appy from appy.shared import appyPath from appy.shared.utils import FolderDeleter, LinesCounter +from appy.shared.packaging import Debianizer from appy.bin.clean import Cleaner from appy.gen.utils import produceNiceMessage @@ -26,34 +27,6 @@ recursive-include appy/gen * recursive-include appy/pod * recursive-include appy/shared * ''' -debianInfo = '''Package: python-appy -Version: %s -Architecture: all -Maintainer: Gaetan Delannay -Installed-Size: %d -Depends: python (>= 2.6), python (<< 3.0) -Section: python -Priority: optional -Homepage: http://appyframework.org -Description: Appy builds simple but complex web Python apps. -''' -debianPostInst = '''#!/bin/sh -set -e -if [ -e /usr/bin/python2.6 ] -then - /usr/bin/python2.6 -m compileall -q /usr/lib/python2.6/appy 2> /dev/null -fi -if [ -e /usr/bin/python2.7 ] -then - /usr/bin/python2.7 -m compileall -q /usr/lib/python2.7/appy 2> /dev/null -fi -''' -debianPreRm = '''#!/bin/sh -set -e -find /usr/lib/python2.6/appy -name "*.pyc" -delete -find /usr/lib/python2.7/appy -name "*.pyc" -delete -''' - def askLogin(): print 'Login: ', login = sys.stdin.readline().strip() @@ -242,9 +215,6 @@ class Text2Html: class Publisher: '''Publishes Appy on the web.''' pageBody = re.compile('(.*)', re.S) - eggVersion = re.compile('version\s*=\s*".*?"') - pythonTargets = ('2.4', '2.5') - svnServer = 'http://svn.communesplone.org/svn/communesplone/appy' def __init__(self): self.genFolder = '%s/temp' % appyPath @@ -303,66 +273,9 @@ class Publisher: def createDebianRelease(self): '''Creates a Debian package for Appy.''' - curdir = os.getcwd() - # Create a temp folder for creating the Debian files hierarchy. - srcFolder = os.path.join(self.genFolder, 'debian', 'usr', 'lib') - os.makedirs(os.path.join(srcFolder, 'python2.6')) - os.makedirs(os.path.join(srcFolder, 'python2.7')) - # Copy Appy sources in it - py26 = os.path.join(srcFolder, 'python2.6', 'appy') - os.rename(os.path.join(self.genFolder, 'appy'), py26) - shutil.copytree(py26, os.path.join(srcFolder, 'python2.7', 'appy')) - # Create data.tar.gz based on it. - debFolder = os.path.join(self.genFolder, 'debian') - os.chdir(debFolder) - os.system('tar czvf data.tar.gz ./usr') - # Get the size of Appy, in Kb. - cmd = subprocess.Popen(['du', '-b', '-s', 'usr'],stdout=subprocess.PIPE) - size = int(int(cmd.stdout.read().split()[0])/1024.0) - # Create control file - f = file('control', 'w') - f.write(debianInfo % (self.versionShort, size)) - f.close() - # Create md5sum file - f = file('md5sums', 'w') - for dir, dirnames, filenames in os.walk('usr'): - for name in filenames: - m = md5.new() - pathName = os.path.join(dir, name) - currentFile = file(pathName, 'rb') - while True: - data = currentFile.read(8096) - if not data: - break - m.update(data) - currentFile.close() - # Add the md5 sum to the file - f.write('%s %s\n' % (m.hexdigest(), pathName)) - f.close() - # Create postinst and prerm - f = file('postinst', 'w') - f.write(debianPostInst) - f.close() - f = file('prerm', 'w') - f.write(debianPreRm) - f.close() - # Create control.tar.gz - os.system('tar czvf control.tar.gz ./control ./md5sums ./postinst ' \ - './prerm') - # Create debian-binary - f = file('debian-binary', 'w') - f.write('2.0\n') - f.close() - # Create the .deb package - debName = 'python-appy-%s.deb' % self.versionShort - os.system('ar -r %s debian-binary control.tar.gz data.tar.gz' % \ - debName) - os.chdir(curdir) - # Move it to folder "versions". - os.rename(os.path.join(debFolder, debName), - os.path.join(appyPath, 'versions', debName)) - # Clean temp files - FolderDeleter.delete(debFolder) + j = os.path.join + Debianizer(j(self.genFolder, 'appy'), j(appyPath, 'versions'), + appVersion=self.versionShort).run() def createDistRelease(self): '''Create the distutils package.''' diff --git a/gen/generator.py b/gen/generator.py index 390db3a..927d78f 100644 --- a/gen/generator.py +++ b/gen/generator.py @@ -115,12 +115,12 @@ CODE_HEADER = '''# -*- coding: utf-8 -*- ''' class Generator: '''Abstract base class for building a generator.''' - def __init__(self, application, outputFolder, options): + def __init__(self, application, options): self.application = application # Determine application name self.applicationName = os.path.basename(application) # Determine output folder (where to store the generated product) - self.outputFolder = outputFolder + self.outputFolder = os.path.join(application, 'zope') self.options = options # Determine templates folder genFolder = os.path.dirname(__file__) @@ -186,8 +186,13 @@ class Generator: IMPORT_ERROR = 'Warning: error while importing module %s (%s)' SYNTAX_ERROR = 'Warning: error while parsing module %s (%s)' + noVisit = ('tr', 'zope') def walkModule(self, moduleName): '''Visits a given (sub-*)module into the application.''' + # Some sub-modules must not be visited + for nv in self.noVisit: + nvName = '%s.%s' % (self.applicationName, nv) + if moduleName == nvName: return try: exec 'import %s' % moduleName exec 'moduleObj = %s' % moduleName @@ -359,7 +364,7 @@ class ZopeGenerator(Generator): versionRex = re.compile('(.*?\s+build)\s+(\d+)') def initialize(self): # Determine version number - self.version = '0.1 build 1' + self.version = '0.1.0 build 1' versionTxt = os.path.join(self.outputFolder, 'version.txt') if os.path.exists(versionTxt): f = file(versionTxt) diff --git a/shared/packaging.py b/shared/packaging.py new file mode 100644 index 0000000..6cec459 --- /dev/null +++ b/shared/packaging.py @@ -0,0 +1,119 @@ +# ------------------------------------------------------------------------------ +import os, os.path, subprocess, md5, shutil +from appy.shared.utils import getOsTempFolder, FolderDeleter + +# ------------------------------------------------------------------------------ +debianInfo = '''Package: python-appy%s +Version: %s +Architecture: all +Maintainer: Gaetan Delannay +Installed-Size: %d +Depends: python (>= %s), python (<= %s)%s +Section: python +Priority: optional +Homepage: http://appyframework.org +Description: Appy builds simple but complex web Python apps. +''' + +class Debianizer: + '''This class allows to produce a Debian package from a Python (Appy) + package.''' + + def __init__(self, app, out, appVersion='0.1.0', + pythonVersions=('2.6', '2.7')): + # app is the path to the Python package to Debianize. + self.app = app + self.appName = os.path.basename(app) + # out is the folder where the Debian package will be generated. + self.out = out + # What is the version number for this app ? + self.appVersion = appVersion + # On which Python versions will the Debian package depend? + self.pythonVersions = pythonVersions + + def run(self): + '''Generates the Debian package.''' + curdir = os.getcwd() + j = os.path.join + tempFolder = getOsTempFolder() + # Create, in the temp folder, the required sub-structure for the Debian + # package. + debFolder = j(tempFolder, 'debian') + if os.path.exists(debFolder): + FolderDeleter.delete(debFolder) + # Copy the Python package into it + srcFolder = j(debFolder, 'usr', 'lib') + for version in self.pythonVersions: + libFolder = j(srcFolder, 'python%s' % version) + os.makedirs(libFolder) + shutil.copytree(self.app, j(libFolder, self.appName)) + # Create data.tar.gz based on it. + os.chdir(debFolder) + os.system('tar czvf data.tar.gz ./usr') + # Get the size of the app, in Kb. + cmd = subprocess.Popen(['du', '-b', '-s', 'usr'],stdout=subprocess.PIPE) + size = int(int(cmd.stdout.read().split()[0])/1024.0) + # Create the control file + f = file('control', 'w') + nameSuffix = '' + depends = '' + if self.appName != 'appy': + nameSuffix = '-%s' % self.appName + depends = ', python-appy' + f.write(debianInfo % (nameSuffix, self.appVersion, size, + self.pythonVersions[0], self.pythonVersions[1], + depends)) + f.close() + # Create md5sum file + f = file('md5sums', 'w') + for dir, dirnames, filenames in os.walk('usr'): + for name in filenames: + m = md5.new() + pathName = j(dir, name) + currentFile = file(pathName, 'rb') + while True: + data = currentFile.read(8096) + if not data: + break + m.update(data) + currentFile.close() + # Add the md5 sum to the file + f.write('%s %s\n' % (m.hexdigest(), pathName)) + f.close() + # Create postinst, a script that will bytecompile Python files after the + # Debian install. + f = file('postinst', 'w') + content = '#!/bin/sh\nset -e\n' + for version in self.pythonVersions: + content += 'if [ -e /usr/bin/python%s ]\nthen\n ' \ + '/usr/bin/python%s -m compileall -q ' \ + '/usr/lib/python%s/%s 2> /dev/null\nfi\n' % \ + (version, version, version, self.appName) + f.write(content) + f.close() + # Create prerm, a script that will remove all pyc files before removing + # the Debian package. + f = file('prerm', 'w') + content = '#!/bin/sh\nset -e\n' + for version in self.pythonVersions: + content += 'find /usr/lib/python%s/%s -name "*.pyc" -delete\n' % \ + (version, self.appName) + f.write(content) + f.close() + # Create control.tar.gz + os.system('tar czvf control.tar.gz ./control ./md5sums ./postinst ' \ + './prerm') + # Create debian-binary + f = file('debian-binary', 'w') + f.write('2.0\n') + f.close() + # Create the .deb package + debName = 'python-appy%s-%s.deb' % (nameSuffix, self.appVersion) + os.system('ar -r %s debian-binary control.tar.gz data.tar.gz' % \ + debName) + # Move it to self.out + os.rename(j(debFolder, debName), j(self.out, debName)) + # Clean temp files + FolderDeleter.delete(debFolder) + os.chdir(curdir) +# ------------------------------------------------------------------------------ diff --git a/shared/utils.py b/shared/utils.py index a8e8a06..6ec33c2 100644 --- a/shared/utils.py +++ b/shared/utils.py @@ -364,7 +364,10 @@ class CodeAnalysis: # ------------------------------------------------------------------------------ class LinesCounter: '''Counts and classifies the lines of code within a folder hierarchy.''' - def __init__(self, folderOrModule): + defaultExcludes = ('%s.svn' % os.sep, '%s.bzr' % os.sep, '%stmp' % os.sep, + '%stemp' % os.sep) + + def __init__(self, folderOrModule, excludes=None): if isinstance(folderOrModule, basestring): # It is the path of some folder self.folder = folderOrModule @@ -378,12 +381,20 @@ class LinesCounter: True: CodeAnalysis('ZPT (test)')} # Are we currently analysing real or test code? self.inTest = False + # Which paths to exclude from the analysis? + self.excludes = list(self.defaultExcludes) + if excludes: self.excludes += excludes def printReport(self): '''Displays on stdout a small analysis report about self.folder.''' for zone in (False, True): self.python[zone].printReport() for zone in (False, True): self.zpt[zone].printReport() + def isExcluded(self, path): + '''Must p_path be excluded from the analysis?''' + for excl in self.excludes: + if excl in path: return True + def run(self): '''Let's start the analysis of self.folder.''' # The test markers will allow us to know if we are analysing test code @@ -394,10 +405,7 @@ class LinesCounter: testMarker4 = '%stests' % os.sep j = os.path.join for root, folders, files in os.walk(self.folder): - rootName = os.path.basename(root) - if rootName.startswith('.') or \ - (rootName in ('tmp', 'temp')): - continue + if self.isExcluded(root): continue # Are we in real code or in test code ? self.inTest = False if root.endswith(testMarker2) or (root.find(testMarker1) != -1) or \