appypod-rattail/shared/packaging.py

# ------------------------------------------------------------------------------
import os, os.path, subprocess, md5, shutil, random
from appy.shared.utils import getOsTempFolder, FolderDeleter, cleanFolder

# ------------------------------------------------------------------------------
debianInfo = '''Package: python-appy%s
Version: %s
Architecture: all
Maintainer: Gaetan Delannay <gaetan.delannay@geezteem.com>
Installed-Size: %d
Depends: python (>= %s)%s
Section: python
Priority: optional
Homepage: http://appyframework.org
Description: Appy builds simple but complex web Python apps.
'''
appCtl = '''#! /usr/lib/zope2.12/bin/python
import sys
from appy.bin.zopectl import ZopeRunner
args = ' '.join(sys.argv[1:])
sys.argv = [sys.argv[0], '-C', '/etc/%s.conf', args]
ZopeRunner().run()
'''
appRun = '''#! /bin/sh
exec "/usr/lib/zope2.12/bin/runzope" -C "/etc/%s.conf" "$@"
'''
ooStart = 'soffice -invisible -headless -nofirststartwizard ' \
          '"-accept=socket,host=localhost,port=2002;urp;"'
ooStartSh = '#! /bin/sh\n%s\n' % ooStart
zopeConf = '''# Zope configuration.
%%define INSTANCE %s
%%define DATA %s
%%define LOG %s
%%define HTTPPORT %s
%%define ZOPE_USER zope

instancehome $INSTANCE
effective-user $ZOPE_USER
%s
<eventlog>
  level info
  <logfile>
    path $LOG/event.log
    level info
  </logfile>
</eventlog>
<logger access>
  level WARN
  <logfile>
    path $LOG/Z2.log
    format %%(message)s
  </logfile>
</logger>
<http-server>
  address $HTTPPORT
</http-server>
<zodb_db main>
  <filestorage>
    path $DATA/Data.fs
  </filestorage>
  mount-point /
</zodb_db>
<zodb_db temporary>
  <temporarystorage>
   name temporary storage for sessioning
  </temporarystorage>
  mount-point /temp_folder
  container-class Products.TemporaryFolder.TemporaryContainer
</zodb_db>
'''
# initScript below will be used to define the scripts that will run the
# app-powered Zope instance and OpenOffice in server mode at boot time.
initScript = '''#! /bin/sh
### BEGIN INIT INFO
# Provides:          %s
# Required-Start:    $syslog $remote_fs
# Required-Stop:     $syslog $remote_fs
# Should-Start:      $remote_fs
# Should-Stop:       $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start %s
# Description:       Start the Zope and Appy-based %s application.
### END INIT INFO

case "$1" in
  start)
    %s
    ;;
  restart|reload|force-reload)
    %s
    ;;
  stop)
    %s
    ;;
  *)
    echo "Usage: $0 start|restart|stop" >&2
    exit 3
    ;;
esac
exit 0
'''

class Debianizer:
    '''This class allows to produce a Debian package from a Python (Appy)
       package.'''

    def __init__(self, app, out, appVersion='0.1.0',
                 pythonVersions=('2.6',), zopePort=8080,
                 depends=('zope2.12', 'openoffice.org', 'imagemagick'),
                 sign=True):
        # app is the path to the Python package to Debianize.
        self.app = app
        self.appName = os.path.basename(app)
        self.appNameLower = self.appName.lower()
        # Must we sign the Debian package? If yes, we make the assumption that
        # the currently logged user has a public/private key pair in ~/.gnupg,
        # generated with command "gpg --gen-key".
        self.sign = sign
        # out is the folder where the Debian package will be generated.
        self.out = out
        # What is the version number for this app ?
        self.appVersion = appVersion
        # On which Python versions will the Debian package depend?
        self.pythonVersions = pythonVersions
        # Port for Zope
        self.zopePort = zopePort
        # Debian package dependencies
        self.depends = depends
        # Zope 2.12 requires Python 2.6
        if 'zope2.12' in depends: self.pythonVersions = ('2.6',)

    def run(self):
        '''Generates the Debian package.'''
        curdir = os.getcwd()
        j = os.path.join
        tempFolder = getOsTempFolder()
        # Create, in the temp folder, the required sub-structure for the Debian
        # package.
        debFolder = j(tempFolder, 'debian')
        if os.path.exists(debFolder):
            FolderDeleter.delete(debFolder)
        # Copy the Python package into it
        srcFolder = j(debFolder, 'usr', 'lib')
        for version in self.pythonVersions:
            libFolder = j(srcFolder, 'python%s' % version)
            os.makedirs(libFolder)
            destFolder = j(libFolder, self.appName)
            shutil.copytree(self.app, destFolder)
            # Clean dest folder (.svn/.bzr files)
            cleanFolder(destFolder, folders=('.svn', '.bzr'))
        # When packaging Appy itself, everything is in /usr/lib/pythonX. When
        # packaging an Appy app, we will generate more files for creating a
        # running instance.
        if self.appName != 'appy':
            # Create the folders that will collectively represent the deployed
            # Zope instance.
            binFolder = j(debFolder, 'usr', 'bin')
            os.makedirs(binFolder)
            # <app>ctl
            name = '%s/%sctl' % (binFolder, self.appNameLower)
            f = file(name, 'w')
            f.write(appCtl % self.appNameLower)
            os.chmod(name, 0744) # Make it executable by owner.
            f.close()
            # <app>run
            name = '%s/%srun' % (binFolder, self.appNameLower)
            f = file(name, 'w')
            f.write(appRun % self.appNameLower)
            os.chmod(name, 0744) # Make it executable by owner.
            f.close()
            # startoo
            name = '%s/startoo' % binFolder
            f = file(name, 'w')
            f.write(ooStartSh)
            f.close()
            os.chmod(name, 0744) # Make it executable by owner.
            # /var/lib/<app> (will store Data.fs, lock files, etc)
            varLibFolder = j(debFolder, 'var', 'lib', self.appNameLower)
            os.makedirs(varLibFolder)
            f = file('%s/README' % varLibFolder, 'w')
            f.write('This folder stores the %s database.\n' % self.appName)
            f.close()
            # /var/log/<app> (will store event.log and Z2.log)
            varLogFolder = j(debFolder, 'var', 'log', self.appNameLower)
            os.makedirs(varLogFolder)
            f = file('%s/README' % varLogFolder, 'w')
            f.write('This folder stores the log files for %s.\n' % self.appName)
            f.close()
            # /etc/<app>.conf (Zope configuration file)
            etcFolder = j(debFolder, 'etc')
            os.makedirs(etcFolder)
            name = '%s/%s.conf' % (etcFolder, self.appNameLower)
            n = self.appNameLower
            f = file(name, 'w')
            productsFolder = '/usr/lib/python%s/%s/zope' % \
                             (self.pythonVersions[0], self.appName)
            f.write(zopeConf % ('/var/lib/%s' % n, '/var/lib/%s' % n,
                                '/var/log/%s' % n, str(self.zopePort),
                                'products %s\n' % productsFolder))
            f.close()
            # /etc/init.d/<app> (start the app at boot time)
            initdFolder = j(etcFolder, 'init.d')
            os.makedirs(initdFolder)
            name = '%s/%s' % (initdFolder, self.appNameLower)
            f = file(name, 'w')
            n = self.appNameLower
            f.write(initScript % (n, n, 'Start the Zope and Appy-based %s ' \
                                  'application.' % n, '%sctl start' % n,
                                  '%sctl restart' % n, '%sctl stop' % n))
            f.close()
            os.chmod(name, 0744) # Make it executable by owner.
            # /etc/init.d/oo (start OpenOffice at boot time)
            name = '%s/oo' % initdFolder
            f = file(name, 'w')
            f.write(initScript % ('oo', 'oo', 'Start OpenOffice in server mode',
                                  ooStart, ooStart, "#Can't stop OO."))
            f.close()
            os.chmod(name, 0744) # Make it executable by owner.
        # Get the size of the app, in Kb.
        os.chdir(tempFolder)
        cmd = subprocess.Popen(['du', '-b', '-s', 'debian'],
                               stdout=subprocess.PIPE)
        size = int(int(cmd.stdout.read().split()[0])/1024.0)
        os.chdir(debFolder)
        # Create data.tar.gz based on it.
        os.system('tar czvf data.tar.gz *')
        # Create the control file
        f = file('control', 'w')
        nameSuffix = ''
        dependencies = []
        if self.appName != 'appy':
            nameSuffix = '-%s' % self.appNameLower
            dependencies.append('python-appy')
        if self.depends:
            for d in self.depends: dependencies.append(d)
        depends = ''
        if dependencies:
            depends = ', ' + ', '.join(dependencies)
        f.write(debianInfo % (nameSuffix, self.appVersion, size,
                              self.pythonVersions[0], depends))
        f.close()
        # Create md5sum file
        f = file('md5sums', 'w')
        toWalk = ['usr']
        if self.appName != 'appy':
            toWalk += ['etc', 'var']
        for folderToWalk in toWalk:
            for dir, dirnames, filenames in os.walk(folderToWalk):
                for name in filenames:
                    m = md5.new()
                    pathName = j(dir, name)
                    currentFile = file(pathName, 'rb')
                    while True:
                        data = currentFile.read(8096)
                        if not data:
                            break
                        m.update(data)
                    currentFile.close()
                    # Add the md5 sum to the file
                    f.write('%s  %s\n' % (m.hexdigest(), pathName))
        f.close()
        # Create postinst, a script that will:
        # - bytecompile Python files after the Debian install
        # - change ownership of some files if required
        # - [in the case of an app-package] call update-rc.d for starting it at
        #   boot time.
        f = file('postinst', 'w')
        content = '#!/bin/sh\nset -e\n'
        for version in self.pythonVersions:
            bin = '/usr/bin/python%s' % version
            lib = '/usr/lib/python%s' % version
            cmds = ' %s -m compileall -q %s/%s 2> /dev/null\n' % (bin, lib,
                                                                  self.appName)
            content += 'if [ -e %s ]\nthen\n%sfi\n' % (bin, cmds)
        if self.appName != 'appy':
            # Allow user "zope", that runs the Zope instance, to write the
            # database and log files.
            content += 'chown -R zope:root /var/lib/%s\n' % self.appNameLower
            content += 'chown -R zope:root /var/log/%s\n' % self.appNameLower
            # Call update-rc.d for starting the app at boot time
            content += 'update-rc.d %s defaults\n' % self.appNameLower
            content += 'update-rc.d oo defaults\n'
            # (re-)start the app
            content += '%sctl restart\n' % self.appNameLower
            # (re-)start oo
            content += 'startoo\n'
        f.write(content)
        f.close()
        # Create prerm, a script that will remove all pyc files before removing
        # the Debian package.
        f = file('prerm', 'w')
        content = '#!/bin/sh\nset -e\n'
        for version in self.pythonVersions:
            content += 'find /usr/lib/python%s/%s -name "*.pyc" -delete\n' % \
                       (version, self.appName)
        f.write(content)
        f.close()
        # Create control.tar.gz
        os.system('tar czvf control.tar.gz ./control ./md5sums ./postinst ' \
                  './prerm')
        # Create debian-binary
        f = file('debian-binary', 'w')
        f.write('2.0\n')
        f.close()
        # Create the signature if required
        if self.sign:
            # Create the concatenated version of all files within the deb
            os.system('cat debian-binary control.tar.gz data.tar.gz > ' \
                      '/tmp/combined-contents')
            os.system('gpg -abs -o _gpgorigin /tmp/combined-contents')
            signFile = '_gpgorigin '
            os.remove('/tmp/combined-contents')
            # Export the public key and name it according to its ID as found by
            # analyzing the result of command "gpg --fingerprint".
            cmd = subprocess.Popen(['gpg', '--fingerprint'],
                                   stdout=subprocess.PIPE)
            fingerprint = cmd.stdout.read().split('\n')
            id = 'pubkey'
            for line in fingerprint:
                if '=' not in line: continue
                id = line.split('=')[1].strip()
                id = ''.join(id.split()[-4:])
                break
            os.system('gpg --export -a > %s/%s.asc' % (self.out, id))
        else:
            signFile = ''
        # Create the .deb package
        debName = 'python-appy%s-%s.deb' % (nameSuffix, self.appVersion)
        os.system('ar -r %s %sdebian-binary control.tar.gz data.tar.gz' % \
                  (debName, signFile))
        # Move it to self.out
        os.rename(j(debFolder, debName), j(self.out, debName))
        # Clean temp files
        FolderDeleter.delete(debFolder)
        os.chdir(curdir)

# ------------------------------------------------------------------------------
definitionJson = '''{
  "name": "%s",
  "description": "%s, a Appy-based application",
  "packages": [{"name": "python-appy" }, {"name": "python-appy-%s" }],
  "files": [
    { "group": "root", "mode": "644", "name": "%s.conf",
      "owner": "root", "path": "/etc/%s.conf",
      "template": "%s"
    }],
  "handlers": [
    { "on": ["_install"] },
    { "on": ["_uninstall" ] },
    { "on": ["%s_http_port"],
      "do": [
        { "action": "update", "resource": "file://%s.conf" }, 
        { "action": "restart", "resource": "service://%s" }
      ]}
    ],
  "services": [
    { "name": "%s", "enabled": "true", "running": "false" }, 
    { "name": "oo",  "enabled": "true",  "running": "false" }]
}
'''
definitionJsonConf = '''{
  "name": "%s.conf",
  "uuid": "%s",
  "parameters": [
    { "key": "%s_http_port",  "name": "%s HTTP port",
      "description": "%s HTTP port for the Zope process",
      "value": "8080"}
  ]
}
'''

class Cortexer:
    '''This class allows to produce a Cortex application definition for
       a Debianized Python/Appy application.'''
    def __init__(self, app, pythonVersions=('2.6',)):
        self.appName = os.path.basename(app)
        self.pythonVersions = pythonVersions
        appFolder = os.path.dirname(app)
        # Prepare the output folder (remove any existing one)
        cortexFolder = os.path.join(appFolder, 'cortex.admin')
        if os.path.exists(cortexFolder):
            FolderDeleter.delete(cortexFolder)
        allFolders = os.path.join(cortexFolder, 'applications', self.appName)
        os.makedirs(allFolders)
        self.out = allFolders

    uuidChars= ['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F']
    def generateUid(self):
        '''Generates a 32-chars-wide UID for identifying the configuration file
           in the Cortex DB.'''
        res = ''
        for i in range(32):
            res += self.uuidChars[random.randint(0,15)]
        return res

    def run(self):
        # Create the root app description file "definition.json".
        uid = self.generateUid()
        name = os.path.join(self.out, 'definition.json')
        f = file(name, 'w')
        n = self.appName
        nl = self.appName.lower()
        f.write(definitionJson % (n, n, nl, nl, nl, uid, nl, nl, nl, nl))
        f.close()
        # Create the folder corresponding to the config file, and its content.
        confFolder = os.path.join(self.out, '%s.conf' % nl)
        os.mkdir(confFolder)
        # Create the definition file for this config file, that will contain
        # the names of Cortex-managed variables within the configuration file.
        name = os.path.join(confFolder, 'definition.json')
        f = file(name, 'w')
        f.write(definitionJsonConf % (nl, uid, nl, n, n))
        f.close()
        # Create the Zope config file, with Cortex-like variables within in.
        name = os.path.join(confFolder, '%s.conf' % nl)
        f = file(name, 'w')
        productsFolder='/usr/lib/python%s/%s/zope' % (self.pythonVersions[0],n)
        f.write(zopeConf % ('/var/lib/%s' % n, '/var/lib/%s' % n,
                            '/var/log/%s' % n, '${%s_http_port}' % nl,
                            'products %s\n' % productsFolder))
        f.close()
# ------------------------------------------------------------------------------