490 lines
20 KiB
Python
490 lines
20 KiB
Python
#!/usr/bin/python
|
|
# Imports ----------------------------------------------------------------------
|
|
import os, os.path, sys, shutil, re, zipfile, sys, ftplib, time
|
|
import appy
|
|
from appy.shared import appyPath
|
|
from appy.shared.utils import FolderDeleter, LinesCounter
|
|
from appy.shared.packaging import Debianizer
|
|
from appy.bin.clean import Cleaner
|
|
from appy.gen.utils import produceNiceMessage
|
|
|
|
# ------------------------------------------------------------------------------
|
|
versionRex = re.compile('(\d+\.\d+\.\d+)')
|
|
distInfo = '''from distutils.core import setup
|
|
setup(name = "appy", version = "%s",
|
|
description = "The Appy framework",
|
|
long_description = "Appy builds simple but complex web Python apps.",
|
|
author = "Gaetan Delannay",
|
|
author_email = "gaetan.delannay AT geezteem.com",
|
|
license = "GPL", platforms="all",
|
|
url = 'http://appyframework.org',
|
|
packages = [%s],
|
|
package_data = {'':["*.*"]})
|
|
'''
|
|
manifestInfo = '''
|
|
recursive-include appy/bin *
|
|
recursive-include appy/gen *
|
|
recursive-include appy/pod *
|
|
recursive-include appy/shared *
|
|
'''
|
|
def askLogin():
|
|
print 'Login: ',
|
|
login = sys.stdin.readline().strip()
|
|
print 'Password: ',
|
|
passwd = sys.stdin.readline().strip()
|
|
return (login, passwd)
|
|
|
|
class FtpFolder:
|
|
'''Represents a folder on a FTP site.'''
|
|
def __init__(self, name):
|
|
self.name = name
|
|
self.parent = None
|
|
self.subFolders = []
|
|
self.files = []
|
|
self.isComplete = False # Is True if all contained files and direct
|
|
# subFolders were analysed.
|
|
def getFullName(self):
|
|
if not self.parent:
|
|
res = '.'
|
|
else:
|
|
res = '%s/%s' % (self.parent.getFullName(), self.name)
|
|
return res
|
|
def addSubFolder(self, subFolder):
|
|
self.subFolders.append(subFolder)
|
|
subFolder.parent = self
|
|
def isFullyComplete(self):
|
|
res = self.isComplete
|
|
for subFolder in self.subFolders:
|
|
res = res and subFolder.isFullyComplete()
|
|
return res
|
|
def getIncompleteSubFolders(self):
|
|
res = []
|
|
for subFolder in self.subFolders:
|
|
if not subFolder.isComplete:
|
|
res.append(subFolder)
|
|
elif not subFolder.isFullyComplete():
|
|
res += subFolder.getIncompleteSubFolders()
|
|
return res
|
|
def __str__(self):
|
|
res = 'Folder %s' % self.getFullName()
|
|
if self.files:
|
|
res += '\nFiles:\n'
|
|
for f in self.files:
|
|
res += '%s\n' % f
|
|
if self.subFolders:
|
|
res += '\nSubFolders:\n'
|
|
for subFolder in self.subFolders:
|
|
res += str(subFolder)
|
|
return res
|
|
def clean(self, site):
|
|
'''Cleans this folder'''
|
|
# First, clean subFolders if they exist
|
|
for subFolder in self.subFolders:
|
|
subFolder.clean(site)
|
|
# Remove the subFolder
|
|
site.rmd(subFolder.getFullName())
|
|
# Then, remove the files contained in the folder.
|
|
for f in self.files:
|
|
fileName = '%s/%s' % (self.getFullName(), f)
|
|
site.delete(fileName)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
class AppySite:
|
|
'''Represents the Appy web sie where the project is published.'''
|
|
name = 'appyframework.org'
|
|
textExtensions = ('.htm', '.html', '.css', '.txt')
|
|
def __init__(self):
|
|
# Delete the "egg" folder on not-yet-copied local site.
|
|
eggFolder = '%s/temp/egg' % appyPath
|
|
if os.path.isdir(eggFolder):
|
|
FolderDeleter.delete(eggFolder)
|
|
# Ask user id and password for FTP transfer
|
|
userId, userPassword = askLogin()
|
|
self.site = ftplib.FTP(self.name)
|
|
self.site.login(userId, userPassword)
|
|
self.rootFolder = None # Root folder of appy site ~FtpFolder~
|
|
self.currentFolder = None # Currently visited folder ~FtpFolder~
|
|
def analyseFolderEntry(self, folderEntry):
|
|
'''p_line corresponds to a 'ls' entry.'''
|
|
elems = folderEntry.split(' ')
|
|
elemName = elems[len(elems)-1]
|
|
if (not elemName.startswith('.')) and \
|
|
(not elemName.startswith('_')):
|
|
if elems[0].startswith('d'):
|
|
self.currentFolder.addSubFolder(FtpFolder(elemName))
|
|
else:
|
|
self.currentFolder.files.append(elemName)
|
|
def createFolderProxies(self):
|
|
'''Creates a representation of the FTP folders of the appy site in the
|
|
form of FtpFolder instances.'''
|
|
self.rootFolder = FtpFolder('.')
|
|
self.currentFolder = self.rootFolder
|
|
self.site.dir(self.currentFolder.getFullName(), self.analyseFolderEntry)
|
|
self.rootFolder.isComplete = True
|
|
while not self.rootFolder.isFullyComplete():
|
|
incompleteFolders = self.rootFolder.getIncompleteSubFolders()
|
|
for folder in incompleteFolders:
|
|
self.currentFolder = folder
|
|
self.site.dir(self.currentFolder.getFullName(),
|
|
self.analyseFolderEntry)
|
|
self.currentFolder.isComplete = True
|
|
def copyFile(self, fileName):
|
|
'''Copies a file on the FTP server.'''
|
|
localFile = file(fileName)
|
|
cmd = 'STOR %s' % fileName
|
|
fileExt = os.path.splitext(fileName)[1]
|
|
if fileExt in self.textExtensions:
|
|
# Make a transfer in text mode
|
|
print 'Transfer file %s (text mode)' % fileName
|
|
self.site.storlines(cmd, localFile)
|
|
else:
|
|
# Make a transfer in binary mode
|
|
print 'Transfer file %s (binary mode)' % fileName
|
|
self.site.storbinary(cmd, localFile)
|
|
def publish(self):
|
|
# Delete the existing content of the distant site
|
|
self.createFolderProxies()
|
|
print 'Removing existing data on site...'
|
|
self.rootFolder.clean(self.site)
|
|
curDir = os.getcwd()
|
|
os.chdir('%s/temp' % appyPath)
|
|
for root, dirs, files in os.walk('.'):
|
|
for folder in dirs:
|
|
folderName = '%s/%s' % (root, folder)
|
|
self.site.mkd(folderName)
|
|
for f in files:
|
|
fileName = '%s/%s' % (root, f)
|
|
self.copyFile(fileName)
|
|
os.chdir(curDir)
|
|
self.site.close()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
class Text2Html:
|
|
'''Converts a text file into a HTML file.'''
|
|
def __init__(self, txtFile, htmlFile):
|
|
self.txtFile = file(txtFile)
|
|
self.htmlFile = file(htmlFile, 'w')
|
|
def retainLine(self, line):
|
|
'''Must we dump this line in the result ?'''
|
|
pass
|
|
def getFirstChar(self, line):
|
|
'''Gets the first relevant character of the line. For a TodoConverter
|
|
this is not really the first one because lines taken into account start
|
|
with a 'v' character.'''
|
|
return line[self.firstChar]
|
|
def getCleanLine(self, line, isTitle=False):
|
|
'''Gets the line as it will be inserted in the HTML result: remove some
|
|
leading and trailing characters.'''
|
|
start = self.firstChar
|
|
if not isTitle:
|
|
start += 1
|
|
return line[start:-1]
|
|
def getProlog(self):
|
|
'''If you want to write a small prolog in the HTML file, you may
|
|
generate it here.'''
|
|
return ''
|
|
def run(self):
|
|
self.htmlFile.write('<html>\n\n<head><title>%s</title></head>\n\n' \
|
|
'<body>\n' % self.title)
|
|
self.htmlFile.write(self.getProlog())
|
|
inList = False
|
|
for line in self.txtFile:
|
|
if self.retainLine(line):
|
|
firstChar = self.getFirstChar(line)
|
|
if firstChar == '-':
|
|
if not inList:
|
|
# Begin a new bulleted list
|
|
self.htmlFile.write('<ul>\n')
|
|
inList = True
|
|
self.htmlFile.write(
|
|
'<li>%s</li>\n' % self.getCleanLine(line))
|
|
elif firstChar == ' ':
|
|
pass
|
|
else:
|
|
# It is a title
|
|
if inList:
|
|
self.htmlFile.write('</ul>\n')
|
|
inList = False
|
|
self.htmlFile.write(
|
|
'<h1>%s</h1>\n' % self.getCleanLine(line, True))
|
|
self.htmlFile.write('\n</ul>\n</body>\n</html>')
|
|
self.txtFile.close()
|
|
self.htmlFile.close()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
class Publisher:
|
|
'''Publishes Appy on the web.'''
|
|
pageBody = re.compile('<body.*?>(.*)</body>', re.S)
|
|
|
|
def __init__(self):
|
|
self.genFolder = '%s/temp' % appyPath
|
|
self.ftp = None # FTP connection to appyframework.org
|
|
# Retrieve version-related information
|
|
versionFileName = '%s/doc/version.txt' % appyPath
|
|
f = file(versionFileName)
|
|
self.versionShort = f.read().strip()
|
|
# Long version includes release date
|
|
self.versionLong = '%s (%s)' % (self.versionShort,
|
|
time.strftime('%Y/%m/%d %H:%M'))
|
|
f.close()
|
|
# In silent mode (option -s), no question is asked, default answers are
|
|
# automatically given.
|
|
if (len(sys.argv) > 1) and (sys.argv[1] == '-s'):
|
|
self.silent = True
|
|
else:
|
|
self.silent = False
|
|
|
|
def askQuestion(self, question, default='yes'):
|
|
'''Asks a question to the user (yes/no) and returns True if the user
|
|
answered "yes".'''
|
|
if self.silent: return (default == 'yes')
|
|
defaultIsYes = (default.lower() in ('y', 'yes'))
|
|
if defaultIsYes:
|
|
yesNo = '[Y/n]'
|
|
else:
|
|
yesNo = '[y/N]'
|
|
print question + ' ' + yesNo + ' ',
|
|
response = sys.stdin.readline().strip().lower()
|
|
res = False
|
|
if response in ('y', 'yes'):
|
|
res = True
|
|
elif response in ('n', 'no'):
|
|
res = False
|
|
elif not response:
|
|
# It depends on default value
|
|
if defaultIsYes:
|
|
res = True
|
|
else:
|
|
res = False
|
|
return res
|
|
|
|
def executeCommand(self, cmd):
|
|
'''Executes the system command p_cmd.'''
|
|
print 'Executing %s...' % cmd
|
|
os.system(cmd)
|
|
|
|
distExcluded = ('appy/doc', 'appy/temp', 'appy/versions', 'appy/gen/test')
|
|
def isDistExcluded(self, name):
|
|
'''Returns True if folder named p_name must be included in the
|
|
distribution.'''
|
|
if '.bzr' in name: return True
|
|
for prefix in self.distExcluded:
|
|
if name.startswith(prefix): return True
|
|
|
|
def createDebianRelease(self):
|
|
'''Creates a Debian package for Appy.'''
|
|
j = os.path.join
|
|
Debianizer(j(self.genFolder, 'appy'), j(appyPath, 'versions'),
|
|
appVersion=self.versionShort, depends=[]).run()
|
|
|
|
def createDistRelease(self):
|
|
'''Create the distutils package.'''
|
|
curdir = os.getcwd()
|
|
distFolder = '%s/dist' % self.genFolder
|
|
# Create setup.py
|
|
os.mkdir(distFolder)
|
|
f = file('%s/setup.py' % distFolder, 'w')
|
|
# List all packages to include
|
|
packages = []
|
|
os.chdir(os.path.dirname(appyPath))
|
|
for dir, dirnames, filenames in os.walk('appy'):
|
|
if self.isDistExcluded(dir): continue
|
|
packageName = dir.replace('/', '.')
|
|
packages.append('"%s"' % packageName)
|
|
f.write(distInfo % (self.versionShort, ','.join(packages)))
|
|
f.close()
|
|
# Create MANIFEST.in
|
|
f = file('%s/MANIFEST.in' % distFolder, 'w')
|
|
f.write(manifestInfo)
|
|
f.close()
|
|
# Move appy sources within the dist folder
|
|
os.rename('%s/appy' % self.genFolder, '%s/appy' % distFolder)
|
|
# Create the source distribution
|
|
os.chdir(distFolder)
|
|
self.executeCommand('python setup.py sdist')
|
|
# DistUtils has created the .tar.gz file. Move it to folder "versions"
|
|
name = 'appy-%s.tar.gz' % self.versionShort
|
|
os.rename('%s/dist/%s' % (distFolder, name),
|
|
'%s/versions/%s' % (appyPath, name))
|
|
# Clean temp files
|
|
os.chdir(curdir)
|
|
# Keep the Appy source for building the Debian package afterwards
|
|
os.rename(os.path.join(self.genFolder, 'dist', 'appy'), \
|
|
os.path.join(self.genFolder, 'appy'))
|
|
FolderDeleter.delete(os.path.join(self.genFolder, 'dist'))
|
|
return name
|
|
|
|
def uploadOnPypi(self, name):
|
|
print 'Uploading %s on PyPI...' % name
|
|
#self.executeCommand('python setup.py sdist upload')
|
|
|
|
def createZipRelease(self):
|
|
'''Creates a zip file with the appy sources.'''
|
|
newZipRelease = '%s/versions/appy-%s.zip' % (appyPath,self.versionShort)
|
|
if os.path.exists(newZipRelease):
|
|
if not self.askQuestion('"%s" already exists. Replace it?' % \
|
|
newZipRelease, default='yes'):
|
|
print 'Publication canceled.'
|
|
sys.exit(1)
|
|
print 'Removing obsolete %s...' % newZipRelease
|
|
os.remove(newZipRelease)
|
|
zipFile = zipfile.ZipFile(newZipRelease, 'w', zipfile.ZIP_DEFLATED)
|
|
curdir = os.getcwd()
|
|
os.chdir(self.genFolder)
|
|
for dir, dirnames, filenames in os.walk('appy'):
|
|
for f in filenames:
|
|
fileName = os.path.join(dir, f)
|
|
zipFile.write(fileName)
|
|
# [2:] is there to avoid havin './' in the path in the zip file.
|
|
zipFile.close()
|
|
os.chdir(curdir)
|
|
|
|
def applyTemplate(self):
|
|
'''Decorates each page with the template.'''
|
|
# First, load the template into memory
|
|
templateFileName = '%s/doc/template.html' % appyPath
|
|
templateFile = open(templateFileName)
|
|
templateContent = templateFile.read()
|
|
templateFile.close()
|
|
# Then, decorate each other html file
|
|
for pageName in os.listdir(self.genFolder):
|
|
if pageName.endswith('.html'):
|
|
pageFileName = '%s/%s' % (self.genFolder, pageName)
|
|
pageFile = file(pageFileName)
|
|
pageContent = pageFile.read()
|
|
pageFile.close()
|
|
# Extract the page title
|
|
i, j = pageContent.find('<title>'), pageContent.find('</title>')
|
|
pageTitle = pageContent[i+7:j]
|
|
# Extract the body tag content from the page
|
|
pageContent = self.pageBody.search(pageContent).group(1)
|
|
pageFile = open(pageFileName, 'w')
|
|
templateWithTitle = templateContent.replace('{{ title }}',
|
|
pageTitle)
|
|
pageFile.write(templateWithTitle.replace('{{ content }}',
|
|
pageContent))
|
|
pageFile.close()
|
|
|
|
def _getPageTitle(self, url):
|
|
'''Returns the documentation page title from its URL.'''
|
|
res = url.split('.')[0]
|
|
if res not in ('pod', 'gen'):
|
|
res = produceNiceMessage(res[3:])
|
|
return res
|
|
|
|
mainToc = re.compile('<span class="doc"(.*?)</span>', re.S)
|
|
tocLink = re.compile('<a href="(.*?)">(.*?)</a>')
|
|
subSection = re.compile('<h1>(.*?)</h1>')
|
|
subSectionContent = re.compile('<a name="(.*?)">.*?</a>(.*)')
|
|
def createDocToc(self):
|
|
res = '<table width="100%"><tr>'
|
|
docToc = '%s/docToc.html' % self.genFolder
|
|
# First, parse template.html to get the main TOC structure
|
|
template = file('%s/doc/template.html' % appyPath)
|
|
mainData = self.mainToc.search(template.read()).group(0)
|
|
links = self.tocLink.findall(mainData)[1:]
|
|
sectionNb = 0
|
|
for url, title in links:
|
|
if title in ('gen', 'pod'):
|
|
tag = 'h1'
|
|
indent = 0
|
|
styleBegin = ''
|
|
styleEnd = ''
|
|
if title == 'pod':
|
|
res += '</td>'
|
|
res += '<td>'
|
|
else:
|
|
tag = 'p'
|
|
indent = 2
|
|
styleBegin = '<i>'
|
|
styleEnd = '</i>'
|
|
tabs = ' ' * indent * 2
|
|
res += '<%s>%s%s<a href="%s">%s</a>%s</%s>\n' % \
|
|
(tag, tabs, styleBegin, url, self._getPageTitle(url),
|
|
styleEnd, tag)
|
|
# Parse each HTML file and retrieve sections title that have an
|
|
# anchor defined
|
|
docFile = file('%s/doc/%s' % (appyPath, url))
|
|
docContent = docFile.read()
|
|
docFile.close()
|
|
sections = self.subSection.findall(docContent)
|
|
for section in sections:
|
|
r = self.subSectionContent.search(section)
|
|
if r:
|
|
sectionNb += 1
|
|
tabs = ' ' * 8
|
|
res += '<div>%s%d. <a href="%s#%s">%s</a></div>\n' % \
|
|
(tabs, sectionNb, url, r.group(1), r.group(2))
|
|
res += '</td></tr></table>'
|
|
f = file(docToc)
|
|
toc = f.read()
|
|
f.close()
|
|
toc = toc.replace('{{ doc }}', res)
|
|
f = file(docToc, 'w')
|
|
f.write(toc)
|
|
f.close()
|
|
|
|
privateScripts = ('publish.py', 'zip.py', 'startoo.sh')
|
|
def prepareGenFolder(self, minimalist=False):
|
|
'''Creates the basic structure of the temp folder where the appy
|
|
website will be generated.'''
|
|
# Reinitialise temp folder where the generated website will be dumped
|
|
if os.path.exists(self.genFolder):
|
|
FolderDeleter.delete(self.genFolder)
|
|
shutil.copytree('%s/doc' % appyPath, self.genFolder)
|
|
# Create a temp clean copy of appy sources (without .svn folders, etc)
|
|
genSrcFolder = '%s/appy' % self.genFolder
|
|
os.mkdir(genSrcFolder)
|
|
for aFile in ('__init__.py',):
|
|
shutil.copy('%s/%s' % (appyPath, aFile), genSrcFolder)
|
|
for aFolder in ('gen', 'pod', 'shared', 'bin'):
|
|
shutil.copytree('%s/%s' % (appyPath, aFolder),
|
|
'%s/%s' % (genSrcFolder, aFolder))
|
|
# Remove some scripts from bin
|
|
for script in self.privateScripts:
|
|
os.remove('%s/bin/%s' % (genSrcFolder, script))
|
|
if minimalist:
|
|
FolderDeleter.delete('%s/pod/test' % genSrcFolder)
|
|
# Write the appy version into the code itself (in appy/version.py)'''
|
|
print 'Publishing version %s...' % self.versionShort
|
|
# Dump version info in appy/version.py
|
|
f = file('%s/version.py' % genSrcFolder, 'w')
|
|
f.write('short = "%s"\n' % self.versionShort)
|
|
f.write('verbose = "%s"' % self.versionLong)
|
|
f.close()
|
|
# Remove unwanted files
|
|
os.remove('%s/version.txt' % self.genFolder)
|
|
os.remove('%s/license.txt' % self.genFolder)
|
|
os.remove('%s/template.html' % self.genFolder)
|
|
os.remove('%s/artwork.odg' % self.genFolder)
|
|
# Remove subversion folders
|
|
for root, dirs, files in os.walk(self.genFolder):
|
|
for dirName in dirs:
|
|
if dirName == '.svn':
|
|
FolderDeleter.delete(os.path.join(root, dirName))
|
|
|
|
def run(self):
|
|
Cleaner().run(verbose=False)
|
|
# Perform a small analysis on the Appy code
|
|
LinesCounter(appy).run()
|
|
print 'Generating site in %s...' % self.genFolder
|
|
minimalist = self.askQuestion('Minimalist (shipped without tests)?',
|
|
default='no')
|
|
self.prepareGenFolder(minimalist)
|
|
self.createDocToc()
|
|
self.applyTemplate()
|
|
self.createZipRelease()
|
|
tarball = self.createDistRelease()
|
|
self.createDebianRelease()
|
|
if self.askQuestion('Upload %s on PyPI?' % tarball, default='no'):
|
|
self.uploadOnPypi(tarball)
|
|
if self.askQuestion('Publish on appyframework.org?', default='no'):
|
|
AppySite().publish()
|
|
if self.askQuestion('Delete locally generated site ?', default='yes'):
|
|
FolderDeleter.delete(self.genFolder)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
if __name__ == '__main__':
|
|
Publisher().run()
|
|
# ------------------------------------------------------------------------------
|