appypod-rattail/bin/backup.py

372 lines
19 KiB
Python

# ------------------------------------------------------------------------------
import sys, time, os, os.path, smtplib, socket, popen2, shutil
from optparse import OptionParser
import ZODB.FileStorage
import ZODB.serialize
from DateTime import DateTime
from StringIO import StringIO
folderName = os.path.dirname(__file__)
# ------------------------------------------------------------------------------
class BackupError(Exception): pass
ERROR_CODE = 1
# ------------------------------------------------------------------------------
class ZodbBackuper:
'''This backuper will run every night (after 00.00). Every night excepted
Sunday, it will perform an incremental backup. Every Sunday, the script
will pack the ZODB, perform a full backup, and, if successful, remove all
previous (full and incremental) backups.'''
fullBackupExts = ('.fs', '.fsz')
toRemoveExts = ('.doc', '.pdf', '.rtf', '.odt')
def __init__(self, storageLocation, backupFolder, options):
self.storageLocation = storageLocation
self.backupFolder = backupFolder
self.options = options
# Unwrap some options directly on self.
self.repozo = options.repozo or './repozo.py'
self.zopectl = options.zopectl or './zopectl'
self.logFile = file(options.logFile, 'a')
self.logMem = StringIO() # We keep a log of the last script execution,
# so we can send this info by email.
self.emails = options.emails
self.tempFolder = options.tempFolder
self.logsBackupFolder = options.logsBackupFolder
self.zopeUser = options.zopeUser
self.keepSeconds = int(options.keepSeconds)
def log(self, msg):
for logPlace in (self.logFile, self.logMem):
logPlace.write(msg)
logPlace.write('\n')
def executeCommand(self, cmd):
'''Executes command p_cmd.'''
w = self.log
w('Executing "%s"...' % cmd)
outstream, instream = popen2.popen4(cmd)
outTxt = outstream.readlines()
instream.close()
outstream.close()
for line in outTxt:
w(line[:-1])
w('Done.')
def packZodb(self):
'''Packs the ZODB and keeps one week history.'''
storage = ZODB.FileStorage.FileStorage(self.storageLocation)
#storage.pack(time.time()-(7*24*60*60), ZODB.serialize.referencesf)
storage.pack(time.time()-self.keepSeconds, ZODB.serialize.referencesf)
for fileSuffix in ('', '.index'):
fileName = self.storageLocation + fileSuffix
os.system('chown %s %s' % (self.zopeUser, fileName))
folderCreateError = 'Could not create backup folder. Backup of log ' \
'files will not take place. %s'
def backupLogs(self):
w = self.log
if not os.path.exists(self.logsBackupFolder):
# Try to create the folder when to store backups of the log files
try:
w('Try to create backup folder for logs "%s"...' % \
self.logsBackupFolder)
os.mkdir(self.logsBackupFolder)
except IOError, ioe:
w(folderCreateError % str(ioe))
except OSError, oe:
w(folderCreateError % str(oe))
if os.path.exists(self.logsBackupFolder):
# Ok, we can make the backup of the log files.
# Get the folder where logs lie
logsFolder = self.options.logsFolder
d = os.path.dirname
j = os.path.join
if not logsFolder:
logsFolder = j(d(d(self.storageLocation)), 'log')
if not os.path.isdir(logsFolder):
w('Cannot backup log files because folder "%s" does not ' \
'exist. Try using option "-g".' % logsFolder)
return
for logFileName in os.listdir(logsFolder):
if logFileName.endswith('.log'):
backupTime = DateTime().strftime('%Y_%m_%d_%H_%M')
parts = os.path.splitext(logFileName)
copyFileName = '%s.%s%s' % (parts[0], backupTime, parts[1])
absCopyFileName = j(self.logsBackupFolder, copyFileName)
absLogFileName = j(logsFolder, logFileName)
w('Moving "%s" to "%s"...' % (absLogFileName,
absCopyFileName))
shutil.copyfile(absLogFileName, absCopyFileName)
os.remove(absLogFileName)
# I do a "copy" + a "remove" instead of a "rename" because
# a "rename" fails if the source and dest files are on
# different physical devices.
def getDate(self, dateString):
'''Returns a DateTime instance from p_dateString, which has the form
YYYY-MM-DD-HH-MM-SS.'''
return DateTime('%s/%s/%s %s:%s:%s' % tuple(dateString.split('-')))
def removeOldBackups(self):
'''This method removes all files (full & incremental backups) that are
older than the last full backup.'''
w = self.log
# Determine date of the oldest full backup
oldestFullBackupDate = eighties = DateTime('1980/01/01')
for backupFile in os.listdir(self.backupFolder):
fileDate, ext = os.path.splitext(backupFile)
if ext in self.fullBackupExts:
# I have found a full backup
fileDate = self.getDate(fileDate)
if fileDate > oldestFullBackupDate:
oldestFullBackupDate = fileDate
# Remove all backup files older that oldestFullBackupDate
if oldestFullBackupDate != eighties:
w('Last full backup date: %s' % str(oldestFullBackupDate))
for backupFile in os.listdir(self.backupFolder):
fileDate, ext = os.path.splitext(backupFile)
if self.getDate(fileDate) < oldestFullBackupDate:
fullFileName = '%s/%s' % (self.backupFolder, backupFile)
w('Removing old backup file %s...' % fullFileName)
os.remove(fullFileName)
def sendEmails(self):
'''Send content of self.logMem to self.emails.'''
w = self.log
subject = 'Backup notification.'
msg = 'From: %s\nTo: %s\nSubject: %s\n\n%s' % (self.options.fromAddress,
self.emails, subject, self.logMem.getvalue())
try:
w('> Sending mail notifications to %s...' % self.emails)
smtpInfo = self.options.smtpServer.split(':', 3)
login = password = None
if len(smtpInfo) == 2:
# We simply have server and port
server, port = smtpInfo
else:
# We also have login and password
server, port, login, password = smtpInfo
smtpServer = smtplib.SMTP(server, port=int(port))
if login:
smtpServer.login(login, password)
res = smtpServer.sendmail(self.options.fromAddress,
self.emails.split(','), msg)
smtpServer.quit()
if res:
w('Could not send mail to some recipients. %s' % str(res))
w('Done.')
except socket.error, se:
w('Could not connect to SMTP server %s (%s).' % \
(self.options.smtpServer, str(se)))
def removeTempFiles(self):
'''For EGW, OO produces temp files that EGW tries do delete at the time
they are produced. But in some cases EGW can't do it (ie Zope runs
with a given user and OO runs with root and produces files that can't
be deleted by the user running Zope). This is why in this script we
remove the temp files that could not be removed by Zope.'''
w = self.log
w('Removing temp files in "%s"...' % self.tempFolder)
pdfCount = docCount = rtfCount = odtCount = 0
for fileName in os.listdir(self.tempFolder):
ext = os.path.splitext(fileName)[1]
if ext in self.toRemoveExts:
exec '%sCount += 1' % ext[1:]
fullFileName = os.path.join(self.tempFolder, fileName)
#w('Removing "%s"...' % fullFileName)
try:
os.remove(fullFileName)
except OSError, oe:
w('Could not remove "%s" (%s).' % (fullFileName, str(oe)))
w('%d .pdf, %d .doc, %d .rtf and %d .odt file(s) removed.' % \
(pdfCount, docCount, rtfCount, odtCount))
def run(self):
w = self.log
startTime = time.time()
w('\n****** Backup launched at %s ******' % str(time.asctime()))
# Shutdown the Zope instance
w('> Shutting down Zope instance...')
self.executeCommand('%s stop' % self.zopectl)
# If we are on the "full backup day", let's pack the ZODB first
if time.asctime().startswith(self.options.dayFullBackup):
w('> Day is "%s", packing the ZODB...' % self.options.dayFullBackup)
self.packZodb()
w('> Make a backup of log files...')
self.backupLogs()
w('Done.')
# Do the backup with repozo
w('> Performing backup...')
self.executeCommand('%s %s -BvzQ -r %s -f %s' % (self.options.python,
self.repozo, self.backupFolder, self.storageLocation))
# Remove previous full backups.
self.removeOldBackups()
# If a command is specified, run Zope to execute this command
if self.options.command:
w('> Executing command "%s"...' % self.options.command)
jobScript = '%s/job.py' % folderName
cmd = '%s run %s %s' % (self.zopectl, jobScript,
self.options.command)
self.executeCommand(cmd)
# Start the instance again, in normal mode.
w('> Restarting Zope instance...')
self.executeCommand('%s start' % self.zopectl)
self.removeTempFiles()
stopTime = time.time()
w('Done in %d minute(s).' % ((stopTime-startTime)/60))
if self.emails:
self.sendEmails()
self.logFile.close()
print self.logMem.getvalue()
self.logMem.close()
# ------------------------------------------------------------------------------
class ZodbBackupScript:
'''usage: python backup.py storageLocation backupFolder [options]
storageLocation is the path to a ZODB database (file storage) (ie
/opt/ZopeInstance/var/Data.fs);
backupFolder is a folder exclusively dedicated for storing backups
of the mentioned storage (ie /data/zodbbackups).'''
weekDays = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
def checkArgs(self, options, args):
'''Check that the scripts arguments are correct.'''
# Do I have the correct number of args?
if len(args) != 2:
raise BackupError('Wrong number of arguments.')
# Check storageLocation
if not os.path.exists(args[0]) or not os.path.isfile(args[0]):
raise BackupError('"%s" does not exist or is not a file.' % args[0])
# Check backupFolder
if not os.path.isdir(args[1]):
raise BackupError('"%s" does not exist or is not a folder.'%args[1])
# Check logs folder
if options.logsFolder and not os.path.isdir(options.logsFolder):
raise BackupError('"%s" is not a folder.' % options.logsFolder)
# Try to create a file in this folder to check if we have write
# access in it.
fileName = '%s/%s.tmp' % (args[1], str(time.time()))
try:
f = file(fileName, 'w')
f.write('Hello.')
f.close()
os.remove(fileName)
except OSError, oe:
raise BackupError('I do not have the right to write in ' \
'folder "%s".' % args[1])
# Check temp folder
if not os.path.isdir(options.tempFolder):
raise BackupError('Temp folder "%s" does not exist or is not ' \
'a folder.' % options.tempFolder)
# Check day of week
if options.dayFullBackup not in self.weekDays:
raise BackupError(
'Day of week must be one of %s' % str(self.weekDays))
# Check command format
if options.command:
parts = options.command.split(':')
if len(parts) not in (4,5):
raise BackupError('Command format must be ' \
'<ZopeAdmin><PloneInstancePath>:<ApplicationName>:' \
'<ToolMethodName>[:<args>]')
def run(self):
optParser = OptionParser(usage=ZodbBackupScript.__doc__)
optParser.add_option("-p", "--python", dest="python",
help="The path to the Python interpreter running "\
"Zope",
default='python2.4',metavar="REPOZO",type='string')
optParser.add_option("-r", "--repozo", dest="repozo",
help="The path to repozo.py",
default='', metavar="REPOZO", type='string')
optParser.add_option("-z", "--zopectl", dest="zopectl",
help="The path to Zope instance's zopectl script",
default='', metavar="ZOPECTL", type='string')
optParser.add_option("-l", "--logfile", dest="logFile",
help="Log file where this script will append " \
"output (defaults to ./backup.log)",
default='./backup.log', metavar="LOGFILE",
type='string')
optParser.add_option("-d", "--day-full-backup", dest="dayFullBackup",
help="Day of the week where the full backup " \
"must be performed (defaults to 'Sun'). " \
"Must be one of %s" % str(self.weekDays),
default='Sun', metavar="DAYFULLBACKUP",
type='string')
optParser.add_option("-e", "--emails", dest="emails",
help="Comma-separated list of emails that will " \
"receive the log of this script.",
default='', metavar="EMAILS", type='string')
optParser.add_option("-f", "--from-address", dest="fromAddress",
help="From address for the sent mails",
default='', metavar="FROMADDRESS", type='string')
optParser.add_option("-s", "--smtp-server", dest="smtpServer",
help="SMTP server and port (ie: localhost:25) " \
"for sending mails. You can also embed " \
"username and password if the SMTP server " \
"requires authentication, ie " \
"localhost:25:myLogin:myPassword",
default='localhost:25', metavar="SMTPSERVER",
type='string')
optParser.add_option("-t", "--tempFolder", dest="tempFolder",
help="Folder used by OO for producing temp " \
"files. Defaults to /tmp.",
default='/tmp', metavar="TEMP", type='string')
optParser.add_option("-g", "--logsFolder",dest="logsFolder",
help="Folder where Zope log files are " \
"(typically: event.log and Z2.log). If no " \
"folder is provided, we will consider to " \
"work on a standard Zope instance and " \
"decide that the log folder is, from " \
"'storageLocation', located at ../log",
metavar="LOGSFOLDER", type='string')
optParser.add_option("-b", "--logsBackupFolder",dest="logsBackupFolder",
help="Folder where backups of log files " \
"(event.log and Z2.log) will be stored.",
default='./logsbackup', metavar="LOGSBACKUPFOLDER",
type='string')
optParser.add_option("-u", "--user", dest="zopeUser",
help="User and group that must own Data.fs. " \
"Defaults to zope:www-data. If " \
"this script is launched by root, for " \
"example, when packing the ZODB this script "\
"may produce a new Data.fs that the user " \
"running Zope may not be able to read " \
"anymore. After packing, this script makes " \
"a 'chmod' on Data.fs.",
default='zope:www-data', metavar="USER",
type='string')
optParser.add_option("-k", "--keep-seconds", dest="keepSeconds",
help="Number of seconds to leave in the ZODB " \
"history when the ZODB is packed.",
default='86400', metavar="KEEPSECONDS",
type='string')
optParser.add_option("-c", "--command", dest="command",
help="Command to execute while Zope is running. It must have the " \
"following format: <ZopeAdmin>:<PloneInstancePath>:" \
"<ApplicationName>:<ToolMethodName>[:<args>]. <ZopeAdmin> is the " \
"user name of the Zope administrator; <PloneInstancePath> is the " \
"path, within Zope, to the Plone Site object (if not at the " \
"root of the Zope hierarchy, use '/' as folder separator); " \
"<ApplicationName> is the name of the Appy application; " \
"<ToolMethodName> is the name of the method to call on the tool " \
"in this Appy application; (optional) <args> are the arguments " \
"to give to this method (only strings are supported). Several " \
"arguments must be separated by '*'.", default='',
metavar="COMMAND", type='string')
(options, args) = optParser.parse_args()
try:
self.checkArgs(options, args)
backuper = ZodbBackuper(args[0], args[1], options)
backuper.run()
except BackupError, be:
sys.stderr.write(str(be) + '\nRun the script without any ' \
'argument for getting help.\n')
sys.exit(ERROR_CODE)
# ------------------------------------------------------------------------------
if __name__ == '__main__':
ZodbBackupScript().run()
# ------------------------------------------------------------------------------