diff --git a/edbob/commands.py b/edbob/commands.py index 0974ed1..0dc4e9b 100644 --- a/edbob/commands.py +++ b/edbob/commands.py @@ -32,6 +32,7 @@ import sys import argparse import subprocess import logging +import platform import edbob from edbob.util import entry_point_map, requires_impl @@ -420,32 +421,58 @@ class FileMonitorCommand(Subcommand): stop.set_defaults(subcommand='stop') if sys.platform == 'win32': - install = subparsers.add_parser('install', - help="Install (register) service") + install = subparsers.add_parser('install', help="Install service") install.set_defaults(subcommand='install') - uninstall = subparsers.add_parser('uninstall', - help="Uninstall (unregister) service") + install.add_argument('-a', '--auto-start', action='store_true', + help="Configure service to start automatically") + remove = subparsers.add_parser('remove', help="Uninstall (remove) service") + remove.set_defaults(subcommand='remove') + uninstall = subparsers.add_parser('uninstall', help="Uninstall (remove) service") uninstall.set_defaults(subcommand='remove') - def manage_service(self, args, win32): + def get_win32_module(self): + from edbob.filemon import win32 + return win32 + + def get_win32_service(self): + from edbob.filemon.win32 import FileMonitorService + return FileMonitorService + + def get_win32_service_name(self): + service = self.get_win32_service() + return service._svc_name_ + + def run(self, args): if sys.platform == 'linux2': + from edbob.filemon import linux as filemon + if args.subcommand == 'start': - from edbob.filemon.linux import start_daemon - start_daemon() + filemon.start_daemon() + elif args.subcommand == 'stop': - from edbob.filemon.linux import stop_daemon - stop_daemon() + filemon.stop_daemon() elif sys.platform == 'win32': - win32.exec_server_command(args.subcommand) + from edbob import win32 + + filemon = self.get_win32_module() + + # Execute typical service command. + options = [] + if args.subcommand == 'install' and args.auto_start: + options = ['--startup', 'auto'] + win32.execute_service_command(filemon, args.subcommand, *options) + + # If installing auto-start service on Windows 7, we should update + # its startup type to be "Automatic (Delayed Start)". + if args.subcommand == 'install' and args.auto_start: + if platform.release() == '7': + name = self.get_win32_service_name() + win32.delayed_auto_start_service(name) else: print "Sorry, file monitor is not supported on platform %s." % sys.platform - def run(self, args): - from edbob.filemon import win32 - self.manage_service(args, win32) - class ShellCommand(Subcommand): """ diff --git a/edbob/errors.py b/edbob/errors.py index ee1085b..4151350 100644 --- a/edbob/errors.py +++ b/edbob/errors.py @@ -52,11 +52,14 @@ def init(config): sys.excepthook = excepthook -def email_exception(type, value, traceback): +def email_exception(type=None, value=None, traceback=None): """ Sends an email containing a traceback to the configured recipient(s). """ + if not (type and value and traceback): + type, value, traceback = sys.exc_info() + body = StringIO() hostname = socket.gethostname() diff --git a/edbob/filemon/__init__.py b/edbob/filemon/__init__.py index c3293b9..1020da8 100644 --- a/edbob/filemon/__init__.py +++ b/edbob/filemon/__init__.py @@ -26,8 +26,13 @@ ``edbob.filemon`` -- File Monitoring Service """ +import os.path +import logging + import edbob -from edbob.exceptions import ConfigError + + +log = logging.getLogger(__name__) class MonitorProfile(object): @@ -36,11 +41,69 @@ class MonitorProfile(object): monitor service. """ - def __init__(self, key): + def __init__(self, appname, key): + self.appname = appname self.key = key - self.dirs = eval(edbob.config.require('edbob.filemon', '%s.dirs' % key)) - if not self.dirs: - raise ConfigError('edbob.filemon', '%s.dirs' % key) - self.actions = eval(edbob.config.require('edbob.filemon', '%s.actions' % key)) - if not self.actions: - raise ConfigError('edbob.filemon', '%s.actions' % key) + + self.dirs = edbob.config.require('%s.filemon' % appname, '%s.dirs' % key) + self.dirs = eval(self.dirs) + + actions = edbob.config.require('%s.filemon' % appname, '%s.actions' % key) + actions = eval(actions) + + self.actions = [] + for action in actions: + if isinstance(action, tuple): + spec = action[0] + args = list(action[1:]) + else: + spec = action + args = [] + func = edbob.load_spec(spec) + self.actions.append((spec, func, args)) + + +def get_monitor_profiles(appname): + """ + Convenience function to load monitor profiles from config. + """ + + monitored = {} + + # Read monitor profile(s) from config. + keys = edbob.config.require('%s.filemon' % appname, 'monitored') + keys = keys.split(',') + for key in keys: + key = key.strip() + profile = MonitorProfile(appname, key) + monitored[key] = profile + for path in profile.dirs[:]: + + # Ensure the monitored path exists. + if not os.path.exists(path): + log.warning("get_monitor_profiles: Profile '%s' has nonexistent " + "path, which will be pruned: %s" % (key, path)) + profile.dirs.remove(path) + + # Ensure the monitored path is a folder. + elif not os.path.isdir(path): + log.warning("get_monitor_profiles: Profile '%s' has non-folder " + "path, which will be pruned: %s" % (key, path)) + profile.dirs.remove(path) + + for key in monitored.keys(): + profile = monitored[key] + + # Prune any profiles with no valid folders to monitor. + if not profile.dirs: + log.warning("get_monitor_profiles: Profile '%s' has no folders to " + "monitor, and will be pruned." % key) + del monitored[key] + + # Prune any profiles with no valid actions to perform. + elif not profile.actions: + log.warning("get_monitor_profiles: Profile '%s' has no actions to " + "perform, and will be pruned." % key) + del monitored[key] + + return monitored diff --git a/edbob/filemon/win32.py b/edbob/filemon/win32.py index 0b063b3..a2a85e3 100644 --- a/edbob/filemon/win32.py +++ b/edbob/filemon/win32.py @@ -23,136 +23,213 @@ ################################################################################ """ -``edbob.filemon.win32`` -- File Monitor for Windows +``edbob.filemon.win32`` -- File Monitoring Service for Windows """ -# Much of the Windows monitoring code below was borrowed from Tim Golden: -# http://timgolden.me.uk/python/win32_how_do_i/watch_directory_for_changes.html - -import sys import os.path -import threading +import sys +import Queue import logging -import subprocess +import threading import edbob -from edbob.exceptions import ConfigError +from edbob.errors import email_exception +from edbob.filemon import get_monitor_profiles +from edbob.win32 import file_is_free -if sys.platform == 'win32': - import win32file +if sys.platform == 'win32': # docs should build for everyone + import win32api import win32con + import win32event + import win32file + import win32service + import win32serviceutil + import winnt -FILE_LIST_DIRECTORY = 0x0001 - -ACTION_CREATE = 1 -ACTION_DELETE = 2 -ACTION_UPDATE = 3 -ACTION_RENAME_TO = 4 -ACTION_RENAME_FROM = 5 - log = logging.getLogger(__name__) -def exec_server_command(command): +class FileMonitorService(win32serviceutil.ServiceFramework): """ - Executes ``command`` against the file monitor Windows service, i.e. one of: - - * ``'install'`` - * ``'start'`` - * ``'stop'`` - * ``'remove'`` + Implements edbob's file monitor Windows service. """ - server_path = os.path.join(os.path.dirname(__file__), 'filemon_server.py') - subprocess.call([sys.executable, server_path, command]) + _svc_name_ = "Edbob File Monitor" + _svc_display_name_ = "Edbob : File Monitoring Service" + _svc_description_ = ("Monitors one or more folders for incoming files, " + "and performs configured actions as new files arrive.") + + appname = 'edbob' + + def __init__(self, args): + """ + Constructor. + """ + + # super(FileMonitorService, self).__init__(args) + win32serviceutil.ServiceFramework.__init__(self, args) + + # Create "wait stop" event, for main worker loop. + self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) + + def Initialize(self): + """ + Service initialization. + """ + + # Read configuration file(s). + edbob.init(self.appname) + + # Read monitor profile(s) from config. + self.monitored = get_monitor_profiles(self.appname) + + # Make sure we have something to do. + if not self.monitored: + return False + + # Create monitor and action threads for each profile. + for key, profile in self.monitored.iteritems(): + + # Create a file queue for the profile. + queue = Queue.Queue() + + # Create a monitor thread for each folder in profile. + for i, path in enumerate(profile.dirs, 1): + name = 'monitor-%s-%u' % (key, i) + log.debug("Initialize: Starting '%s' thread for folder: %s" % + (name, path)) + thread = threading.Thread( + target=monitor_files, + name=name, + args=(queue, path, profile)) + thread.daemon = True + thread.start() + + # Create an action thread for the profile. + name = 'actions-%s' % key + log.debug("Initialize: Starting '%s' thread" % name) + thread = threading.Thread( + target=perform_actions, + name=name, + args=(queue, profile)) + thread.daemon = True + thread.start() + + return True + + def SvcDoRun(self): + """ + This method is invoked when the service starts. + """ + + import servicemanager + + # Write start occurrence to Windows Event Log. + servicemanager.LogMsg( + servicemanager.EVENTLOG_INFORMATION_TYPE, + servicemanager.PYS_SERVICE_STARTED, + (self._svc_name_, '')) + + # Figure out what we're supposed to be doing. + if self.Initialize(): + + # Wait infinitely for stop request, while threads do their thing. + log.info("SvcDoRun: All threads started; waiting for stop request.") + win32event.WaitForSingleObject(self.hWaitStop, win32event.INFINITE) + log.info("SvcDoRun: Stop request received.") + + else: # Nothing to be done... + msg = "Nothing to do! No valid monitor profiles found in config." + servicemanager.LogWarningMsg(msg) + log.warning("SvcDoRun: %s" % msg) + self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) + + # Write stop occurrence to Windows Event Log. + servicemanager.LogMsg( + servicemanager.EVENTLOG_INFORMATION_TYPE, + servicemanager.PYS_SERVICE_STOPPED, + (self._svc_name_, '')) + + def SvcStop(self): + """ + This method is invoked when the service is requested to stop itself. + """ + + # Let the SCM know we're trying to stop. + self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) + + # Let worker loop know its job is done. + win32event.SetEvent(self.hWaitStop) -def monitor_win32(path, include_subdirs=False): - """ - This is the workhorse of file monitoring on the Windows platform. It is a - generator function which yields a ``(file_type, file_path, action)`` tuple - whenever changes occur in the monitored folder. ``file_type`` will be one - of: - * ``'file'`` - * ``'folder'`` - * ``''`` - - ``file_path`` will be the path to the changed object; and ``action`` will - be one of: - - * ``ACTION_CREATE`` - * ``ACTION_DELETE`` - * ``ACTION_UPDATE`` - * ``ACTION_RENAME_TO`` - * ``ACTION_RENAME_FROM`` - - (The above are "constants" importable from ``edbob.filemon``.) - - This function leverages the ``ReadDirectoryChangesW()`` Windows API - function. This is nice because the OS now reports changes to us so that we - needn't poll to find them. - - However ``ReadDirectoryChangesW()`` is a blocking call, so in practice this - means that if, say, you're using this function in a python shell, you will - not be able to stop the process with a keyboard interrupt. (Actually you - sort of can, as long as you send the interrupt and then perform some file - operation which will trigger the monitoring function to return.) - - Fortunately the primary need for this function is by the Windows service, - which is of course "disconnected" from the user's desktop and never really - needs to close under normal circumstances. +def monitor_files(queue, path, profile): """ + Callable target for file monitor threads. + """ + hDir = win32file.CreateFile( path, - FILE_LIST_DIRECTORY, + winnt.FILE_LIST_DIRECTORY, win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE, None, win32con.OPEN_EXISTING, win32con.FILE_FLAG_BACKUP_SEMANTICS, None) + if hDir == win32file.INVALID_HANDLE_VALUE: + log.warning("monitor_files: Can't open directory with CreateFile(): %s" % path) + return + while True: - results = win32file.ReadDirectoryChangesW ( + results = win32file.ReadDirectoryChangesW( hDir, 1024, - include_subdirs, - win32con.FILE_NOTIFY_CHANGE_FILE_NAME | - win32con.FILE_NOTIFY_CHANGE_DIR_NAME | - win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES | - win32con.FILE_NOTIFY_CHANGE_SIZE | - win32con.FILE_NOTIFY_CHANGE_LAST_WRITE | - win32con.FILE_NOTIFY_CHANGE_SECURITY, - None, - None) + False, + win32con.FILE_NOTIFY_CHANGE_FILE_NAME) - for action, fn in results: - fpath = os.path.join(path, fn) - if not os.path.exists(fpath): - ftype = "" - elif os.path.isdir(fpath): - ftype = "folder" - else: - ftype = "file" - yield ftype, fpath, action + log.debug("monitor_files: ReadDirectoryChangesW() results: %s" % results) + for action, fname in results: + fpath = os.path.join(path, fname) + if action in (winnt.FILE_ACTION_ADDED, + winnt.FILE_ACTION_RENAMED_NEW_NAME): + log.debug("monitor_files: Queueing '%s' file: %s" % + (profile.key, fpath)) + queue.put(fpath) -class WatcherWin32(threading.Thread): +def perform_actions(queue, profile): """ - A ``threading.Thread`` subclass which is responsible for monitoring a - particular folder (on Windows platforms). + Callable target for action threads. """ - def __init__(self, key, path, queue, include_subdirs=False, **kwargs): - threading.Thread.__init__(self, **kwargs) - self.setDaemon(1) - self.key = key - self.path = path - self.queue = queue - self.include_subdirs = include_subdirs - self.start() + while True: - def run(self): - for result in monitor_win32(self.path, - include_subdirs=self.include_subdirs): - self.queue.put((self.key,) + result) + try: + path = queue.get_nowait() + except Queue.Empty: + pass + else: + + while not file_is_free(path): + win32api.Sleep(0) + + for spec, func, args in profile.actions: + + log.info("perform_actions: Calling function '%s' on file: %s" % + (spec, path)) + + try: + func(path, *args) + + except: + log.exception("perform_actions: An exception occurred " + "while processing file: %s" % path) + email_exception() + + # This file probably shouldn't be processed any further. + break + + +if __name__ == '__main__': + win32serviceutil.HandleCommandLine(FileMonitorService) diff --git a/edbob/filemon/win32_server.py b/edbob/filemon/win32_server.py deleted file mode 100644 index 4065f86..0000000 --- a/edbob/filemon/win32_server.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -################################################################################ -# -# edbob -- Pythonic Software Framework -# Copyright © 2010-2012 Lance Edgar -# -# This file is part of edbob. -# -# edbob is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# edbob is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with edbob. If not, see . -# -################################################################################ - -""" -``edbob.filemon_server`` -- File Monitoring Service for Windows -""" - -import os.path -import sys -import socket -import time -import Queue -import logging -from traceback import format_exception - -import edbob -from edbob.filemon import MonitorProfile -from edbob.filemon.win32 import WatcherWin32, ACTION_CREATE, ACTION_UPDATE -from edbob.win32 import file_is_free - -if sys.platform == 'win32': # docs should build for everyone - import win32serviceutil - import win32service - import win32event - import servicemanager - import win32api - - -log = logging.getLogger(__name__) - - -class FileMonitorService(win32serviceutil.ServiceFramework): - """ - Implements edbob's file monitor Windows service. - """ - - _svc_name_ = "Edbob File Monitor" - _svc_display_name_ = "Edbob : File Monitoring Service" - _svc_description_ = ("Monitors one or more folders for incoming files, " - "and performs configured actions as new files arrive.") - - appname = 'edbob' - - def __init__(self, *args): - win32serviceutil.ServiceFramework.__init__(self, *args) - self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) - socket.setdefaulttimeout(60) - self.stop_requested = False - - def SvcStop(self): - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - win32event.SetEvent(self.hWaitStop) - self.stop_requested = True - - def SvcDoRun(self): - servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, - servicemanager.PYS_SERVICE_STARTED, - (self._svc_name_, '')) - edbob.init(self.appname) - self.main() - - def main(self): - self.monitored = {} - monitored = edbob.config.require('edbob.filemon', 'monitored') - monitored = monitored.split(',') - for key in monitored: - key = key.strip() - profile = MonitorProfile(key) - self.monitored[key] = profile - log.debug("Monitoring profile '%s': %s" % (key, profile.dirs)) - for path in profile.dirs: - if not os.path.exists(path): - log.warning("Path does not exist: %s" % path) - queue = Queue.Queue() - - for key in self.monitored: - for d in self.monitored[key].dirs: - WatcherWin32(key, d, queue) - - while not self.stop_requested: - try: - key, ftype, fpath, action = queue.get_nowait() - except Queue.Empty: - pass - else: - log.debug("Got notification: %s, %s, %s" % (key, ftype, fpath)) - # if ftype == 'file' and action in ( - # ACTION_CREATE, ACTION_UPDATE): - if ftype == 'file' and action == ACTION_CREATE: - self.do_actions(key, fpath) - win32api.SleepEx(250, True) - - servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, - servicemanager.PYS_SERVICE_STOPPED, - (self._svc_name_, '')) - - def do_actions(self, key, path): - if not os.path.exists(path): - return - while not file_is_free(path): - # TODO: Add configurable timeout so long-open files can't hijack - # our prcessing. - win32api.SleepEx(250, True) - for action in self.monitored[key].actions: - if isinstance(action, tuple): - func = action[0] - args = action[1:] - else: - func = action - args = [] - func = edbob.load_spec(func) - - try: - func(path, *args) - except: - - exc_info = sys.exc_info() - - # Call the system exception hook in case anything special has - # been registered there, e.g. if edbob.errors.init() has - # happened. Note that this is especially necessary since - # PythonService.exe doesn't seem to honor sys.excepthook. - sys.excepthook(*exc_info) - - # Go ahead and write exception info to the Windows Event Log - # while we're at it. - msg = "File monitor action failed.\n" - msg += "\n" - msg += "Profile: %s\n" % key - msg += "Action: %s\n" % action - msg += "File Path: %s\n" % path - msg += "\n" - msg += ''.join(format_exception(*exc_info)) - servicemanager.LogErrorMsg(msg) - - # Don't re-raise the exception since the service should - # continue running despite any problems it encounters. But - # this file probably shouldn't be processed any further. - break - - -if __name__ == '__main__': - win32serviceutil.HandleCommandLine(FileMonitorService) diff --git a/edbob/win32.py b/edbob/win32.py index d33015c..4051118 100644 --- a/edbob/win32.py +++ b/edbob/win32.py @@ -27,13 +27,16 @@ """ import sys +import subprocess + if sys.platform == 'win32': # docs should build for everyone + import pywintypes import win32api import win32con - import pywintypes import win32file - import winerror import win32print + import win32service + import winerror def RegDeleteTree(key, subkey): @@ -80,6 +83,62 @@ def RegDeleteTree(key, subkey): pass +def delayed_auto_start_service(name): + """ + Configures the Windows service named ``name`` such that its startup type is + "Automatic (Delayed Start)". + + .. note:: + It is assumed that the service is already configured to start + automatically. This function only modifies the service so that its + automatic startup is delayed. + """ + + hSCM = win32service.OpenSCManager( + None, + None, + win32service.SC_MANAGER_ENUMERATE_SERVICE) + + hService = win32service.OpenService( + hSCM, + name, + win32service.SERVICE_CHANGE_CONFIG) + + win32service.ChangeServiceConfig2( + hService, + win32service.SERVICE_CONFIG_DELAYED_AUTO_START_INFO, + True) + + win32service.CloseServiceHandle(hService) + win32service.CloseServiceHandle(hSCM) + + +def execute_service_command(module, command, *args): + """ + Executes ``command`` against the Windows service contained in ``module``. + + ``module`` must be a proper module object, which is assumed to implement a + command line interface when invoked directly by the Python interpreter, a + la ``win32serviceutil.HandleCommandLine()``. + + ``command`` may be anything supported by ``HandleCommandLine()``, e.g.: + + * ``'install'`` + * ``'remove'`` + * ``'start'`` + * ``'stop'`` + * ``'restart'`` + + ``args``, if present, are assumed to be "option" arguments and will precede + ``command`` when the command line is constructed. + """ + + command = [command] + if args: + command = list(args) + command + subprocess.call([sys.executable, module.__file__] + command) + + def file_is_free(path): """ Returns boolean indicating whether or not the file located at ``path`` is