add filemon stuff

edbob/commands.py

@@ -384,6 +384,45 @@ class DatabaseCommand(Subcommand):
 #         print "Use 'rattail {activate|deactivate} EXTENSION' to change."
 
         
+class FileMonitorCommand(Subcommand):
+    """
+    Interacts with the file monitor Windows service; called as ``edbob
+    filemon``.  This command expects a subcommand; one of the following:
+
+    * ``edbob filemon install``
+    * ``edbob filemon start``
+    * ``edbob filemon stop``
+    * ``edbob filemon uninstall``
+
+    .. note::
+       The Windows Vista family of operating systems requires you to launch
+       ``cmd.exe`` as an Administrator in order to have sufficient rights to
+       run the above commands.
+
+    See :doc:`howto.use_filemon` for more information.
+    """
+
+    name = 'filemon'
+    description = "Manage the file monitor service on Windows"
+
+    def add_parser_args(self, parser):
+        subparsers = parser.add_subparsers(title='subcommands')
+        install = subparsers.add_parser('install',
+                                        help="Install (register) service")
+        install.set_defaults(subcommand='install')
+        uninstall = subparsers.add_parser('uninstall',
+                                          help="Uninstall (unregister) service")
+        uninstall.set_defaults(subcommand='remove')
+        start = subparsers.add_parser('start', help="Start service")
+        start.set_defaults(subcommand='start')
+        stop = subparsers.add_parser('stop', help="Stop service")
+        stop.set_defaults(subcommand='stop')
+
+    def run(self, args):
+        from edbob.filemon import exec_server_command
+        exec_server_command(args.subcommand)
+
+
 class ShellCommand(Subcommand):
     """
     Launches a Python shell (of your choice) with ``edbob`` pre-loaded; called

edbob/db/__init__.py

@@ -26,6 +26,8 @@
 ``edbob.db`` -- Database Framework
 """
 
+from __future__ import absolute_import
+
 from sqlalchemy import engine_from_config, MetaData
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.declarative import declarative_base

edbob/filemon.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+# -*- coding: utf-8  -*-
+################################################################################
+#
+#  edbob -- Pythonic Software Framework
+#  Copyright © 2010-2012 Lance Edgar
+#
+#  This file is part of edbob.
+#
+#  edbob is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU Affero General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option)
+#  any later version.
+#
+#  edbob is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with edbob.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+
+"""
+``edbob.filemon`` -- File Monitoring Service
+"""
+
+# Much of the Windows monitoring code below was borrowed from Tim Golden:
+# http://timgolden.me.uk/python/win32_how_do_i/watch_directory_for_changes.html
+
+import sys
+import os.path
+import threading
+import logging
+import subprocess
+
+import edbob
+from edbob.exceptions import ConfigError
+
+if sys.platform == 'win32':
+    import win32file
+    import win32con
+
+
+FILE_LIST_DIRECTORY = 0x0001
+
+ACTION_CREATE           = 1
+ACTION_DELETE           = 2
+ACTION_UPDATE           = 3
+ACTION_RENAME_TO        = 4
+ACTION_RENAME_FROM      = 5
+
+log = logging.getLogger(__name__)
+
+
+def exec_server_command(command):
+    """
+    Executes ``command`` against the file monitor Windows service, i.e. one of:
+
+    * ``'install'``
+    * ``'start'``
+    * ``'stop'``
+    * ``'remove'``
+    """
+    server_path = os.path.join(os.path.dirname(__file__), 'filemon_server.py')
+    subprocess.call([sys.executable, server_path, command])
+
+
+class MonitorProfile(object):
+    """
+    This is a simple profile class, used to represent configuration of the file
+    monitor service.
+    """
+
+    def __init__(self, key):
+        self.key = key
+        self.dirs = eval(edbob.config.require('edbob.filemon', '%s.dirs' % key))
+        if not self.dirs:
+            raise ConfigError('edbob.filemon', '%s.dirs' % key)
+        self.actions = eval(edbob.config.require('edbob.filemon', '%s.actions' % key))
+        if not self.actions:
+            raise ConfigError('edbob.filemon', '%s.actions' % key)
+
+
+def monitor_win32(path, include_subdirs=False):
+    """
+    This is the workhorse of file monitoring on the Windows platform.  It is a
+    generator function which yields a ``(file_type, file_path, action)`` tuple
+    whenever changes occur in the monitored folder.  ``file_type`` will be one
+    of:
+    
+    * ``'file'``
+    * ``'folder'``
+    * ``'<deleted>'``
+
+    ``file_path`` will be the path to the changed object; and ``action`` will
+    be one of:
+
+    * ``ACTION_CREATE``
+    * ``ACTION_DELETE``
+    * ``ACTION_UPDATE``
+    * ``ACTION_RENAME_TO``
+    * ``ACTION_RENAME_FROM``
+
+    (The above are "constants" importable from ``edbob.filemon``.)
+
+    This function leverages the ``ReadDirectoryChangesW()`` Windows API
+    function.  This is nice because the OS now reports changes to us so that we
+    needn't poll to find them.
+
+    However ``ReadDirectoryChangesW()`` is a blocking call, so in practice this
+    means that if, say, you're using this function in a python shell, you will
+    not be able to stop the process with a keyboard interrupt.  (Actually you
+    sort of can, as long as you send the interrupt and then perform some file
+    operation which will trigger the monitoring function to return.)
+
+    Fortunately the primary need for this function is by the Windows service,
+    which is of course "disconnected" from the user's desktop and never really
+    needs to close under normal circumstances.
+    """
+    hDir = win32file.CreateFile(
+        path,
+        FILE_LIST_DIRECTORY,
+        win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
+        None,
+        win32con.OPEN_EXISTING,
+        win32con.FILE_FLAG_BACKUP_SEMANTICS,
+        None)
+
+    while True:
+        results = win32file.ReadDirectoryChangesW (
+            hDir,
+            1024,
+            include_subdirs,
+            win32con.FILE_NOTIFY_CHANGE_FILE_NAME | 
+            win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
+            win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
+            win32con.FILE_NOTIFY_CHANGE_SIZE |
+            win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
+            win32con.FILE_NOTIFY_CHANGE_SECURITY,
+            None,
+            None)
+
+        for action, fn in results:
+            fpath = os.path.join(path, fn)
+            if not os.path.exists(fpath):
+                ftype = "<deleted>"
+            elif os.path.isdir(fpath):
+                ftype = "folder"
+            else:
+                ftype = "file"
+            yield ftype, fpath, action
+
+
+class WatcherWin32(threading.Thread):
+    """
+    A ``threading.Thread`` subclass which is responsible for monitoring a
+    particular folder (on Windows platforms).
+    """
+
+    def __init__(self, key, path, queue, include_subdirs=False, **kwargs):
+        threading.Thread.__init__(self, **kwargs)
+        self.setDaemon(1)
+        self.key = key
+        self.path = path
+        self.queue = queue
+        self.include_subdirs = include_subdirs
+        self.start()
+
+    def run(self):
+        for result in monitor_win32(self.path,
+                                    include_subdirs=self.include_subdirs):
+            self.queue.put((self.key,) + result)

edbob/filemon_server.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# -*- coding: utf-8  -*-
+################################################################################
+#
+#  edbob -- Pythonic Software Framework
+#  Copyright © 2010-2012 Lance Edgar
+#
+#  This file is part of edbob.
+#
+#  edbob is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU Affero General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option)
+#  any later version.
+#
+#  edbob is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with edbob.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+
+"""
+``edbob.filemon_server`` -- File Monitoring Service for Windows
+"""
+
+import os.path
+import socket
+import time
+import Queue
+
+import edbob
+from edbob.win32 import file_is_free
+from edbob.filemon import (MonitorProfile, WatcherWin32,
+                             ACTION_CREATE, ACTION_UPDATE)
+
+import sys
+if sys.platform == 'win32': # docs should build for everyone
+    import win32serviceutil
+    import win32service
+    import win32event
+    import servicemanager
+    import win32api
+
+
+class FileMonitorService(win32serviceutil.ServiceFramework):
+    """
+    Implements edbob's file monitor Windows service.
+    """
+
+    _svc_name_ = "Edbob File Monitor"
+    _svc_display_name_ = "Edbob : File Monitoring Service"
+    _svc_description_ = ("Monitors one or more folders for incoming files, "
+                         "and performs configured actions as new files arrive.")
+
+    appname = 'edbob'
+
+    def __init__(self, *args):
+        win32serviceutil.ServiceFramework.__init__(self, *args)
+        self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
+        socket.setdefaulttimeout(60)
+        self.stop_requested = False
+
+    def SvcStop(self):
+        self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
+        win32event.SetEvent(self.hWaitStop)
+        self.stop_requested = True
+
+    def SvcDoRun(self):
+        servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
+                              servicemanager.PYS_SERVICE_STARTED,
+                              (self._svc_name_, ''))
+        edbob.init(self.appname)
+        self.main()
+
+    def main(self):
+        self.monitored = {}
+        monitored = edbob.config.require('edbob.filemon', 'monitored')
+        monitored = monitored.split(',')
+        for m in monitored:
+            m = m.strip()
+            self.monitored[m] = MonitorProfile(m)
+        queue = Queue.Queue()
+
+        for key in self.monitored:
+            for d in self.monitored[key].dirs:
+                WatcherWin32(key, d, queue)
+
+        while not self.stop_requested:
+            try:
+                key, ftype, fpath, action = queue.get_nowait()
+            except Queue.Empty:
+                pass
+            else:
+                if ftype == 'file' and action in (
+                    ACTION_CREATE, ACTION_UPDATE):
+                    self.do_actions(key, fpath)
+            win32api.SleepEx(250, True)
+
+        servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
+                              servicemanager.PYS_SERVICE_STOPPED,
+                              (self._svc_name_, ''))
+
+    def do_actions(self, key, path):
+        if not os.path.exists(path):
+            return
+        while not file_is_free(path):
+            # TODO: Add configurable timeout so long-open files can't hijack
+            # our prcessing.
+            time.sleep(0.25)
+        for action in self.monitored[key].actions:
+            if isinstance(action, tuple):
+                func = action[0]
+                args = action[1:]
+            else:
+                func = action
+                args = []
+            func = edbob.load_spec(func)
+            func(path, *args)
+
+    
+if __name__ == '__main__':
+    win32serviceutil.HandleCommandLine(FileMonitorService)

edbob/pyramid/scaffolds/edbob/+package+/filemon.py_tmpl

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+"""
+``{{package}}.filemon`` -- Windows File Monitor
+"""
+
+from edbob import filemon_server
+
+
+class FileMonitorService(filemon_server.FileMonitorService):
+    """
+    Implements the {{project}} file monitor Windows service.
+    """
+
+    _svc_name_ = "{{project}} File Monitor"
+    _svc_display_name_ = "{{project}} : File Monitoring Service"
+
+    appname = '{{package}}'
+
+
+if __name__ == '__main__':
+    import win32serviceutil
+    win32serviceutil.HandleCommandLine(FileMonitorService)

edbob/win32.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -*- coding: utf-8  -*-
+################################################################################
+#
+#  edbob -- Pythonic Software Framework
+#  Copyright © 2010-2012 Lance Edgar
+#
+#  This file is part of edbob.
+#
+#  edbob is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU Affero General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option)
+#  any later version.
+#
+#  edbob is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with edbob.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+
+"""
+``edbob.win32`` -- Stuff for Microsoft Windows
+"""
+
+import sys
+if sys.platform == 'win32': # docs should build for everyone
+    import win32api
+    import win32con
+    import pywintypes
+    import win32file
+    import winerror
+
+
+def RegDeleteTree(key, subkey):
+    """
+    This is a clone of ``win32api.RegDeleteTree()``, since that apparently
+    requires Vista or later.
+    """
+
+    def delete_contents(key):
+        subkeys = []
+        for name, reserved, class_, mtime in win32api.RegEnumKeyEx(key):
+            subkeys.append(name)
+        for subkey_name in subkeys:
+            subkey = win32api.RegOpenKeyEx(key, subkey_name, 0, win32con.KEY_ALL_ACCESS)
+            delete_contents(subkey)
+            win32api.RegCloseKey(subkey)
+            win32api.RegDeleteKey(key, subkey_name)
+        values = []
+        i = 0
+        while True:
+            try:
+                name, value, type_ = win32api.RegEnumValue(key, i)
+            except pywintypes.error, e:
+                if e[0] == winerror.ERROR_NO_MORE_ITEMS:
+                    break
+            values.append(name)
+            i += 1
+        for value in values:
+            win32api.RegDeleteValue(key, value)
+
+    orig_key = key
+    try:
+        key = win32api.RegOpenKeyEx(orig_key, subkey, 0, win32con.KEY_ALL_ACCESS)
+    except pywintypes.error, e:
+        if e[0] != winerror.ERROR_FILE_NOT_FOUND:
+            raise
+    else:
+        delete_contents(key)
+        win32api.RegCloseKey(key)
+    try:
+        win32api.RegDeleteKey(orig_key, subkey)
+    except pywintypes.error, e:
+        if e[0] == winerror.ERROR_FILE_NOT_FOUND:
+            pass
+
+
+def file_is_free(path):
+    """
+    Returns boolean indicating whether or not the file located at ``path`` is
+    currently tied up in any way by another process.
+    """
+
+    # This code was borrowed from Nikita Nemkin:
+    # http://stackoverflow.com/a/2848266
+
+    handle = None
+    try:
+        handle = win32file.CreateFile(
+            path,
+            win32file.GENERIC_WRITE,
+            0,
+            None,
+            win32file.OPEN_EXISTING,
+            win32file.FILE_ATTRIBUTE_NORMAL,
+            None)
+        return True
+    except pywintypes.error, e:
+        if e[0] == winerror.ERROR_SHARING_VIOLATION:
+            return False
+        raise
+    finally:
+        if handle:
+            win32file.CloseHandle(handle)

setup.py

@@ -178,6 +178,7 @@ edbob = edbob.pyramid.scaffolds:Template
 
 [edbob.commands]
 db = edbob.commands:DatabaseCommand
+filemon = edbob.commands:FileMonitorCommand
 shell = edbob.commands:ShellCommand
 uuid = edbob.commands:UuidCommand