lance/edbob
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add filemon stuff

Browse source
This commit is contained in:
Lance Edgar 2012-05-07 15:56:45 -07:00
parent 76ed40950a
commit 371bbf7191
7 changed files with 472 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

39
edbob/commands.py
View file

@ -384,6 +384,45 @@ class DatabaseCommand(Subcommand):
# print "Use 'rattail {activate|deactivate} EXTENSION' to change."
class FileMonitorCommand(Subcommand):
"""
Interacts with the file monitor Windows service; called as ``edbob
filemon``. This command expects a subcommand; one of the following:
* ``edbob filemon install``
* ``edbob filemon start``
* ``edbob filemon stop``
* ``edbob filemon uninstall``
.. note::
The Windows Vista family of operating systems requires you to launch
``cmd.exe`` as an Administrator in order to have sufficient rights to
run the above commands.
See :doc:`howto.use_filemon` for more information.
"""
name = 'filemon'
description = "Manage the file monitor service on Windows"
def add_parser_args(self, parser):
subparsers = parser.add_subparsers(title='subcommands')
install = subparsers.add_parser('install',
help="Install (register) service")
install.set_defaults(subcommand='install')
uninstall = subparsers.add_parser('uninstall',
help="Uninstall (unregister) service")
uninstall.set_defaults(subcommand='remove')
start = subparsers.add_parser('start', help="Start service")
start.set_defaults(subcommand='start')
stop = subparsers.add_parser('stop', help="Stop service")
stop.set_defaults(subcommand='stop')
def run(self, args):
from edbob.filemon import exec_server_command
exec_server_command(args.subcommand)
class ShellCommand(Subcommand):
"""
Launches a Python shell (of your choice) with ``edbob`` pre-loaded; called

2
edbob/db/__init__.py
View file

@ -26,6 +26,8 @@
``edbob.db`` -- Database Framework
"""
from __future__ import absolute_import
from sqlalchemy import engine_from_config, MetaData
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base

174
edbob/filemon.py Normal file
View file

@ -0,0 +1,174 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.filemon`` -- File Monitoring Service
"""
# Much of the Windows monitoring code below was borrowed from Tim Golden:
# http://timgolden.me.uk/python/win32_how_do_i/watch_directory_for_changes.html
import sys
import os.path
import threading
import logging
import subprocess
import edbob
from edbob.exceptions import ConfigError
if sys.platform == 'win32':
import win32file
import win32con
FILE_LIST_DIRECTORY = 0x0001
ACTION_CREATE = 1
ACTION_DELETE = 2
ACTION_UPDATE = 3
ACTION_RENAME_TO = 4
ACTION_RENAME_FROM = 5
log = logging.getLogger(__name__)
def exec_server_command(command):
"""
Executes ``command`` against the file monitor Windows service, i.e. one of:
* ``'install'``
* ``'start'``
* ``'stop'``
* ``'remove'``
"""
server_path = os.path.join(os.path.dirname(__file__), 'filemon_server.py')
subprocess.call([sys.executable, server_path, command])
class MonitorProfile(object):
"""
This is a simple profile class, used to represent configuration of the file
monitor service.
"""
def __init__(self, key):
self.key = key
self.dirs = eval(edbob.config.require('edbob.filemon', '%s.dirs' % key))
if not self.dirs:
raise ConfigError('edbob.filemon', '%s.dirs' % key)
self.actions = eval(edbob.config.require('edbob.filemon', '%s.actions' % key))
if not self.actions:
raise ConfigError('edbob.filemon', '%s.actions' % key)
def monitor_win32(path, include_subdirs=False):
"""
This is the workhorse of file monitoring on the Windows platform. It is a
generator function which yields a ``(file_type, file_path, action)`` tuple
whenever changes occur in the monitored folder. ``file_type`` will be one
of:
* ``'file'``
* ``'folder'``
* ``'<deleted>'``
``file_path`` will be the path to the changed object; and ``action`` will
be one of:
* ``ACTION_CREATE``
* ``ACTION_DELETE``
* ``ACTION_UPDATE``
* ``ACTION_RENAME_TO``
* ``ACTION_RENAME_FROM``
(The above are "constants" importable from ``edbob.filemon``.)
This function leverages the ``ReadDirectoryChangesW()`` Windows API
function. This is nice because the OS now reports changes to us so that we
needn't poll to find them.
However ``ReadDirectoryChangesW()`` is a blocking call, so in practice this
means that if, say, you're using this function in a python shell, you will
not be able to stop the process with a keyboard interrupt. (Actually you
sort of can, as long as you send the interrupt and then perform some file
operation which will trigger the monitoring function to return.)
Fortunately the primary need for this function is by the Windows service,
which is of course "disconnected" from the user's desktop and never really
needs to close under normal circumstances.
"""
hDir = win32file.CreateFile(
path,
FILE_LIST_DIRECTORY,
win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
None,
win32con.OPEN_EXISTING,
win32con.FILE_FLAG_BACKUP_SEMANTICS,
None)
while True:
results = win32file.ReadDirectoryChangesW (
hDir,
1024,
include_subdirs,
win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
win32con.FILE_NOTIFY_CHANGE_SIZE |
win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
win32con.FILE_NOTIFY_CHANGE_SECURITY,
None,
None)
for action, fn in results:
fpath = os.path.join(path, fn)
if not os.path.exists(fpath):
ftype = "<deleted>"
elif os.path.isdir(fpath):
ftype = "folder"
else:
ftype = "file"
yield ftype, fpath, action
class WatcherWin32(threading.Thread):
"""
A ``threading.Thread`` subclass which is responsible for monitoring a
particular folder (on Windows platforms).
"""
def __init__(self, key, path, queue, include_subdirs=False, **kwargs):
threading.Thread.__init__(self, **kwargs)
self.setDaemon(1)
self.key = key
self.path = path
self.queue = queue
self.include_subdirs = include_subdirs
self.start()
def run(self):
for result in monitor_win32(self.path,
include_subdirs=self.include_subdirs):
self.queue.put((self.key,) + result)

125
edbob/filemon_server.py Normal file
View file

@ -0,0 +1,125 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.filemon_server`` -- File Monitoring Service for Windows
"""
import os.path
import socket
import time
import Queue
import edbob
from edbob.win32 import file_is_free
from edbob.filemon import (MonitorProfile, WatcherWin32,
ACTION_CREATE, ACTION_UPDATE)
import sys
if sys.platform == 'win32': # docs should build for everyone
import win32serviceutil
import win32service
import win32event
import servicemanager
import win32api
class FileMonitorService(win32serviceutil.ServiceFramework):
"""
Implements edbob's file monitor Windows service.
"""
_svc_name_ = "Edbob File Monitor"
_svc_display_name_ = "Edbob : File Monitoring Service"
_svc_description_ = ("Monitors one or more folders for incoming files, "
"and performs configured actions as new files arrive.")
appname = 'edbob'
def __init__(self, *args):
win32serviceutil.ServiceFramework.__init__(self, *args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
socket.setdefaulttimeout(60)
self.stop_requested = False
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
self.stop_requested = True
def SvcDoRun(self):
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
(self._svc_name_, ''))
edbob.init(self.appname)
self.main()
def main(self):
self.monitored = {}
monitored = edbob.config.require('edbob.filemon', 'monitored')
monitored = monitored.split(',')
for m in monitored:
m = m.strip()
self.monitored[m] = MonitorProfile(m)
queue = Queue.Queue()
for key in self.monitored:
for d in self.monitored[key].dirs:
WatcherWin32(key, d, queue)
while not self.stop_requested:
try:
key, ftype, fpath, action = queue.get_nowait()
except Queue.Empty:
pass
else:
if ftype == 'file' and action in (
ACTION_CREATE, ACTION_UPDATE):
self.do_actions(key, fpath)
win32api.SleepEx(250, True)
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STOPPED,
(self._svc_name_, ''))
def do_actions(self, key, path):
if not os.path.exists(path):
return
while not file_is_free(path):
# TODO: Add configurable timeout so long-open files can't hijack
# our prcessing.
time.sleep(0.25)
for action in self.monitored[key].actions:
if isinstance(action, tuple):
func = action[0]
args = action[1:]
else:
func = action
args = []
func = edbob.load_spec(func)
func(path, *args)
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(FileMonitorService)

23
edbob/pyramid/scaffolds/edbob/+package+/filemon.py_tmpl Normal file
View file

@ -0,0 +1,23 @@
#!/usr/bin/env python
"""
``{{package}}.filemon`` -- Windows File Monitor
"""
from edbob import filemon_server
class FileMonitorService(filemon_server.FileMonitorService):
"""
Implements the {{project}} file monitor Windows service.
"""
_svc_name_ = "{{project}} File Monitor"
_svc_display_name_ = "{{project}} : File Monitoring Service"
appname = '{{package}}'
if __name__ == '__main__':
import win32serviceutil
win32serviceutil.HandleCommandLine(FileMonitorService)

108
edbob/win32.py Normal file
View file

@ -0,0 +1,108 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.win32`` -- Stuff for Microsoft Windows
"""
import sys
if sys.platform == 'win32': # docs should build for everyone
import win32api
import win32con
import pywintypes
import win32file
import winerror
def RegDeleteTree(key, subkey):
"""
This is a clone of ``win32api.RegDeleteTree()``, since that apparently
requires Vista or later.
"""
def delete_contents(key):
subkeys = []
for name, reserved, class_, mtime in win32api.RegEnumKeyEx(key):
subkeys.append(name)
for subkey_name in subkeys:
subkey = win32api.RegOpenKeyEx(key, subkey_name, 0, win32con.KEY_ALL_ACCESS)
delete_contents(subkey)
win32api.RegCloseKey(subkey)
win32api.RegDeleteKey(key, subkey_name)
values = []
i = 0
while True:
try:
name, value, type_ = win32api.RegEnumValue(key, i)
except pywintypes.error, e:
if e[0] == winerror.ERROR_NO_MORE_ITEMS:
break
values.append(name)
i += 1
for value in values:
win32api.RegDeleteValue(key, value)
orig_key = key
try:
key = win32api.RegOpenKeyEx(orig_key, subkey, 0, win32con.KEY_ALL_ACCESS)
except pywintypes.error, e:
if e[0] != winerror.ERROR_FILE_NOT_FOUND:
raise
else:
delete_contents(key)
win32api.RegCloseKey(key)
try:
win32api.RegDeleteKey(orig_key, subkey)
except pywintypes.error, e:
if e[0] == winerror.ERROR_FILE_NOT_FOUND:
pass
def file_is_free(path):
"""
Returns boolean indicating whether or not the file located at ``path`` is
currently tied up in any way by another process.
"""
# This code was borrowed from Nikita Nemkin:
# http://stackoverflow.com/a/2848266
handle = None
try:
handle = win32file.CreateFile(
path,
win32file.GENERIC_WRITE,
0,
None,
win32file.OPEN_EXISTING,
win32file.FILE_ATTRIBUTE_NORMAL,
None)
return True
except pywintypes.error, e:
if e[0] == winerror.ERROR_SHARING_VIOLATION:
return False
raise
finally:
if handle:
win32file.CloseHandle(handle)

1
setup.py
View file

@ -178,6 +178,7 @@ edbob = edbob.pyramid.scaffolds:Template
[edbob.commands]
db = edbob.commands:DatabaseCommand
filemon = edbob.commands:FileMonitorCommand
shell = edbob.commands:ShellCommand
uuid = edbob.commands:UuidCommand