lance/edbob
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add linux file monitor

Browse source
This commit is contained in:
Lance Edgar 2012-07-14 15:24:58 -05:00
parent b3b4e40bcf
commit 79c4bfa289
5 changed files with 202 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

46
edbob/filemon/__init__.py Normal file
View file

@ -0,0 +1,46 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.filemon`` -- File Monitoring Service
"""
import edbob
from edbob.exceptions import ConfigError
class MonitorProfile(object):
"""
This is a simple profile class, used to represent configuration of the file
monitor service.
"""
def __init__(self, key):
self.key = key
self.dirs = eval(edbob.config.require('edbob.filemon', '%s.dirs' % key))
if not self.dirs:
raise ConfigError('edbob.filemon', '%s.dirs' % key)
self.actions = eval(edbob.config.require('edbob.filemon', '%s.actions' % key))
if not self.actions:
raise ConfigError('edbob.filemon', '%s.actions' % key)

120
edbob/filemon/linux.py Normal file
View file

@ -0,0 +1,120 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.filemon.linux`` -- File Monitor for Linux
"""
import sys
import os
import os.path
import signal
import pyinotify
import edbob
from edbob.filemon import MonitorProfile
class EventHandler(pyinotify.ProcessEvent):
"""
Event processor for file monitor daemon.
"""
def my_init(self, actions=[], **kwargs):
self.actions = actions
def process_IN_CREATE(self, event):
self.perform_actions(event.pathname)
def process_IN_MOVED_TO(self, event):
self.perform_actions(event.pathname)
def perform_actions(self, path):
for action in self.actions:
if isinstance(action, tuple):
func = action[0]
args = action[1:]
else:
func = action
args = []
func = edbob.load_spec(func)
func(path, *args)
def get_pid_path():
"""
Returns the path to the PID file for the file monitor daemon.
"""
basename = os.path.basename(sys.argv[0])
return '/tmp/%s_filemon.pid' % basename
def start_daemon():
"""
Starts the file monitor daemon.
"""
pid_path = get_pid_path()
if os.path.exists(pid_path):
print "File monitor is already running"
return
wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm)
monitored = {}
keys = edbob.config.require('edbob.filemon', 'monitored')
keys = keys.split(',')
for key in keys:
key = key.strip()
monitored[key] = MonitorProfile(key)
mask = pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO
for profile in monitored.itervalues():
for path in profile.dirs:
wm.add_watch(path, mask, proc_fun=EventHandler(actions=profile.actions))
notifier.loop(daemonize=True, pid_file=pid_path)
def stop_daemon():
"""
Stops the file monitor daemon.
"""
pid_path = get_pid_path()
if not os.path.exists(pid_path):
print "File monitor is not running"
return
f = open(pid_path)
pid = f.read().strip()
f.close()
if not pid.isdigit():
print "Hm, found bogus PID:", pid
return
os.kill(int(pid), signal.SIGKILL)
os.remove(pid_path)

158
edbob/filemon/win32.py Normal file
View file

@ -0,0 +1,158 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.filemon.win32`` -- File Monitor for Windows
"""
# Much of the Windows monitoring code below was borrowed from Tim Golden:
# http://timgolden.me.uk/python/win32_how_do_i/watch_directory_for_changes.html
import sys
import os.path
import threading
import logging
import subprocess
import edbob
from edbob.exceptions import ConfigError
if sys.platform == 'win32':
import win32file
import win32con
FILE_LIST_DIRECTORY = 0x0001
ACTION_CREATE = 1
ACTION_DELETE = 2
ACTION_UPDATE = 3
ACTION_RENAME_TO = 4
ACTION_RENAME_FROM = 5
log = logging.getLogger(__name__)
def exec_server_command(command):
"""
Executes ``command`` against the file monitor Windows service, i.e. one of:
* ``'install'``
* ``'start'``
* ``'stop'``
* ``'remove'``
"""
server_path = os.path.join(os.path.dirname(__file__), 'filemon_server.py')
subprocess.call([sys.executable, server_path, command])
def monitor_win32(path, include_subdirs=False):
"""
This is the workhorse of file monitoring on the Windows platform. It is a
generator function which yields a ``(file_type, file_path, action)`` tuple
whenever changes occur in the monitored folder. ``file_type`` will be one
of:
* ``'file'``
* ``'folder'``
* ``'<deleted>'``
``file_path`` will be the path to the changed object; and ``action`` will
be one of:
* ``ACTION_CREATE``
* ``ACTION_DELETE``
* ``ACTION_UPDATE``
* ``ACTION_RENAME_TO``
* ``ACTION_RENAME_FROM``
(The above are "constants" importable from ``edbob.filemon``.)
This function leverages the ``ReadDirectoryChangesW()`` Windows API
function. This is nice because the OS now reports changes to us so that we
needn't poll to find them.
However ``ReadDirectoryChangesW()`` is a blocking call, so in practice this
means that if, say, you're using this function in a python shell, you will
not be able to stop the process with a keyboard interrupt. (Actually you
sort of can, as long as you send the interrupt and then perform some file
operation which will trigger the monitoring function to return.)
Fortunately the primary need for this function is by the Windows service,
which is of course "disconnected" from the user's desktop and never really
needs to close under normal circumstances.
"""
hDir = win32file.CreateFile(
path,
FILE_LIST_DIRECTORY,
win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
None,
win32con.OPEN_EXISTING,
win32con.FILE_FLAG_BACKUP_SEMANTICS,
None)
while True:
results = win32file.ReadDirectoryChangesW (
hDir,
1024,
include_subdirs,
win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
win32con.FILE_NOTIFY_CHANGE_SIZE |
win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
win32con.FILE_NOTIFY_CHANGE_SECURITY,
None,
None)
for action, fn in results:
fpath = os.path.join(path, fn)
if not os.path.exists(fpath):
ftype = "<deleted>"
elif os.path.isdir(fpath):
ftype = "folder"
else:
ftype = "file"
yield ftype, fpath, action
class WatcherWin32(threading.Thread):
"""
A ``threading.Thread`` subclass which is responsible for monitoring a
particular folder (on Windows platforms).
"""
def __init__(self, key, path, queue, include_subdirs=False, **kwargs):
threading.Thread.__init__(self, **kwargs)
self.setDaemon(1)
self.key = key
self.path = path
self.queue = queue
self.include_subdirs = include_subdirs
self.start()
def run(self):
for result in monitor_win32(self.path,
include_subdirs=self.include_subdirs):
self.queue.put((self.key,) + result)

125
edbob/filemon/win32_server.py Normal file
View file

@ -0,0 +1,125 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
#
# edbob -- Pythonic Software Framework
# Copyright © 2010-2012 Lance Edgar
#
# This file is part of edbob.
#
# edbob is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# edbob is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with edbob. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
``edbob.filemon_server`` -- File Monitoring Service for Windows
"""
import os.path
import socket
import time
import Queue
import edbob
from edbob.filemon import MonitorProfile
from edbob.filemon.win32 import WatcherWin32, ACTION_CREATE, ACTION_UPDATE
from edbob.win32 import file_is_free
import sys
if sys.platform == 'win32': # docs should build for everyone
import win32serviceutil
import win32service
import win32event
import servicemanager
import win32api
class FileMonitorService(win32serviceutil.ServiceFramework):
"""
Implements edbob's file monitor Windows service.
"""
_svc_name_ = "Edbob File Monitor"
_svc_display_name_ = "Edbob : File Monitoring Service"
_svc_description_ = ("Monitors one or more folders for incoming files, "
"and performs configured actions as new files arrive.")
appname = 'edbob'
def __init__(self, *args):
win32serviceutil.ServiceFramework.__init__(self, *args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
socket.setdefaulttimeout(60)
self.stop_requested = False
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
self.stop_requested = True
def SvcDoRun(self):
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
(self._svc_name_, ''))
edbob.init(self.appname)
self.main()
def main(self):
self.monitored = {}
monitored = edbob.config.require('edbob.filemon', 'monitored')
monitored = monitored.split(',')
for m in monitored:
m = m.strip()
self.monitored[m] = MonitorProfile(m)
queue = Queue.Queue()
for key in self.monitored:
for d in self.monitored[key].dirs:
WatcherWin32(key, d, queue)
while not self.stop_requested:
try:
key, ftype, fpath, action = queue.get_nowait()
except Queue.Empty:
pass
else:
if ftype == 'file' and action in (
ACTION_CREATE, ACTION_UPDATE):
self.do_actions(key, fpath)
win32api.SleepEx(250, True)
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STOPPED,
(self._svc_name_, ''))
def do_actions(self, key, path):
if not os.path.exists(path):
return
while not file_is_free(path):
# TODO: Add configurable timeout so long-open files can't hijack
# our prcessing.
win32api.SleepEx(250, True)
for action in self.monitored[key].actions:
if isinstance(action, tuple):
func = action[0]
args = action[1:]
else:
func = action
args = []
func = edbob.load_spec(func)
func(path, *args)
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(FileMonitorService)