From 896a320da5683df42af1842c81f41a3379f2a6a8 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 16 Feb 2013 17:11:02 -0800 Subject: [PATCH 01/18] Overhauled file monitors. Added the "process existing" (defaults to True) and "stop on error" (defaults to False) features to both Linux and Win32 file monitors. Both features may be overridden in config. The Linux file monitor was rewritten as an ``edbob.daemon.Daemon`` class. --- edbob/commands.py | 9 +-- edbob/filemon/__init__.py | 116 +++++++++++++++++++++++++++++++ edbob/filemon/linux.py | 141 +++++++++++++++++++------------------- edbob/filemon/win32.py | 63 +++++------------ 4 files changed, 204 insertions(+), 125 deletions(-) diff --git a/edbob/commands.py b/edbob/commands.py index 6c47598..3505473 100644 --- a/edbob/commands.py +++ b/edbob/commands.py @@ -432,11 +432,6 @@ class FileMonitorCommand(Subcommand): uninstall = subparsers.add_parser('uninstall', help="Uninstall (remove) service") uninstall.set_defaults(subcommand='remove') - else: - parser.add_argument('-D', '--dont-daemonize', - action='store_false', dest='daemonize', - help="Don't daemonize when starting") - def get_win32_module(self): from edbob.filemon import win32 return win32 @@ -454,10 +449,10 @@ class FileMonitorCommand(Subcommand): from edbob.filemon import linux as filemon if args.subcommand == 'start': - filemon.start_daemon(self.appname, daemonize=args.daemonize) + filemon.start_daemon(self.appname) elif args.subcommand == 'stop': - filemon.stop_daemon() + filemon.stop_daemon(self.appname) elif sys.platform == 'win32': from edbob import win32 diff --git a/edbob/filemon/__init__.py b/edbob/filemon/__init__.py index 07662e5..205ac31 100644 --- a/edbob/filemon/__init__.py +++ b/edbob/filemon/__init__.py @@ -26,10 +26,18 @@ ``edbob.filemon`` -- File Monitoring Service """ +import os import os.path +import sys +import Queue import logging import edbob +from edbob.errors import email_exception + +if sys.platform == 'win32': + import win32api + from edbob.win32 import file_is_free log = logging.getLogger(__name__) @@ -65,6 +73,12 @@ class MonitorProfile(object): self.locks = edbob.config.getboolean( '%s.filemon' % appname, '%s.locks' % key, default=False) + self.process_existing = edbob.config.getboolean( + '%s.filemon' % appname, '%s.process_existing' % key, default=True) + + self.stop_on_error = edbob.config.getboolean( + '%s.filemon' % appname, '%s.stop_on_error' % key, default=False) + def get_monitor_profiles(appname): """ @@ -110,3 +124,105 @@ def get_monitor_profiles(appname): del monitored[key] return monitored + + +def queue_existing(profile, path): + """ + Adds files found in a watched folder to a processing queue. This is called + when the monitor first starts, to handle the case of files which exist + prior to startup. + + If files are found, they are first sorted by modification timestamp, using + a lexical sort on the filename as a tie-breaker, and then added to the + queue in that order. + + :param profile: Monitor profile for which the folder is to be watched. The + profile is expected to already have a queue attached; any existing files + will be added to this queue. + :type profile: :class:`edbob.filemon.MonitorProfile` instance + + :param path: Folder path which is to be checked for files. + :type path: string + + :returns: ``None`` + """ + + def sorter(x, y): + mtime_x = os.path.getmtime(x) + mtime_y = os.path.getmtime(y) + if mtime_x < mtime_y: + return -1 + if mtime_x > mtime_y: + return 1 + return cmp(x, y) + + paths = [os.path.join(path, x) for x in os.listdir(path)] + for path in sorted(paths, cmp=sorter): + + # Only process normal files. + if not os.path.isfile(path): + continue + + # If using locks, don't process "in transit" files. + if profile.locks and path.endswith('.lock'): + continue + + log.debug("queue_existing: queuing existing file for " + "profile '%s': %s" % (profile.key, path)) + profile.queue.put(path) + + +def perform_actions(profile): + """ + Callable target for action threads. + """ + + keep_going = True + while keep_going: + + try: + path = profile.queue.get_nowait() + except Queue.Empty: + pass + else: + + # In some cases, processing one file may cause other related files + # to also be processed. When this happens, a path on the queue may + # point to a file which no longer exists. + if not os.path.exists(path): + log.info("perform_actions: path does not exist: %s" % path) + continue + + log.debug("perform_actions: processing file: %s" % path) + + if sys.platform == 'win32': + while not file_is_free(path): + win32api.Sleep(0) + + for spec, func, args in profile.actions: + + log.info("perform_actions: calling function '%s' on file: %s" % + (spec, path)) + + try: + func(path, *args) + + except: + log.exception("perform_actions: exception occurred " + "while processing file: %s" % path) + email_exception() + + # Don't process any more files if the profile is so + # configured. + if profile.stop_on_error: + keep_going = False + + # Either way this particular file probably shouldn't be + # processed any further. + log.warning("perform_actions: no further processing " + "will be done for file: %s" % path) + break + + log.warning("perform_actions: error encountered, and configuration " + "dictates that no more actions will be processed for " + "profile: %s" % profile.key) diff --git a/edbob/filemon/linux.py b/edbob/filemon/linux.py index d5902cc..6d5ab83 100644 --- a/edbob/filemon/linux.py +++ b/edbob/filemon/linux.py @@ -27,14 +27,16 @@ """ import sys -import os import os.path -import signal -import logging import pyinotify +import threading +import Queue +import logging import edbob -from edbob.filemon import get_monitor_profiles +from edbob import filemon +from edbob.daemon import Daemon +from edbob.errors import email_exception log = logging.getLogger(__name__) @@ -45,9 +47,8 @@ class EventHandler(pyinotify.ProcessEvent): Event processor for file monitor daemon. """ - def my_init(self, actions=[], locks=False, **kwargs): - self.actions = actions - self.locks = locks + def my_init(self, profile=None, **kwargs): + self.profile = profile def process_IN_ACCESS(self, event): log.debug("EventHandler: IN_ACCESS: %s" % event.pathname) @@ -57,87 +58,85 @@ class EventHandler(pyinotify.ProcessEvent): def process_IN_CLOSE_WRITE(self, event): log.debug("EventHandler: IN_CLOSE_WRITE: %s" % event.pathname) - if not self.locks: - self.perform_actions(event.pathname) + if not self.profile.locks: + self.profile.queue.put(event.pathname) def process_IN_CREATE(self, event): log.debug("EventHandler: IN_CREATE: %s" % event.pathname) def process_IN_DELETE(self, event): log.debug("EventHandler: IN_DELETE: %s" % event.pathname) - if self.locks and event.pathname.endswith('.lock'): - self.perform_actions(event.pathname[:-5]) + if self.profile.locks and event.pathname.endswith('.lock'): + self.profile.queue.put(event.pathname[:-5]) def process_IN_MODIFY(self, event): log.debug("EventHandler: IN_MODIFY: %s" % event.pathname) def process_IN_MOVED_TO(self, event): log.debug("EventHandler: IN_MOVED_TO: %s" % event.pathname) - if not self.locks: - self.perform_actions(event.pathname) - - def perform_actions(self, path): - for spec, func, args in self.actions: - func(path, *args) + if not self.profile.locks: + self.profile.queue.put(event.pathname) -def get_pid_path(): - """ - Returns the path to the PID file for the file monitor daemon. - """ +class FileMonitorDaemon(Daemon): - basename = os.path.basename(sys.argv[0]) - pid_path = edbob.config.get('%s.filemon' % basename, 'pid_path') + def run(self): + + wm = pyinotify.WatchManager() + notifier = pyinotify.Notifier(wm) + + mask = (pyinotify.IN_ACCESS + | pyinotify.IN_ATTRIB + | pyinotify.IN_CLOSE_WRITE + | pyinotify.IN_CREATE + | pyinotify.IN_DELETE + | pyinotify.IN_MODIFY + | pyinotify.IN_MOVED_TO) + + monitored = filemon.get_monitor_profiles(self.appname) + for key, profile in monitored.iteritems(): + + # Create a file queue for the profile. + profile.queue = Queue.Queue() + + # Perform setup for each of the watched folders. + for path in profile.dirs: + + # Maybe put all pre-existing files in the queue. + if profile.process_existing: + filemon.queue_existing(profile, path) + + # Create a watch for the folder. + log.debug("start_daemon: profile '%s' watches folder: %s" % (key, path)) + wm.add_watch(path, mask, proc_fun=EventHandler(profile=profile)) + + # Create an action thread for the profile. + name = 'actions-%s' % key + log.debug("start_daemon: starting action thread: %s" % name) + thread = threading.Thread(target=filemon.perform_actions, + name=name, args=(profile,)) + thread.daemon = True + thread.start() + + # Fire up the watchers. + notifier.loop() + + +def get_daemon(appname=None): + if appname is None: + appname = os.path.basename(sys.argv[0]) + pid_path = edbob.config.get('%s.filemon' % appname, 'pid_path') if not pid_path: - pid_path = '/tmp/%s_filemon.pid' % basename - return pid_path + pid_path = '/tmp/%s_filemon.pid' % appname + + monitor = FileMonitorDaemon(pid_path) + monitor.appname = appname + return monitor -def start_daemon(appname, daemonize=True): - """ - Starts the file monitor daemon. - """ - - pid_path = get_pid_path() - if os.path.exists(pid_path): - print "File monitor is already running" - return - - wm = pyinotify.WatchManager() - notifier = pyinotify.Notifier(wm) - - monitored = get_monitor_profiles(appname) - - mask = (pyinotify.IN_ACCESS | pyinotify.IN_ATTRIB - | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE - | pyinotify.IN_DELETE | pyinotify.IN_MODIFY - | pyinotify.IN_MOVED_TO) - for profile in monitored.itervalues(): - for path in profile.dirs: - wm.add_watch(path, mask, proc_fun=EventHandler( - actions=profile.actions, locks=profile.locks)) - - if not daemonize: - sys.stderr.write("Starting file monitor. (Press Ctrl+C to quit.)\n") - notifier.loop(daemonize=daemonize, pid_file=pid_path) +def start_daemon(appname): + get_daemon(appname).start() -def stop_daemon(): - """ - Stops the file monitor daemon. - """ - - pid_path = get_pid_path() - if not os.path.exists(pid_path): - print "File monitor is not running" - return - - f = open(pid_path) - pid = f.read().strip() - f.close() - if not pid.isdigit(): - log.warning("stop_daemon: Found bogus PID (%s) in file: %s" % (pid, pid_path)) - return - - os.kill(int(pid), signal.SIGKILL) - os.remove(pid_path) +def stop_daemon(appname): + get_daemon(appname).stop() diff --git a/edbob/filemon/win32.py b/edbob/filemon/win32.py index eca8485..504406c 100644 --- a/edbob/filemon/win32.py +++ b/edbob/filemon/win32.py @@ -33,8 +33,8 @@ import logging import threading import edbob +from edbob import filemon from edbob.errors import email_exception -from edbob.filemon import get_monitor_profiles from edbob.win32 import Service, file_is_free if sys.platform == 'win32': # docs should build for everyone @@ -69,7 +69,7 @@ class FileMonitorService(Service): return False # Read monitor profile(s) from config. - self.monitored = get_monitor_profiles(self.appname) + self.monitored = filemon.get_monitor_profiles(self.appname) # Make sure we have something to do. if not self.monitored: @@ -79,34 +79,36 @@ class FileMonitorService(Service): for key, profile in self.monitored.iteritems(): # Create a file queue for the profile. - queue = Queue.Queue() + profile.queue = Queue.Queue() - # Create a monitor thread for each folder in profile. + # Perform setup for each of the watched folders. for i, path in enumerate(profile.dirs, 1): + + # Maybe put all pre-existing files in the queue. + if profile.process_existing: + filemon.queue_existing(profile, path) + + # Create a monitor thread for the folder. name = 'monitor-%s-%u' % (key, i) log.debug("Initialize: Starting '%s' thread for folder: %s" % (name, path)) - thread = threading.Thread( - target=monitor_files, - name=name, - args=(queue, path, profile)) + thread = threading.Thread(target=monitor_files, + name=name, args=(profile, path)) thread.daemon = True thread.start() # Create an action thread for the profile. name = 'actions-%s' % key log.debug("Initialize: Starting '%s' thread" % name) - thread = threading.Thread( - target=perform_actions, - name=name, - args=(queue, profile)) + thread = threading.Thread(target=filemon.perform_actions, + name=name, args=(profile,)) thread.daemon = True thread.start() return True -def monitor_files(queue, path, profile): +def monitor_files(profile, path): """ Callable target for file monitor threads. """ @@ -138,40 +140,7 @@ def monitor_files(queue, path, profile): winnt.FILE_ACTION_RENAMED_NEW_NAME): log.debug("monitor_files: Queueing '%s' file: %s" % (profile.key, fpath)) - queue.put(fpath) - - -def perform_actions(queue, profile): - """ - Callable target for action threads. - """ - - while True: - - try: - path = queue.get_nowait() - except Queue.Empty: - pass - else: - - while not file_is_free(path): - win32api.Sleep(0) - - for spec, func, args in profile.actions: - - log.info("perform_actions: Calling function '%s' on file: %s" % - (spec, path)) - - try: - func(path, *args) - - except: - log.exception("perform_actions: An exception occurred " - "while processing file: %s" % path) - email_exception() - - # This file probably shouldn't be processed any further. - break + profile.queue.put(fpath) if __name__ == '__main__': From ea7473f18deed1d546a788aeb0356d3a085b4ae2 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 17 Feb 2013 08:59:47 -0800 Subject: [PATCH 02/18] update changelog --- CHANGES.txt | 8 ++++++++ edbob/_version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index db1d00d..1f7a572 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,12 @@ +0.1a27 +------ + +- [feature] Overhauled file monitors. Added the "process existing" (defaults + to true) and "stop on error" (defaults to false) features to both Linux and + Win32 file monitors. Both features may be overridden in config. The Linux + file monitor was rewritten as an ``edbob.daemon.Daemon`` class. + 0.1a26 ------ diff --git a/edbob/_version.py b/edbob/_version.py index 581b4f3..d1df1fb 100644 --- a/edbob/_version.py +++ b/edbob/_version.py @@ -1 +1 @@ -__version__ = '0.1a26' +__version__ = '0.1a27' From 6c517a91269370c9b763fc86608c6d936639ec28 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 22 Apr 2013 08:15:46 -0700 Subject: [PATCH 03/18] Secured daemon PID files. They are no longer readable or writeable (whoops) by anyone other than the user who owns the process. --- edbob/daemon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edbob/daemon.py b/edbob/daemon.py index 11fbfaa..05c90cb 100644 --- a/edbob/daemon.py +++ b/edbob/daemon.py @@ -3,10 +3,11 @@ from __future__ import absolute_import -# This code was stolen from: +# This code was (mostly, with some tweaks) stolen from: # http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ import sys, os, time, atexit +import stat from signal import SIGTERM class Daemon: @@ -65,6 +66,7 @@ class Daemon: atexit.register(self.delpid) pid = str(os.getpid()) file(self.pidfile,'w+').write("%s\n" % pid) + os.chmod(self.pidfile, stat.S_IRUSR|stat.S_IWUSR) def delpid(self): os.remove(self.pidfile) From 9262b598accee6d5b36d625adcfc0f98f8f74870 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Apr 2013 16:12:20 -0700 Subject: [PATCH 04/18] Added `--progress` argument to command system. --- edbob/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edbob/commands.py b/edbob/commands.py index 3505473..c6e5b0f 100644 --- a/edbob/commands.py +++ b/edbob/commands.py @@ -105,6 +105,7 @@ Options: Config path (may be specified more than once) -n, --no-init Don't load config before executing command -d, --debug Increase logging level to DEBUG + -P, --progress Show progress indicators (where relevant) -v, --verbose Increase logging level to INFO -V, --version Display program version and exit @@ -132,6 +133,7 @@ Try '%(name)s help ' for more help.""" % self metavar='PATH') parser.add_argument('-d', '--debug', action='store_true', dest='debug') parser.add_argument('-n', '--no-init', action='store_true', default=False) + parser.add_argument('-P', '--progress', action='store_true', default=False) parser.add_argument('-v', '--verbose', action='store_true', dest='verbose') parser.add_argument('-V', '--version', action='version', version="%%(prog)s %s" % self.version) @@ -182,6 +184,7 @@ Try '%(name)s help ' for more help.""" % self # And finally, do something of real value... cmd = self.subcommands[cmd](parent=self) + cmd.show_progress = args.progress cmd._run(*(args.command + args.argv)) From 33c951e7feaff3d67c63ce5ee3631f63cc8fc75c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Apr 2013 16:13:47 -0700 Subject: [PATCH 05/18] Added initial Fabric script. --- fabfile.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 fabfile.py diff --git a/fabfile.py b/fabfile.py new file mode 100644 index 0000000..8e01404 --- /dev/null +++ b/fabfile.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################ +# +# edbob -- Pythonic Software Framework +# Copyright © 2010-2012 Lance Edgar +# +# This file is part of edbob. +# +# edbob is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# edbob is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with edbob. If not, see . +# +################################################################################ + +import os.path + +from fabric.api import * + + +execfile(os.path.join(os.path.dirname(__file__), 'edbob', '_version.py')) + + +@task +def release(): + """ + Release a new version of 'edbob'. + """ + + local("python setup.py egg_info --tag-build='' sdist --formats=gztar") + + filename = 'edbob-{0}.tar.gz'.format(__version__) + + put(os.path.join('dist', filename), '/srv/pypi/{0}'.format(filename)) + with cd('/srv/pypi'): + run('rm --recursive --force simple') + run('compoze index') From 2e57936ed126d9a4fdb8f5da0db1efa47ba901da Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 30 Apr 2013 16:16:02 -0700 Subject: [PATCH 06/18] update changelog --- CHANGES.txt | 11 +++++++++++ edbob/_version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1f7a572..f703f67 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,15 @@ +0.1a28 +------ + +- [bug] Secured daemon PID files. They are no longer readable or writeable + (whoops) by anyone other than the user who owns the process. + +- [feature] Added ``--progress`` argument to command system. + +- [general] Added initial Fabric script. + + 0.1a27 ------ diff --git a/edbob/_version.py b/edbob/_version.py index d1df1fb..f99350a 100644 --- a/edbob/_version.py +++ b/edbob/_version.py @@ -1 +1 @@ -__version__ = '0.1a27' +__version__ = '0.1a28' From 4dc4f3f1ec89b6d0cd7b07b626be24e38ace9c48 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 9 May 2013 21:47:07 -0700 Subject: [PATCH 07/18] Removed `setup.cfg` file. The `tag_build` setting was not doing us any favors. --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 018c3b4..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[egg_info] -tag_build = .dev From 3bfda431fa75966537f709c7572092ae27e00b79 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 16 May 2013 07:00:40 -0700 Subject: [PATCH 08/18] Changed some logging instances from INFO to DEBUG. I was just getting tired of the noise. --- edbob/db/extensions/__init__.py | 2 +- edbob/time.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edbob/db/extensions/__init__.py b/edbob/db/extensions/__init__.py index cf875da..a235b3e 100644 --- a/edbob/db/extensions/__init__.py +++ b/edbob/db/extensions/__init__.py @@ -366,7 +366,7 @@ def extend_framework(): session.close() for name in sorted(extensions, extension_sorter(extensions)): - log.info("Applying active extension: %s" % name) + log.debug("Applying active extension: %s" % name) ext = extensions[name] # merge_extension_metadata(ext) # ext.extend_classes() diff --git a/edbob/time.py b/edbob/time.py index 46bfeca..c29049a 100644 --- a/edbob/time.py +++ b/edbob/time.py @@ -75,7 +75,7 @@ def init(config): tz = config.get('edbob.time', key) if tz: key = key[5:] - log.info("'%s' timezone set to '%s'" % (key, tz)) + log.debug("'%s' timezone set to '%s'" % (key, tz)) set_timezone(tz, key) if 'local' not in timezones: From 3766dbfff13475155ff20d380f2e4c8225776ef2 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 17 May 2013 12:21:39 -0700 Subject: [PATCH 09/18] Updated ``repr()`` output for model classes. --- edbob/db/extensions/auth/model.py | 9 +++++---- edbob/db/extensions/contact/model.py | 8 +++++--- edbob/db/model.py | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/edbob/db/extensions/auth/model.py b/edbob/db/extensions/auth/model.py index a3f36f0..f167739 100644 --- a/edbob/db/extensions/auth/model.py +++ b/edbob/db/extensions/auth/model.py @@ -51,7 +51,8 @@ class Permission(Base): permission = Column(String(50), primary_key=True) def __repr__(self): - return "" % (self.role, self.permission) + return "Permission(role_uuid={0}, permission={1})".format( + repr(self.role_uuid), repr(self.permission)) def __unicode__(self): return unicode(self.permission or '') @@ -69,7 +70,7 @@ class UserRole(Base): role_uuid = Column(String(32), ForeignKey('roles.uuid')) def __repr__(self): - return "" % (self.user, self.role) + return "UserRole(uuid={0})".format(repr(self.uuid)) class Role(Base): @@ -97,7 +98,7 @@ class Role(Base): getset_factory=getset_factory) def __repr__(self): - return "" % self.name + return "Role(uuid={0})".format(repr(self.uuid)) def __unicode__(self): return unicode(self.name or '') @@ -124,7 +125,7 @@ class User(Base): getset_factory=getset_factory) def __repr__(self): - return "" % self.username + return "User(uuid={0})".format(repr(self.uuid)) def __unicode__(self): return unicode(self.username or '') diff --git a/edbob/db/extensions/contact/model.py b/edbob/db/extensions/contact/model.py index 8cd05de..6576b5c 100644 --- a/edbob/db/extensions/contact/model.py +++ b/edbob/db/extensions/contact/model.py @@ -72,7 +72,8 @@ class PhoneNumber(Base): __mapper_args__ = {'polymorphic_on': parent_type} def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.number) + return "{0}(uuid={1})".format( + self.__class__.__name__, repr(self.uuid)) def __unicode__(self): return unicode(self.number) @@ -103,7 +104,8 @@ class EmailAddress(Base): __mapper_args__ = {'polymorphic_on': parent_type} def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.address) + return "{0}(uuid={1})".format( + self.__class__.__name__, repr(self.uuid)) def __unicode__(self): return unicode(self.address) @@ -131,7 +133,7 @@ class Person(Base): display_name = Column(String(100), default=get_person_display_name) def __repr__(self): - return "" % self.display_name + return "Person(uuid={0})".format(repr(self.uuid)) def __unicode__(self): return unicode(self.display_name or '') diff --git a/edbob/db/model.py b/edbob/db/model.py index a077ba0..b5222c3 100644 --- a/edbob/db/model.py +++ b/edbob/db/model.py @@ -54,7 +54,7 @@ class ActiveExtension(Base): name = Column(String(50), primary_key=True) def __repr__(self): - return "" % self.name + return "ActiveExtension(name={0})".format(repr(self.name)) def __str__(self): return str(self.name or '') @@ -71,4 +71,4 @@ class Setting(Base): value = Column(Text) def __repr__(self): - return "" % self.name + return "Setting(name={0})".format(repr(self.name)) From 4ed6b09a245134050aa88dbe2921d069ee270ada Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 21 May 2013 22:30:03 -0700 Subject: [PATCH 10/18] update changelog --- CHANGES.txt | 10 ++++++++++ edbob/_version.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index f703f67..0e4fdc0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,14 @@ +0.1a29 +------ + +* Removed ``setup.cfg`` file. + +* Changed some logging instances from ``INFO`` to ``DEBUG``. + +* Updated ``repr()`` output for model classes. + + 0.1a28 ------ diff --git a/edbob/_version.py b/edbob/_version.py index f99350a..36ef6a9 100644 --- a/edbob/_version.py +++ b/edbob/_version.py @@ -1 +1 @@ -__version__ = '0.1a28' +__version__ = '0.1a29' From 25611f1e6895338d682dfad54e7b65163e519438 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 22 May 2013 18:09:51 -0700 Subject: [PATCH 11/18] Tweaked Fabric ``release`` command. --- fabfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 8e01404..4e02e11 100644 --- a/fabfile.py +++ b/fabfile.py @@ -23,6 +23,7 @@ ################################################################################ import os.path +import shutil from fabric.api import * @@ -36,7 +37,8 @@ def release(): Release a new version of 'edbob'. """ - local("python setup.py egg_info --tag-build='' sdist --formats=gztar") + shutil.rmtree('edbob.egg-info') + local('python setup.py sdist --formats=gztar register upload') filename = 'edbob-{0}.tar.gz'.format(__version__) From 4fe5ad9bf03b69b31239780f47cf64bc0ba82a80 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 18 Jun 2013 22:19:52 -0700 Subject: [PATCH 12/18] Prepare for tests. This doesn't add any actual tests but it should pave the way for that. Tests may be run like so: .. code-block:: sh python setup.py nosetests --with-coverage --- edbob/filemon/linux.py | 10 +++++++++- edbob/pyramid/views/people.py | 27 ++++++++++++++------------- edbob/pyramid/views/roles.py | 6 +++--- edbob/pyramid/views/users.py | 27 ++++++++++++++------------- edbob/tests/__init__.py | 19 +++++++++++++++++++ setup.cfg | 7 +++++++ setup.py | 2 ++ 7 files changed, 68 insertions(+), 30 deletions(-) create mode 100644 edbob/tests/__init__.py create mode 100644 setup.cfg diff --git a/edbob/filemon/linux.py b/edbob/filemon/linux.py index 6d5ab83..ae1c4a0 100644 --- a/edbob/filemon/linux.py +++ b/edbob/filemon/linux.py @@ -28,11 +28,19 @@ import sys import os.path -import pyinotify import threading import Queue import logging +try: + import pyinotify +except ImportError: + # Mock out for testing on Windows. + class Dummy(object): + pass + pyinotify = Dummy() + pyinotify.ProcessEvent = Dummy + import edbob from edbob import filemon from edbob.daemon import Daemon diff --git a/edbob/pyramid/views/people.py b/edbob/pyramid/views/people.py index afdf657..1f50470 100644 --- a/edbob/pyramid/views/people.py +++ b/edbob/pyramid/views/people.py @@ -33,37 +33,38 @@ from sqlalchemy import and_ # from formalchemy import Field -import edbob # from edbob.pyramid import filters # from edbob.pyramid import forms # from edbob.pyramid import grids # from edbob.pyramid import Session from edbob.pyramid.views import SearchableAlchemyGridView, CrudView +from edbob.db.extensions.contact.model import ( + Person, PersonEmailAddress, PersonPhoneNumber) class PeopleGrid(SearchableAlchemyGridView): - mapped_class = edbob.Person + mapped_class = Person config_prefix = 'people' sort = 'first_name' def join_map(self): return { 'email': - lambda q: q.outerjoin(edbob.PersonEmailAddress, and_( - edbob.PersonEmailAddress.parent_uuid == edbob.Person.uuid, - edbob.PersonEmailAddress.preference == 1)), + lambda q: q.outerjoin(PersonEmailAddress, and_( + PersonEmailAddress.parent_uuid == Person.uuid, + PersonEmailAddress.preference == 1)), 'phone': - lambda q: q.outerjoin(edbob.PersonPhoneNumber, and_( - edbob.PersonPhoneNumber.parent_uuid == edbob.Person.uuid, - edbob.PersonPhoneNumber.preference == 1)), + lambda q: q.outerjoin(PersonPhoneNumber, and_( + PersonPhoneNumber.parent_uuid == Person.uuid, + PersonPhoneNumber.preference == 1)), } def filter_map(self): return self.make_filter_map( ilike=['first_name', 'last_name'], - email=self.filter_ilike(edbob.PersonEmailAddress.address), - phone=self.filter_ilike(edbob.PersonPhoneNumber.number)) + email=self.filter_ilike(PersonEmailAddress.address), + phone=self.filter_ilike(PersonPhoneNumber.number)) def filter_config(self): return self.make_filter_config( @@ -77,8 +78,8 @@ class PeopleGrid(SearchableAlchemyGridView): def sort_map(self): return self.make_sort_map( 'first_name', 'last_name', - email=self.sorter(edbob.PersonEmailAddress.address), - phone=self.sorter(edbob.PersonPhoneNumber.number)) + email=self.sorter(PersonEmailAddress.address), + phone=self.sorter(PersonPhoneNumber.number)) def grid(self): g = self.make_grid() @@ -97,7 +98,7 @@ class PeopleGrid(SearchableAlchemyGridView): class PersonCrud(CrudView): - mapped_class = edbob.Person + mapped_class = Person home_route = 'people' def fieldset(self, model): diff --git a/edbob/pyramid/views/roles.py b/edbob/pyramid/views/roles.py index 2d0aa57..531c95a 100644 --- a/edbob/pyramid/views/roles.py +++ b/edbob/pyramid/views/roles.py @@ -32,10 +32,10 @@ import formalchemy from webhelpers.html import tags from webhelpers.html.builder import HTML -import edbob from edbob.db import auth from edbob.pyramid import Session from edbob.pyramid.views import SearchableAlchemyGridView, CrudView +from edbob.db.extensions.auth.model import Role default_permissions = [ @@ -68,7 +68,7 @@ default_permissions = [ class RolesGrid(SearchableAlchemyGridView): - mapped_class = edbob.Role + mapped_class = Role config_prefix = 'roles' sort = 'name' @@ -161,7 +161,7 @@ def PermissionsFieldRenderer(permissions, *args, **kwargs): class RoleCrud(CrudView): - mapped_class = edbob.Role + mapped_class = Role home_route = 'roles' permissions = default_permissions diff --git a/edbob/pyramid/views/users.py b/edbob/pyramid/views/users.py index a2e201e..42ba461 100644 --- a/edbob/pyramid/views/users.py +++ b/edbob/pyramid/views/users.py @@ -32,28 +32,29 @@ from webhelpers.html.builder import HTML import formalchemy from formalchemy.fields import SelectFieldRenderer -import edbob from edbob.db import auth from edbob.pyramid import Session from edbob.pyramid.views import SearchableAlchemyGridView, CrudView +from edbob.db.extensions.auth.model import User, Role +from edbob.db.extensions.contact.model import Person class UsersGrid(SearchableAlchemyGridView): - mapped_class = edbob.User + mapped_class = User config_prefix = 'users' sort = 'username' def join_map(self): return { 'person': - lambda q: q.outerjoin(edbob.Person), + lambda q: q.outerjoin(Person), } def filter_map(self): return self.make_filter_map( ilike=['username'], - person=self.filter_ilike(edbob.Person.display_name)) + person=self.filter_ilike(Person.display_name)) def filter_config(self): return self.make_filter_config( @@ -65,7 +66,7 @@ class UsersGrid(SearchableAlchemyGridView): def sort_map(self): return self.make_sort_map( 'username', - person=self.sorter(edbob.Person.display_name)) + person=self.sorter(Person.display_name)) def grid(self): g = self.make_grid() @@ -92,7 +93,7 @@ def RolesFieldRenderer(request): class RolesFieldRenderer(SelectFieldRenderer): def render_readonly(self, **kwargs): - roles = Session.query(edbob.Role) + roles = Session.query(Role) html = '' for uuid in self.value: role = roles.get(uuid) @@ -117,15 +118,15 @@ class RolesField(formalchemy.Field): return [x.uuid for x in user.roles] def get_options(self): - q = Session.query(edbob.Role.name, edbob.Role.uuid) - q = q.filter(edbob.Role.uuid != auth.guest_role(Session()).uuid) - q = q.order_by(edbob.Role.name) + q = Session.query(Role.name, Role.uuid) + q = q.filter(Role.uuid != auth.guest_role(Session()).uuid) + q = q.order_by(Role.name) return q.all() def sync(self): if not self.is_readonly(): user = self.model - roles = Session.query(edbob.Role) + roles = Session.query(Role) data = self.renderer.deserialize() user.roles = [roles.get(x) for x in data] @@ -140,7 +141,7 @@ class _ProtectedPersonRenderer(formalchemy.FieldRenderer): def ProtectedPersonRenderer(uuid): - person = Session.query(edbob.Person).get(uuid) + person = Session.query(Person).get(uuid) assert person return type('ProtectedPersonRenderer', (_ProtectedPersonRenderer,), {'person': person}) @@ -187,7 +188,7 @@ class PasswordField(formalchemy.Field): class UserCrud(CrudView): - mapped_class = edbob.User + mapped_class = User home_route = 'users' def fieldset(self, user): @@ -213,7 +214,7 @@ class UserCrud(CrudView): del fs.confirm_password # if fs.edit and user.person: - if isinstance(user, edbob.User) and user.person: + if isinstance(user, User) and user.person: fs.person.set(readonly=True, renderer=LinkedPersonRenderer(self.request)) diff --git a/edbob/tests/__init__.py b/edbob/tests/__init__.py new file mode 100644 index 0000000..e5c6980 --- /dev/null +++ b/edbob/tests/__init__.py @@ -0,0 +1,19 @@ + +import unittest + +from pyramid import testing + + +class TestCase(unittest.TestCase): + """ + Base class for all test suites. + """ + + def setUp(self): + self.config = testing.setUp() + + def tearDown(self): + testing.tearDown() + + def test_something(self): + self.assertTrue(1) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..28110c5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,7 @@ +[nosetests] +nocapture = 1 +cover-package = edbob +cover-erase = 1 +cover-inclusive = 1 +cover-html = 1 +cover-html-dir = htmlcov diff --git a/setup.py b/setup.py index f81a1fb..0b5b2e5 100644 --- a/setup.py +++ b/setup.py @@ -209,6 +209,8 @@ setup( install_requires = requires, extras_require = extras, + tests_require = requires + ['nose'], + test_suite = 'nose.collector', packages = find_packages(), include_package_data = True, From ee9c6622fff2a9daf750dd5547864025bf0c7ce6 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 18 Jun 2013 22:46:58 -0700 Subject: [PATCH 13/18] Add mock import for testing on Linux. --- edbob/win32.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/edbob/win32.py b/edbob/win32.py index 96a76bd..314ddaa 100644 --- a/edbob/win32.py +++ b/edbob/win32.py @@ -38,9 +38,17 @@ if sys.platform == 'win32': # docs should build for everyone import win32file import win32print import win32service - import win32serviceutil import winerror +try: + import win32serviceutil +except ImportError: + # Mock out for testing on Linux. + class Object(object): + pass + win32serviceutil = Object() + win32serviceutil.ServiceFramework = Object + import edbob From 8a0f300fc0e5a9c1fde9cd7e5b9e5128588aa93f Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 31 Jul 2013 22:36:54 -0700 Subject: [PATCH 14/18] Add minimum version requirement for `pytz`. --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0b5b2e5..6c9bf9a 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,10 @@ requires = [ 'decorator', # 3.3.2 'lockfile', # 0.9.1 'progressbar', # 2.3 - 'pytz', # 2012b + + # Hardcode ``pytz`` minimum since apparently it isn't (any longer?) enough + # to simply require the library. + 'pytz>=2013b', # 2013b ] if sys.version_info < (2, 7): From 183530afc9c2fe2a5e577861d6d81615c53e08c1 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 28 Jan 2014 09:13:45 -0800 Subject: [PATCH 15/18] update changelog --- CHANGES.txt | 9 +++++++++ edbob/_version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0e4fdc0..688705d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,13 @@ +0.1.1 +----- + +* Some random things required in production at MaMa Jean's... + + Specifically this is known to replace occurrences of e.g. ``edbob.User`` with + a more standard (properly imported) reference to ``User``. + + 0.1a29 ------ diff --git a/edbob/_version.py b/edbob/_version.py index 36ef6a9..df9144c 100644 --- a/edbob/_version.py +++ b/edbob/_version.py @@ -1 +1 @@ -__version__ = '0.1a29' +__version__ = '0.1.1' From 73978ffeb7c2721b25add9c4f8000a1a3327b180 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 28 Apr 2014 14:26:21 -0700 Subject: [PATCH 16/18] Allow config file to prevent logging configuration from happening. --- edbob/initialization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edbob/initialization.py b/edbob/initialization.py index 410b572..af2aeff 100644 --- a/edbob/initialization.py +++ b/edbob/initialization.py @@ -89,7 +89,8 @@ def init(appname='edbob', *args, **kwargs): shell = kwargs.get('shell', False) for paths in config_paths: config.read(paths, recurse=not shell) - config.configure_logging() + if config.getboolean('edbob', 'configure_logging', default=True): + config.configure_logging() default_modules = 'edbob.time' modules = config.get('edbob', 'init', default=default_modules) From 8fabdf8b72af2ced5893643e30371e9517ef6a76 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 28 Apr 2014 14:27:32 -0700 Subject: [PATCH 17/18] update changelog --- CHANGES.txt | 6 ++++++ edbob/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 688705d..8bfbfbc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,10 @@ +0.1.2 +----- + +* Allow config file to prevent logging configuration from happening. + + 0.1.1 ----- diff --git a/edbob/_version.py b/edbob/_version.py index df9144c..10939f0 100644 --- a/edbob/_version.py +++ b/edbob/_version.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = '0.1.2' From a0b35dfadba1fe61c48edc0b1bcaa460a470ce07 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 28 Apr 2014 14:28:54 -0700 Subject: [PATCH 18/18] Remove custom PyPI stuff from Fabric `release` task. --- fabfile.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fabfile.py b/fabfile.py index 4e02e11..1eb4728 100644 --- a/fabfile.py +++ b/fabfile.py @@ -22,27 +22,15 @@ # ################################################################################ -import os.path import shutil from fabric.api import * -execfile(os.path.join(os.path.dirname(__file__), 'edbob', '_version.py')) - - @task def release(): """ Release a new version of 'edbob'. """ - shutil.rmtree('edbob.egg-info') local('python setup.py sdist --formats=gztar register upload') - - filename = 'edbob-{0}.tar.gz'.format(__version__) - - put(os.path.join('dist', filename), '/srv/pypi/{0}'.format(filename)) - with cd('/srv/pypi'): - run('rm --recursive --force simple') - run('compoze index')