diff --git a/rattail/commands/__init__.py b/rattail/commands/__init__.py index d0269a1d..cb591ba9 100644 --- a/rattail/commands/__init__.py +++ b/rattail/commands/__init__.py @@ -1,8 +1,8 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8; -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar +# Copyright © 2010-2017 Lance Edgar # # This file is part of Rattail. # @@ -26,5 +26,5 @@ Console Commands from __future__ import unicode_literals, absolute_import -from .core import main, Command, Subcommand, OldImportSubcommand, NewImportSubcommand, Dump, date_argument +from .core import main, Command, Subcommand, Dump, date_argument from .importing import ImportSubcommand diff --git a/rattail/commands/core.py b/rattail/commands/core.py index 0a59774b..28965af7 100644 --- a/rattail/commands/core.py +++ b/rattail/commands/core.py @@ -749,251 +749,6 @@ class FileMonitorCommand(Subcommand): service.delayed_auto_start_service(name) -class OldImportSubcommand(Subcommand): - """ - Base class for subcommands which use the data importing system. - """ - supports_versioning = True - - def add_parser_args(self, parser): - handler = self.get_handler(quiet=True) - if self.supports_versioning: - parser.add_argument('--no-versioning', action='store_true', - help="Disables versioning during the import. This is " - "intended to be useful e.g. during initial import, where " - "the process can be quite slow even without the overhead " - "of versioning.") - parser.add_argument('--warnings', '-W', action='store_true', - help="Whether to log warnings if any data model " - "writes occur. Intended to help stay in sync " - "with an external data source.") - parser.add_argument('--max-updates', type=int, - help="Maximum number of record updates (or additions) which, if " - "reached, should cause the importer to stop early. Note that the " - "updates which have completed will be committed unless a dry run " - "is in effect.") - parser.add_argument('--dry-run', action='store_true', - help="Go through the motions and allow logging to occur, " - "but do not actually commit the transaction at the end.") - parser.add_argument('models', nargs='*', metavar='MODEL', - help="Which models to import. If none are specified, all models will " - "be imported. Or, specify only those you wish to import. Supported " - "models are: {0}".format(', '.join(handler.get_importer_keys()))) - - def run(self, args): - log.info("begin {0} for data model(s): {1}".format( - self.name, ', '.join(args.models or ["ALL"]))) - - Session = self.parent.db_session_factory - if self.supports_versioning: - if args.no_versioning: - from rattail.db.continuum import disable_versioning - disable_versioning() - session = Session(continuum_user=self.continuum_user) - else: - session = Session() - - self.import_data(args, session) - - if args.dry_run: - session.rollback() - log.info("dry run, so transaction was rolled back") - else: - session.commit() - log.info("transaction was committed") - session.close() - - def get_handler_factory(self, quiet=False): - """ - This method must return a factory, which will in turn generate a - handler instance to be used by the command. Note that you *must* - override this method. - """ - raise NotImplementedError - - def get_handler(self, **kwargs): - """ - Returns a handler instance to be used by the command. - """ - factory = self.get_handler_factory(quiet=kwargs.pop('quiet', False)) - return factory(getattr(self, 'config', None), **kwargs) - - @property - def continuum_user(self): - """ - Info needed to assign the Continuum user for the database session. - """ - - def import_data(self, args, session): - """ - Perform a data import, with the given arguments and database session. - """ - handler = self.get_handler(session=session) - models = args.models or handler.get_importer_keys() - updates = handler.import_data(models, max_updates=args.max_updates, - progress=self.progress) - if args.warnings and updates: - handler.process_warnings(updates, command=self, models=models, dry_run=args.dry_run, - render_record=self.get_record_renderer(), - progress=self.progress) - - def get_record_renderer(self): - """ - Get the record renderer for email notifications. Note that config may - override the default. - """ - spec = self.config.get('{0}.{1}'.format(self.parent.name, self.name), 'record_renderer', - default='rattail.db.importing:RecordRenderer') - return load_object(spec)(self.config) - - -class NewImportSubcommand(Subcommand): - """ - Base class for subcommands which use the (new) data importing system. - """ - - def get_handler_factory(self, args=None): - """ - This method must return a factory, which will in turn generate a - handler instance to be used by the command. Note that you *must* - override this method. - """ - raise NotImplementedError - - def get_handler(self, args=None, **kwargs): - """ - Returns a handler instance to be used by the command. - """ - factory = self.get_handler_factory(args) - kwargs = self.get_handler_kwargs(args, **kwargs) - kwargs['command'] = self - return factory(getattr(self, 'config', None), **kwargs) - - def get_handler_kwargs(self, args, **kwargs): - """ - Return a dict of kwargs to be passed to the handler factory. - """ - return kwargs - - def add_parser_args(self, parser): - handler = self.get_handler() - - # model names (aka importer keys) - parser.add_argument('models', nargs='*', metavar='MODEL', - help="Which data models to import. If you specify any, then only data " - "for those models will be imported. If you do not specify any, then all " - "*default* models will be imported. Supported models are: ({})".format( - ', '.join(handler.get_importer_keys()))) - - # start/end date - parser.add_argument('--start-date', type=date_argument, - help="Optional (inclusive) starting point for date range, by which host " - "data should be filtered. Only used by certain importers.") - parser.add_argument('--end-date', type=date_argument, - help="Optional (inclusive) ending point for date range, by which host " - "data should be filtered. Only used by certain importers.") - - # allow create? - parser.add_argument('--create', action='store_true', default=True, - help="Allow new records to be created during the import.") - parser.add_argument('--no-create', action='store_false', dest='create', - help="Do not allow new records to be created during the import.") - parser.add_argument('--max-create', type=int, metavar='COUNT', - help="Maximum number of records which may be created, after which a " - "given import task should stop. Note that this applies on a per-model " - "basis and not overall.") - - # allow update? - parser.add_argument('--update', action='store_true', default=True, - help="Allow existing records to be updated during the import.") - parser.add_argument('--no-update', action='store_false', dest='update', - help="Do not allow existing records to be updated during the import.") - parser.add_argument('--max-update', type=int, metavar='COUNT', - help="Maximum number of records which may be updated, after which a " - "given import task should stop. Note that this applies on a per-model " - "basis and not overall.") - - # allow delete? - parser.add_argument('--delete', action='store_true', default=False, - help="Allow records to be deleted during the import.") - parser.add_argument('--no-delete', action='store_false', dest='delete', - help="Do not allow records to be deleted during the import.") - parser.add_argument('--max-delete', type=int, metavar='COUNT', - help="Maximum number of records which may be deleted, after which a " - "given import task should stop. Note that this applies on a per-model " - "basis and not overall.") - - # max total changes, per model - parser.add_argument('--max-total', type=int, metavar='COUNT', - help="Maximum number of *any* record changes which may occur, after which " - "a given import task should stop. Note that this applies on a per-model " - "basis and not overall.") - - # treat changes as warnings? - parser.add_argument('--warnings', '-W', action='store_true', - help="Set this flag if you expect a \"clean\" import, and wish for any " - "changes which do occur to be processed further and/or specially. The " - "behavior of this flag is ultimately up to the import handler, but the " - "default is to send an email notification.") - - # dry run? - parser.add_argument('--dry-run', action='store_true', - help="Go through the full motions and allow logging etc. to " - "occur, but rollback (abort) the transaction at the end.") - - def run(self, args): - log.info("begin `{} {}` for data models: {}".format( - self.parent.name, self.name, ', '.join(args.models or ["(ALL)"]))) - - Session = self.parent.db_session_factory - session = Session() - - self.import_data(args, session) - - if args.dry_run: - session.rollback() - log.info("dry run, so transaction was rolled back") - else: - session.commit() - log.info("transaction was committed") - session.close() - - def import_data(self, args, session): - """ - Perform a data import, with the given arguments and database session. - """ - handler = self.get_handler(args=args, session=session, progress=self.progress) - models = args.models or handler.get_default_keys() - log.debug("using handler: {}".format(handler)) - log.debug("importing models: {}".format(models)) - log.debug("args are: {}".format(args)) - handler.import_data(models, args) - - -class ImportCSV(OldImportSubcommand): - """ - Import data from a CSV file - """ - name = 'import-csv' - description = __doc__.strip() - - def add_parser_args(self, parser): - super(ImportCSV, self).add_parser_args(parser) - parser.add_argument('importer', - help="Spec string for importer class which should handle the import.") - parser.add_argument('csv_path', - help="Path to the data file which will be imported.") - - def import_data(self, args, session): - from rattail.db.importing.providers.csv import make_provider - - provider = make_provider(self.config, session, args.importer, data_path=args.csv_path) - data = provider.get_data(progress=self.progress) - affected = provider.importer.import_data(data, provider.supported_fields, 'uuid', - progress=self.progress) - log.info("added or updated {0} {1} records".format(affected, provider.model_name)) - - class LoadHostDataCommand(Subcommand): """ Loads data from the Rattail host database, if one is configured. diff --git a/rattail/datasync/__init__.py b/rattail/datasync/__init__.py index 49d1758d..35f88f86 100644 --- a/rattail/datasync/__init__.py +++ b/rattail/datasync/__init__.py @@ -1,8 +1,8 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8; -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar +# Copyright © 2010-2017 Lance Edgar # # This file is part of Rattail. # @@ -27,4 +27,4 @@ DataSync Daemon from __future__ import unicode_literals, absolute_import from .watchers import DataSyncWatcher -from .consumers import DataSyncConsumer, DataSyncImportConsumer, NewDataSyncImportConsumer +from .consumers import DataSyncConsumer, NewDataSyncImportConsumer diff --git a/rattail/datasync/consumers.py b/rattail/datasync/consumers.py index 50fd04b9..9472ace6 100644 --- a/rattail/datasync/consumers.py +++ b/rattail/datasync/consumers.py @@ -28,152 +28,9 @@ from __future__ import unicode_literals, absolute_import from rattail import importing from rattail.config import parse_list -from rattail.db.newimporting import ImportHandler from rattail.util import load_object -class DataSyncConsumer(object): - """ - Base class for all DataSync consumers. - """ - - def __init__(self, config, key, dbkey=None): - self.config = config - self.key = key - self.dbkey = dbkey - - def setup(self): - """ - This method is called when the consumer thread is first started. - """ - - def begin_transaction(self): - """ - Called just before the consumer is asked to process changes, possibly - via multiple batches. - """ - - def process_changes(self, session, changes): - """ - Process (consume) a batch of changes. - """ - - def rollback_transaction(self): - """ - Called when any batch of changes failed to process. - """ - - def commit_transaction(self): - """ - Called just after the consumer has successfully finished processing - changes, possibly via multiple batches. - """ - - -class DataSyncImportConsumer(DataSyncConsumer): - """ - Base class for DataSync consumer which is able to leverage a (set of) - importer(s) to do the heavy lifting. - - .. note:: - This assumes "old-style" importers based on - ``rattail.db.newimporting.Importer``. - """ - - def __init__(self, *args, **kwargs): - super(DataSyncImportConsumer, self).__init__(*args, **kwargs) - self.importers = self.get_importers() - - def get_importers(self): - """ - You must override this to return a dict of importer *instances*, keyed - by what you expect the corresponding ``DataSyncChange.payload_type`` to - be, coming from the "host" system, whatever that is. - """ - raise NotImplementedError - - def get_importers_from_handler(self, handler, default_only=True): - if not isinstance(handler, ImportHandler): - handler = handler(config=self.config) - factories = handler.get_importers() - if default_only: - keys = handler.get_default_keys() - else: - keys = factories.keys() - importers = {} - for key in keys: - importers[key] = factories[key](config=self.config) - return importers - - def process_changes(self, session, changes): - """ - Process all changes, leveraging importer(s) as much as possible. - """ - # Update all importers with current Rattail session. - for importer in self.importers.itervalues(): - importer.session = session - - for change in changes: - self.invoke_importer(session, change) - - def invoke_importer(self, session, change): - """ - For the given change, invoke the default importer behavior, if one is - available. - """ - importer = self.importers.get(change.payload_type) - if importer: - if change.deletion: - self.process_deletion(session, importer, change) - else: - return self.process_change(session, importer, change) - - def process_change(self, session, importer, change=None, host_object=None, host_data=None): - """ - Invoke the importer to process the given change / host record. - """ - if host_data is None: - if host_object is None: - host_object = self.get_host_record(session, change) - if host_object is None: - return - host_data = importer.normalize_source_record(host_object) - if host_data is None: - return - key = importer.get_key(host_data) - local_object = importer.get_instance(key) - if local_object: - local_data = importer.normalize_instance(local_object) - if importer.data_diffs(local_data, host_data): - local_object = importer.update_instance(local_object, host_data, local_data) - return local_object - else: - return importer.create_instance(key, host_data) - - def process_deletion(self, session, importer, change): - """ - Attempt to invoke the importer, to delete a local record according to - the change involved. - """ - key = self.get_deletion_key(session, change) - local_object = importer.get_instance(key) - if local_object: - return importer.delete_instance(local_object) - return False - - def get_deletion_key(self, session, change): - return (change.payload_key,) - - def get_host_record(self, session, change): - """ - You must override this, to return a host record from the given - ``DataSyncChange`` instance. Note that the host record need *not* be - normalized, as that will be done by the importer. (This is effectively - the only part of the processing which is not handled by the importer.) - """ - raise NotImplementedError - - class NewDataSyncImportConsumer(DataSyncConsumer): """ Base class for DataSync consumer which is able to leverage a (set of) diff --git a/rattail/db/importing/__init__.py b/rattail/db/importing/__init__.py deleted file mode 100644 index 9b50b7f1..00000000 --- a/rattail/db/importing/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2015 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Data Importing -""" - -from .core import Importer, make_importer, RecordRenderer -from . import models -from .providers import DataProvider, QueryProvider -from .handlers import ImportHandler diff --git a/rattail/db/importing/core.py b/rattail/db/importing/core.py deleted file mode 100644 index 9409711f..00000000 --- a/rattail/db/importing/core.py +++ /dev/null @@ -1,406 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2015 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Core Importer Stuff -""" - -from __future__ import unicode_literals, absolute_import - -import logging - -from sqlalchemy.orm.exc import NoResultFound - -from rattail.db import model -from rattail.core import Object -from rattail.util import load_object -from rattail.db.cache import cache_model - - -log = logging.getLogger(__name__) - - -def make_importer(config, session, spec): - """ - Create an importer instance according to the given spec. For now..see the - source code for more details. - """ - importer = None - if '.' not in spec and ':' not in spec: - from rattail.db.importing import models - if hasattr(models, spec): - importer = getattr(models, spec) - elif hasattr(models, '{0}Importer'.format(spec)): - importer = getattr(models, '{0}Importer'.format(spec)) - else: - importer = load_object(spec) - if importer: - return importer(config, session) - - -class Importer(Object): - """ - Base class for model importers. - """ - supported_fields = [] - normalizer_class = None - cached_data = None - - complex_fields = [] - """ - Sequence of field names which are considered complex and therefore require - custom logic provided by the derived class, etc. - """ - - def __init__(self, config, session, **kwargs): - self.config = config - self.session = session - super(Importer, self).__init__(**kwargs) - - @property - def model_module(self): - """ - Reference to a module which contains all available / necessary data - models. By default this is ``rattail.db.model``. - """ - return model - - @property - def model_class(self): - return getattr(model, self.__class__.__name__[:-8]) - - @property - def model_name(self): - return self.model_class.__name__ - - @property - def simple_fields(self): - return self.supported_fields - - def import_data(self, records, fields, key, count=None, max_updates=None, progress=None): - """ - Import some data. - """ - if count is None: - count = len(records) - if count == 0: - return [], [] - - self.fields = fields - self.key = key - if isinstance(key, basestring): - self.key = (key,) - self.progress = progress - self.normalizer = self.normalizer_class() if self.normalizer_class else None - self.setup() - self.cache_data(progress) - - # Normalize to remove duplicate source records. This is more for the - # sake of sanity since duplicates typically lead to a ping-pong effect - # where an update-less import is impossible. - normalized = {} - for src_data in records: - key = self.get_key(src_data) - if key in normalized: - log.warning("duplicate records from {0}:{1} for key: {2}".format( - self.provider.__class__.__module__, self.provider.__class__.__name__, repr(key))) - normalized[key] = src_data - records = [] - for key in sorted(normalized): - records.append(normalized[key]) - - prog = None - if progress: - prog = progress("Importing {0} data".format(self.model_name), count) - - created = [] - updated = [] - - affected = 0 - keys_seen = set() - for i, src_data in enumerate(records, 1): - key = self.get_key(src_data) - if key in keys_seen: - log.warning("duplicate records from {0}:{1} for key: {2}".format( - self.provider.__class__.__module__, self.provider.__class__.__name__, repr(key))) - else: - keys_seen.add(key) - self.normalize_record(src_data) - - dirty = False - inst_data = self.get_instance_data(src_data) - if inst_data: - if self.data_differs(inst_data, src_data): - instance = self.get_instance(src_data) - self.update_instance(instance, src_data, inst_data) - updated.append(instance) - dirty = True - else: - instance = self.new_instance(src_data) - assert instance, "Failed to create new model instance for data: {0}".format(repr(src_data)) - self.update_instance(instance, src_data) - self.session.add(instance) - self.session.flush() - log.debug("created new {} {}: {}".format(self.model_name, key, instance)) - created.append(instance) - dirty = True - - if dirty: - self.session.flush() - affected += 1 - if max_updates and affected >= max_updates: - log.warning("max of {0} updates has been reached; bailing early".format(max_updates)) - break - - if prog: - prog.update(i) - if prog: - prog.destroy() - - return created, updated - - def setup(self): - """ - Perform any setup necessary, e.g. cache lookups for existing data. - """ - - def cache_query_options(self): - """ - Return a list of options to apply to the cache query, if needed. - """ - - def cache_model(self, model_class, key, **kwargs): - """ - Convenience method for caching a model. - """ - kwargs.setdefault('progress', self.progress) - return cache_model(self.session, model_class, key=key, **kwargs) - - def get_cache_key(self, instance, normalized): - """ - Get the primary model cache key for a given instance/data object. - """ - return tuple(normalized['data'].get(k) for k in self.key) - - def cache_data(self, progress): - """ - Cache all existing model instances as normalized data. - """ - self.cached_data = self.cache_model(self.model_class, self.get_cache_key, - query_options=self.cache_query_options(), - normalizer=self.normalize_cache) - - def normalize_cache(self, instance): - """ - Normalizer for cache data. This adds the instance to the cache in - addition to its normalized data. This is so that if lots of updates - are required, we don't we have to constantly fetch them. - """ - return {'instance': instance, 'data': self.normalize_instance(instance)} - - def data_differs(self, inst_data, src_data): - """ - Compare source record data to instance data to determine if there is a - net change. - """ - for field in self.fields: - if src_data[field] != inst_data[field]: - log.debug("field {0} differed for instance data: {1}, source data: {2}".format( - field, repr(inst_data), repr(src_data))) - return True - return False - - def string_or_null(self, data, *fields): - """ - For each field specified, ensure the data value is a non-empty string, - or ``None``. - """ - for field in fields: - if field in data: - value = data[field] - value = value.strip() if value else None - data[field] = value or None - - def int_or_null(self, data, *fields): - """ - For each field specified, ensure the data value is a non-zero integer, - or ``None``. - """ - for field in fields: - if field in data: - value = data[field] - value = int(value) if value else None - data[field] = value or None - - def prioritize_2(self, data, field): - """ - Prioritize the data values for the pair of fields implied by the given - fieldname. I.e., if only one non-empty value is present, make sure - it's in the first slot. - """ - field2 = '{0}_2'.format(field) - if field in data and field2 in data: - if data[field2] and not data[field]: - data[field], data[field2] = data[field2], None - - def normalize_record(self, data): - """ - Normalize the source data record, if necessary. - """ - - def get_key(self, data): - """ - Return the key value for the given source data record. - """ - return tuple(data.get(k) for k in self.key) - - def get_instance(self, data): - """ - Fetch an instance from our database which corresponds to the source - data, if possible; otherwise return ``None``. - """ - key = self.get_key(data) - if not key: - log.warning("source {0} has no {1}: {2}".format( - self.model_name, self.key, repr(data))) - return None - - if self.cached_data is not None: - data = self.cached_data.get(key) - return data['instance'] if data else None - - q = self.session.query(self.model_class) - for i, k in enumerate(self.key): - q = q.filter(getattr(self.model_class, k) == key[i]) - try: - instance = q.one() - except NoResultFound: - return None - else: - return instance - - def get_instance_data(self, data): - """ - Return a normalized data record for the model instance corresponding to - the source data record, or ``None``. - """ - key = self.get_key(data) - if not key: - log.warning("source {0} has no {1}: {2}".format( - self.model_name, self.key, repr(data))) - return None - if self.cached_data is not None: - data = self.cached_data.get(key) - return data['data'] if data else None - instance = self.get_instance(data) - if instance: - return self.normalize_instance(instance) - - def normalize_instance(self, instance): - """ - Normalize a model instance. - """ - if self.normalizer: - return self.normalizer.normalize(instance) - - data = {} - for field in self.simple_fields: - if field in self.fields: - data[field] = getattr(instance, field) - return data - - def new_instance(self, data): - """ - Return a new model instance to correspond to the source data record. - """ - kwargs = {} - key = self.get_key(data) - for i, k in enumerate(self.key): - if k in self.simple_fields: - kwargs[k] = key[i] - return self.model_class(**kwargs) - - def update_instance(self, instance, data, inst_data=None): - """ - Update the given model instance with the given data. - """ - for field in self.simple_fields: - if field in data: - if not inst_data or inst_data[field] != data[field]: - setattr(instance, field, data[field]) - - -class RecordRenderer(object): - """ - Record renderer for email notifications sent from data import jobs. - """ - - def __init__(self, config): - self.config = config - - def __call__(self, record): - return self.render(record) - - def render(self, record): - """ - Render the given record. Default is to attempt. - """ - key = record.__class__.__name__.lower() - renderer = getattr(self, 'render_{0}'.format(key), None) - if renderer: - return renderer(record) - - label = self.get_label(record) - url = self.get_url(record) - if url: - return '{1}'.format(url, label) - return label - - def get_label(self, record): - key = record.__class__.__name__.lower() - label = getattr(self, 'label_{0}'.format(key), self.label) - return label(record) - - def label(self, record): - return unicode(record) - - def get_url(self, record): - """ - Fetch / generate a URL for the given data record. You should *not* - override this method, but do :meth:`url()` instead. - """ - key = record.__class__.__name__.lower() - url = getattr(self, 'url_{0}'.format(key), self.url) - return url(record) - - def url(self, record): - """ - Fetch / generate a URL for the given data record. - """ - url = self.config.get('tailbone', 'url') - if url: - url = url.rstrip('/') - name = '{0}s'.format(record.__class__.__name__.lower()) - if name == 'persons': # FIXME, obviously this is a hack - name = 'people' - url = '{0}/{1}/{{uuid}}'.format(url, name) - return url.format(uuid=record.uuid) diff --git a/rattail/db/importing/handlers.py b/rattail/db/importing/handlers.py deleted file mode 100644 index 99518dac..00000000 --- a/rattail/db/importing/handlers.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Import Handlers -""" - -from __future__ import unicode_literals, absolute_import - -import logging - -from rattail.util import OrderedDict -from rattail.mail import send_email - - -log = logging.getLogger(__name__) - - -class ImportHandler(object): - """ - Base class for all import handlers. - """ - - def __init__(self, config=None, session=None): - self.config = config - self.session = session - self.importers = self.get_importers() - - def get_importers(self): - """ - Returns a dict of all available importers, where the values are - importer factories. All subclasses will want to override this. Note - that if you return an ``OrderedDict`` instance, you can affect the - ordering of keys in the command line help system, etc. - """ - return {} - - def get_importer_keys(self): - """ - Returns a list of keys corresponding to the available importers. - """ - return list(self.importers.iterkeys()) - - def get_importer(self, key): - """ - Returns an importer instance corresponding to the given key. - """ - return self.importers[key](self.config, self.session, - **self.get_importer_kwargs(key)) - - def get_importer_kwargs(self, key): - """ - Return a dict of kwargs to be used when construcing an importer with - the given key. - """ - return {} - - def import_data(self, keys, max_updates=None, progress=None): - """ - Import all data for the given importer keys. - """ - self.before_import() - updates = OrderedDict() - - for key in keys: - provider = self.get_importer(key) - if not provider: - log.warning("unknown importer; skipping: {0}".format(repr(key))) - continue - - data = provider.get_data(progress=progress) - created, updated = provider.importer.import_data( - data, provider.supported_fields, provider.key, - max_updates=max_updates, progress=progress) - - if hasattr(provider, 'process_deletions'): - deleted = provider.process_deletions(data, progress=progress) - else: - deleted = 0 - - log.info("added {0}, updated {1}, deleted {2} {3} records".format( - len(created), len(updated), deleted, key)) - if created or updated or deleted: - updates[key] = created, updated, deleted - - self.after_import() - return updates - - def before_import(self): - return - - def after_import(self): - return - - def process_warnings(self, updates, command=None, **kwargs): - """ - If an import was run with "warnings" enabled, and work was effectively - done then this method is called to process the updates. The assumption - is that a warning email will be sent with the details, but you can do - anything you like if you override this. - """ - data = kwargs - data['updates'] = updates - - if command: - data['command'] = '{} {}'.format(command.parent.name, command.name) - else: - data['command'] = None - - if command: - key = '{}_{}_updates'.format(command.parent.name, command.name) - key = key.replace('-', '_') - else: - key = 'rattail_import_updates' - - send_email(self.config, key, fallback_key='rattail_import_updates', data=data) diff --git a/rattail/db/importing/models.py b/rattail/db/importing/models.py deleted file mode 100644 index 15678eee..00000000 --- a/rattail/db/importing/models.py +++ /dev/null @@ -1,1674 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Model Importers -""" - -from __future__ import unicode_literals, absolute_import - -import logging - -from sqlalchemy import orm - -from rattail.db import model -from rattail.db.api import set_regular_price, set_current_sale_price -from rattail.db.auth import administrator_role -from rattail.db.importing import Importer, normal - - -log = logging.getLogger(__name__) - - -class PersonImporter(Importer): - """ - Person data importer. - """ - model_class = model.Person - simple_fields = [ - 'uuid', - 'first_name', - 'middle_name', - 'last_name', - 'display_name', - ] - supported_fields = simple_fields - - -class PersonEmailAddressImporter(Importer): - """ - Person email address data importer. - """ - model_class = model.PersonEmailAddress - simple_fields = [ - 'uuid', - 'parent_uuid', - 'type', - 'address', - 'preference', - 'invalid', - ] - supported_fields = simple_fields + [ - 'preferred', - ] - - def normalize_instance(self, email): - data = super(PersonEmailAddressImporter, self).normalize_instance(email) - if 'preferred' in self.fields: - data['preferred'] = email.preference == 1 - return data - - def update_instance(self, email, data, inst_data=None): - super(PersonEmailAddressImporter, self).update_instance(email, data, inst_data) - if 'preferred' in self.fields: - if data['preferred']: - if email.preference != 1: - person = email.person - if not person: - person = self.session.query(model.Person).get(email.parent_uuid) - assert person, "hm: " + email.parent_uuid - if email in person.emails: - person.emails.remove(email) - person.emails.insert(0, email) - person.emails.reorder() - else: - if email.preference == 1: - person = email.person - if not person: - person = self.session.query(model.Person).get(email.parent_uuid) - assert person, "hm: " + email.parent_uuid - if len(person.emails) > 1: - person.emails.remove(email) - person.emails.append(email) - person.emails.reorder() - - # If this is a new record, we may still need to establish its preference. - if email.preference is None: - person = email.person - if not person: - person = self.session.query(model.Person).get(email.parent_uuid) - assert person, "hm: " + email.parent_uuid - if email not in person.emails: - person.emails.append(email) - person.emails.reorder() - - -class PersonPhoneNumberImporter(Importer): - """ - Person phone number data importer. - """ - model_class = model.PersonPhoneNumber - simple_fields = [ - 'uuid', - 'parent_uuid', - 'type', - 'number', - 'preference', - ] - supported_fields = simple_fields + [ - 'preferred', - ] - - def normalize_instance(self, phone): - data = super(PersonPhoneNumberImporter, self).normalize_instance(phone) - if 'preferred' in self.fields: - data['preferred'] = phone.preference == 1 - return data - - def update_instance(self, phone, data, inst_data=None): - super(PersonPhoneNumberImporter, self).update_instance(phone, data, inst_data) - if 'preferred' in self.fields: - if data['preferred']: - if phone.preference != 1: - person = phone.person - if not person: - person = self.session.query(model.Person).get(phone.parent_uuid) - assert person, "hm: " + phone.parent_uuid - if phone in person.phones: - person.phones.remove(phone) - person.phones.insert(0, phone) - person.phones.reorder() - else: - if phone.preference == 1: - person = phone.person - if not person: - person = self.session.query(model.Person).get(phone.parent_uuid) - assert person, "hm: " + phone.parent_uuid - if len(person.phones) > 1: - person.phones.remove(phone) - person.phones.append(phone) - person.phones.reorder() - - # If this is a new record, we may still need to establish its preference. - if phone.preference is None: - person = phone.person - if not person: - person = self.session.query(model.Person).get(phone.parent_uuid) - assert person, "hm: " + phone.parent_uuid - if phone not in person.phones: - person.phones.append(phone) - person.phones.reorder() - - -class PersonMailingAddressImporter(Importer): - """ - Person mailing address data importer. - """ - model_class = model.PersonMailingAddress - supported_fields = [ - 'uuid', - 'parent_uuid', - 'type', - 'preference', - 'street', - 'street2', - 'city', - 'state', - 'zipcode', - 'invalid', - ] - - -class UserImporter(Importer): - """ - User data importer. - """ - model_class = model.User - normalizer_class = normal.UserNormalizer - simple_fields = [ - 'uuid', - 'username', - 'password', - 'salt', - 'person_uuid', - 'active', - ] - supported_fields = simple_fields + [ - 'admin', - ] - - def setup(self): - self.normalizer.admin = self.admin = administrator_role(self.session) - - def update_instance(self, user, data, inst_data=None): - super(UserImporter, self).update_instance(user, data, inst_data) - if 'admin' in data: - if data['admin']: - if self.admin not in user.roles: - user.roles.append(self.admin) - else: - if self.admin in user.roles: - user.roles.remove(self.admin) - - -class MessageImporter(Importer): - """ - User message data importer. - """ - model_class = model.Message - normalizer_class = normal.MessageNormalizer - simple_fields = [ - 'uuid', - 'sender_uuid', - 'subject', - 'body', - 'sent', - ] - - -class MessageRecipientImporter(Importer): - """ - User message recipient data importer. - """ - model_class = model.MessageRecipient - normalizer_class = normal.MessageRecipientNormalizer - simple_fields = [ - 'uuid', - 'message_uuid', - 'recipient_uuid', - 'status', - ] - - -class StoreImporter(Importer): - """ - Store data importer. - """ - model_class = model.Store - simple_fields = [ - 'uuid', - 'id', - 'name', - ] - supported_fields = simple_fields + [ - 'phone_number', - 'fax_number', - ] - - def cache_query_options(self): - if 'phone_number' in self.fields or 'fax_number' in self.fields: - return [orm.joinedload(model.Store.phones)] - - def normalize_record(self, data): - self.string_or_null(data, 'name', 'phone_number', 'fax_number') - - def normalize_instance(self, store): - data = super(StoreImporter, self).normalize_instance(store) - - if 'phone_number' in self.fields: - data['phone_number'] = None - for phone in store.phones: - if phone.type == 'Voice': - data['phone_number'] = phone.number - break - - if 'fax_number' in self.fields: - data['fax_number'] = None - for phone in store.phones: - if phone.type == 'Fax': - data['fax_number'] = phone.number - break - - return data - - def update_instance(self, store, data, inst_data=None): - super(StoreImporter, self).update_instance(store, data, inst_data) - - if 'phone_number' in data: - number = data['phone_number'] or None - if number: - found = False - for phone in store.phones: - if phone.type == 'Voice': - if phone.number != number: - phone.number = number - found = True - break - if not found: - store.add_phone_number(number, type='Voice') - else: - for phone in list(store.phones): - if phone.type == 'Voice': - store.phones.remove(phone) - - if 'fax_number' in data: - number = data['fax_number'] or None - if number: - found = False - for phone in store.phones: - if phone.type == 'Fax': - if phone.number != number: - phone.number = number - found = True - break - if not found: - store.add_phone_number(number, type='Fax') - else: - for phone in list(store.phones): - if phone.type == 'Fax': - store.phones.remove(phone) - - -class EmployeeImporter(Importer): - """ - Employee data importer. - """ - model_class = model.Employee - normalizer_class = normal.EmployeeNormalizer - simple_fields = [ - 'uuid', - 'id', - 'person_uuid', - 'display_name', - 'status', - ] - supported_fields = simple_fields + [ - 'customer_id', - 'first_name', - 'last_name', - 'person_display_name', - 'phone_number', - 'phone_number_2', - 'email_address', - ] - - def setup(self): - if 'customer_id' in self.fields: - self.customers = self.cache_model(model.Customer, 'id') - - def cache_query_options(self): - return [ - orm.joinedload(model.Employee.person).joinedload(model.Person._customers), - orm.joinedload(model.Employee.phones), - orm.joinedload(model.Employee.email), - ] - - def normalize_record(self, data): - self.string_or_null(data, 'customer_id', 'phone_number', - 'phone_number_2', 'email_address') - self.int_or_null(data, 'id', 'status') - self.prioritize_2(data, 'phone_number') - - def update_instance(self, employee, data, inst_data=None): - super(EmployeeImporter, self).update_instance(employee, data, inst_data) - person = employee.person - - if 'first_name' in self.fields: - employee.first_name = data['first_name'] - if 'last_name' in self.fields: - employee.last_name = data['last_name'] - - if 'person_display_name' in self.fields: - if person.display_name != data['person_display_name']: - person.display_name = data['person_display_name'] - - if 'customer_id' in self.fields: - id_ = data['customer_id'] - if id_: - customer = self.customers.get(id_) - if not customer: - customer = model.Customer() - customer.id = id_ - customer.name = employee.display_name - self.session.add(customer) - self.customers[customer.id] = customer - if person not in customer.people: - customer.people.append(person) - else: - for customer in list(person.customers): - if len(customer.people) > 1: - if person in customer.people: - customer.people.remove(person) - - if 'phone_number' in self.fields: - number = data['phone_number'] - if number: - found = False - for phone in employee.phones: - if phone.type == 'Home': - if phone.number != number: - phone.number = number - found = True - break - if not found: - employee.add_phone_number(number, type='Home') - else: - for phone in list(employee.phones): - if phone.type == 'Home': - employee.phones.remove(phone) - - if 'phone_number_2' in self.fields: - number = data['phone_number_2'] - if number: - found = False - first = False - for phone in employee.phones: - if phone.type == 'Home': - if first: - if phone.number != number: - phone.number = number - found = True - break - first = True - if not found: - employee.add_phone_number(number, type='Home') - else: - first = False - for phone in list(employee.phones): - if phone.type == 'Home': - if first: - employee.phones.remove(phone) - break - first = True - - if 'email_address' in self.fields: - address = data['email_address'] - if address: - if employee.email: - if employee.email.address != address: - employee.email.address = address - else: - employee.add_email_address(address) - else: - if len(employee.emails): - del employee.emails[:] - - -class EmployeeStoreImporter(Importer): - """ - Employee/store data importer. - """ - model_class = model.EmployeeStore - normalizer_class = normal.EmployeeStoreNormalizer - simple_fields = [ - 'uuid', - 'employee_uuid', - 'store_uuid', - ] - supported_fields = simple_fields - - -class EmployeeDepartmentImporter(Importer): - """ - Employee/department data importer. - """ - model_class = model.EmployeeDepartment - normalizer_class = normal.EmployeeDepartmentNormalizer - simple_fields = [ - 'uuid', - 'employee_uuid', - 'department_uuid', - ] - supported_fields = simple_fields - - -class CustomerGroupImporter(Importer): - """ - CustomerGroup data importer. - """ - model_class = model.CustomerGroup - supported_fields = [ - 'uuid', - 'id', - 'name', - ] - - def normalize_record(self, data): - self.string_or_null(data, 'name') - - -class CustomerImporter(Importer): - """ - Customer data importer. - """ - model_class = model.Customer - simple_fields = [ - 'uuid', - 'id', - 'name', - 'email_preference', - ] - supported_fields = simple_fields + [ - 'first_name', - 'last_name', - 'phone_number', - 'phone_number_2', - 'email_address', - 'group_id', - 'group_id_2', - ] - - def setup(self): - if 'group_id' in self.fields or 'group_id_2' in self.fields: - self.groups = self.cache_model(model.CustomerGroup, 'id') - - def cache_query_options(self): - options = [] - if 'first_name' in self.fields or 'last_name' in self.fields: - options.append(orm.joinedload(model.Customer._person).joinedload(model.CustomerPerson.person)) - if 'phone_number' in self.fields or 'phone_number_2' in self.fields: - options.append(orm.joinedload(model.Customer.phones)) - if 'email_address' in self.fields: - options.append(orm.joinedload(model.Customer.email)) - if 'group_id' in self.fields or 'group_id_2' in self.fields: - options.append(orm.joinedload_all(model.Customer._groups, model.CustomerGroupAssignment.group)) - return options - - def normalize_record(self, data): - self.string_or_null(data, 'id', 'phone_number', 'phone_number_2', - 'email_address', 'group_id', 'group_id_2') - self.prioritize_2(data, 'phone_number') - self.prioritize_2(data, 'group_id') - - def normalize_instance(self, customer): - data = super(CustomerImporter, self).normalize_instance(customer) - if 'first_name' in self.fields or 'last_name' in self.fields: - person = customer.person - if person: - if 'first_name' in self.fields: - data['first_name'] = person.first_name - if 'last_name' in self.fields: - data['last_name'] = person.last_name - else: - if 'first_name' in self.fields: - data['first_name'] = None - if 'last_name' in self.fields: - data['last_name'] = None - - if 'phone_number' in self.fields or 'phone_number_2' in self.fields: - phones = filter(lambda p: p.type == 'Voice', customer.phones) - if 'phone_number' in self.fields: - data['phone_number'] = phones[0].number if phones else None - if 'phone_number_2' in self.fields: - data['phone_number_2'] = phones[1].number if len(phones) > 1 else None - - if 'email_address' in self.fields: - email = customer.email - data['email_address'] = email.address if email else None - - if 'group_id' in self.fields: - group = customer.groups[0] if customer.groups else None - data['group_id'] = group.id if group else None - - if 'group_id_2' in self.fields: - group = customer.groups[1] if customer.groups and len(customer.groups) > 1 else None - data['group_id_2'] = group.id if group else None - - return data - - def update_instance(self, customer, data, inst_data=None): - super(CustomerImporter, self).update_instance(customer, data, inst_data) - if 'first_name' in data or 'last_name' in data: - person = customer.person - if not person: - person = model.Person() - customer.people.append(person) - if 'first_name' in data: - person.first_name = data['first_name'] - if 'last_name' in data: - person.last_name = data['last_name'] - - if 'phone_number' in data: - phones = filter(lambda p: p.type == 'Voice', customer.phones) - number = data['phone_number'] - if number: - if phones: - phone = phones[0] - if phone.number != number: - phone.number = number - else: - customer.add_phone_number(number, type='Voice') - else: - for phone in phones: - customer.phones.remove(phone) - - if 'phone_number_2' in data: - phones = filter(lambda p: p.type == 'Voice', customer.phones) - number = data['phone_number_2'] - if number: - if len(phones) > 1: - phone = phones[1] - if phone.number != number: - phone.number = number - else: - customer.add_phone_number(number, 'Voice') - else: - for phone in phones[1:]: - customer.phones.remove(phone) - - if 'email_address' in data: - address = data['email_address'] - if address: - if customer.email: - if customer.email.address != address: - customer.email.address = address - else: - customer.add_email_address(address) - else: - if len(customer.emails): - del customer.emails[:] - - if 'group_id' in data: - id_ = data['group_id'] - if id_: - group = self.groups.get(id_) - if not group: - group = model.CustomerGroup() - group.id = id_ - group.name = "(auto-created)" - self.session.add(group) - self.groups[group.id] = group - if group in customer.groups: - if group is not customer.groups[0]: - customer.groups.remove(group) - customer.groups.insert(0, group) - else: - customer.groups.insert(0, group) - else: - if customer.groups: - del customer.groups[:] - - if 'group_id_2' in data: - id_ = data['group_id_2'] - if id_: - group = self.groups.get(id_) - if not group: - group = model.CustomerGroup() - group.id_ = id_ - group.name = "(auto-created)" - self.session.add(group) - self.groups[group.id] = group - if group in customer.groups: - if len(customer.groups) > 1: - if group is not customer.groups[1]: - customer.groups.remove(group) - customer.groups.insert(1, group) - else: - if len(customer.groups) > 1: - customer.groups.insert(1, group) - else: - customer.groups.append(group) - else: - if len(customer.groups) > 1: - del customer.groups[1:] - - -class CustomerPersonImporter(Importer): - """ - CustomerPerson data importer. - """ - model_class = model.CustomerPerson - supported_fields = [ - 'uuid', - 'customer_uuid', - 'person_uuid', - 'ordinal', - ] - - -class CustomerPhoneNumberImporter(Importer): - """ - Customer phone number data importer. - """ - model_class = model.CustomerPhoneNumber - supported_fields = [ - 'uuid', - 'parent_uuid', - 'type', - 'preference', - 'number', - ] - - -class VendorImporter(Importer): - """ - Vendor data importer. - """ - model_class = model.Vendor - simple_fields = [ - 'uuid', - 'id', - 'name', - 'special_discount', - 'lead_time_days', - 'order_interval_days', - ] - phone_fields = [ - 'phone_number', - 'phone_number_2', - 'fax_number', - 'fax_number_2', - ] - contact_fields = [ - 'contact_name', - 'contact_name_2', - ] - complex_fields = [ - 'email_address', - ] - - @property - def supported_fields(self): - return (self.simple_fields + self.phone_fields + self.contact_fields - + self.complex_fields) - - def cache_query_options(self): - options = [] - for field in self.phone_fields: - if field in self.fields: - options.append(orm.joinedload(model.Vendor.phones)) - break - for field in self.contact_fields: - if field in self.fields: - options.append(orm.joinedload(model.Vendor._contacts)) - break - if 'email_address' in self.fields: - options.append(orm.joinedload(model.Vendor.email)) - return options - - def normalize_record(self, data): - self.string_or_null(data, 'id', 'name', 'email_address', - 'phone_number', 'phone_number_2', - 'fax_number', 'fax_number_2', - 'contact_name', 'contact_name_2') - self.prioritize_2(data, 'phone_number') - self.prioritize_2(data, 'fax_number') - self.prioritize_2(data, 'contact_name') - - def normalize_instance(self, vendor): - data = super(VendorImporter, self).normalize_instance(vendor) - - if 'phone_number' in self.fields: - data['phone_number'] = None - for phone in vendor.phones: - if phone.type == 'Voice': - data['phone_number'] = phone.number - break - - if 'phone_number_2' in self.fields: - data['phone_number_2'] = None - first = False - for phone in vendor.phones: - if phone.type == 'Voice': - if first: - data['phone_number_2'] = phone.number - break - first = True - - if 'fax_number' in self.fields: - data['fax_number'] = None - for phone in vendor.phones: - if phone.type == 'Fax': - data['fax_number'] = phone.number - break - - if 'fax_number_2' in self.fields: - data['fax_number_2'] = None - first = False - for phone in vendor.phones: - if phone.type == 'Fax': - if first: - data['fax_number_2'] = phone.number - break - first = True - - if 'contact_name' in self.fields: - contact = vendor.contact - data['contact_name'] = contact.display_name if contact else None - - if 'contact_name_2' in self.fields: - contact = vendor.contacts[1] if len(vendor.contacts) > 1 else None - data['contact_name_2'] = contact.display_name if contact else None - - if 'email_address' in self.fields: - email = vendor.email - data['email_address'] = email.address if email else None - - return data - - def update_instance(self, vendor, data, inst_data=None): - super(VendorImporter, self).update_instance(vendor, data, inst_data) - - if 'phone_number' in data: - number = data['phone_number'] or None - if number: - found = False - for phone in vendor.phones: - if phone.type == 'Voice': - if phone.number != number: - phone.number = number - found = True - break - if not found: - vendor.add_phone_number(number, type='Voice') - else: - for phone in list(vendor.phones): - if phone.type == 'Voice': - vendor.phones.remove(phone) - - if 'phone_number_2' in data: - number = data['phone_number_2'] or None - if number: - found = False - first = False - for phone in vendor.phones: - if phone.type == 'Voice': - if first: - if phone.number != number: - phone.number = number - found = True - break - first = True - if not found: - vendor.add_phone_number(number, type='Voice') - else: - first = False - for phone in list(vendor.phones): - if phone.type == 'Voice': - if first: - vendor.phones.remove(phone) - break - first = True - - if 'fax_number' in data: - number = data['fax_number'] or None - if number: - found = False - for phone in vendor.phones: - if phone.type == 'Fax': - if phone.number != number: - phone.number = number - found = True - break - if not found: - vendor.add_phone_number(number, type='Fax') - else: - for phone in list(vendor.phones): - if phone.type == 'Fax': - vendor.phones.remove(phone) - - if 'fax_number_2' in data: - number = data['fax_number_2'] or None - if number: - found = False - first = False - for phone in vendor.phones: - if phone.type == 'Fax': - if first: - if phone.number != number: - phone.number = number - found = True - break - first = True - if not found: - vendor.add_phone_number(number, type='Fax') - else: - first = False - for phone in list(vendor.phones): - if phone.type == 'Fax': - if first: - vendor.phones.remove(phone) - break - first = True - - if 'contact_name' in data: - if data['contact_name']: - contact = vendor.contact - if not contact: - contact = model.Person() - self.session.add(contact) - vendor.contacts.append(contact) - contact.display_name = data['contact_name'] - else: - if len(vendor.contacts): - del vendor.contacts[:] - - if 'contact_name_2' in data: - if data['contact_name_2']: - contact = vendor.contacts[1] if len(vendor.contacts) > 1 else None - if not contact: - contact = model.Person() - self.session.add(contact) - vendor.contacts.append(contact) - contact.display_name = data['contact_name_2'] - else: - if len(vendor.contacts) > 1: - del vendor.contacts[1:] - - if 'email_address' in data: - address = data['email_address'] or None - if address: - if vendor.email: - if vendor.email.address != address: - vendor.email.address = address - else: - vendor.add_email_address(address) - else: - if len(vendor.emails): - del vendor.emails[:] - - -class DepartmentImporter(Importer): - """ - Department data importer. - """ - model_class = model.Department - normalizer_class = normal.DepartmentNormalizer - simple_fields = [ - 'uuid', - 'number', - 'name', - ] - supported_fields = simple_fields - - -class SubdepartmentImporter(Importer): - """ - Subdepartment data importer. - """ - model_class = model.Subdepartment - simple_fields = [ - 'uuid', - 'number', - 'name', - 'department_uuid', - ] - supported_fields = simple_fields + [ - 'department_number', - ] - - def setup(self): - self.departments = self.cache_model(model.Department, 'number') - - def cache_query_options(self): - if 'department_number' in self.fields: - return [orm.joinedload(model.Subdepartment.department)] - - def normalize_record(self, data): - self.string_or_null(data, 'name') - - def normalize_instance(self, subdepartment): - data = super(SubdepartmentImporter, self).normalize_instance(subdepartment) - if 'department_number' in self.fields: - dept = subdepartment.department - data['department_number'] = dept.number if dept else None - return data - - def update_instance(self, subdepartment, data, inst_data=None): - super(SubdepartmentImporter, self).update_instance(subdepartment, data, inst_data) - if 'department_number' in self.fields: - dept = self.departments.get(data['department_number']) - if not dept: - dept = model.Department() - dept.number = data['department_number'] - dept.name = "(auto-created)" - self.session.add(dept) - self.departments[dept.number] = dept - subdepartment.department = dept - - -class CategoryImporter(Importer): - """ - Category data importer. - """ - model_class = model.Category - supported_fields = [ - 'uuid', - 'number', - 'name', - 'department_uuid', - ] - - def normalize_record(self, data): - self.string_or_null(data, 'name') - - -class FamilyImporter(Importer): - """ - Family data importer. - """ - model_class = model.Family - supported_fields = [ - 'uuid', - 'code', - 'name', - ] - - def normalize_record(self, data): - self.string_or_null(data, 'name') - - -class ReportCodeImporter(Importer): - """ - ReportCode data importer. - """ - model_class = model.ReportCode - supported_fields = [ - 'uuid', - 'code', - 'name', - ] - - def normalize_record(self, data): - self.int_or_null(data, 'code') - self.string_or_null(data, 'name') - - -class DepositLinkImporter(Importer): - """ - Deposit link data importer. - """ - model_class = model.DepositLink - supported_fields = [ - 'uuid', - 'code', - 'description', - 'amount', - ] - - def normalize_record(self, data): - self.int_or_null(data, 'code') - self.string_or_null(data, 'description') - - -class TaxImporter(Importer): - """ - Tax data importer. - """ - model_class = model.Tax - supported_fields = [ - 'uuid', - 'code', - 'description', - 'rate', - ] - - def normalize_record(self, data): - self.int_or_null(data, 'code') - self.string_or_null(data, 'description') - - -class BrandImporter(Importer): - """ - Brand data importer. - """ - model_class = model.Brand - supported_fields = [ - 'uuid', - 'name', - ] - - def normalize_record(self, data): - self.string_or_null(data, 'name') - - -class ProductImporter(Importer): - """ - Data importer for :class:`rattail.db.model.Product`. - """ - model_class = model.Product - simple_fields = [ - 'uuid', - 'upc', - 'description', - 'unit_size', - 'size', - 'department_uuid', - 'subdepartment_uuid', - 'category_uuid', - 'family_uuid', - 'report_code_uuid', - 'deposit_link_uuid', - 'tax_uuid', - 'brand_uuid', - 'unit_of_measure', - 'case_pack', - 'weighed', - 'organic', - 'discountable', - 'special_order', - 'not_for_sale', - 'last_sold', - ] - regular_price_fields = [ - 'regular_price_price', - 'regular_price_multiple', - 'regular_price_pack_price', - 'regular_price_pack_multiple', - 'regular_price_type', - 'regular_price_level', - 'regular_price_starts', - 'regular_price_ends', - ] - sale_price_fields = [ - 'sale_price_price', - 'sale_price_multiple', - 'sale_price_pack_price', - 'sale_price_pack_multiple', - 'sale_price_type', - 'sale_price_level', - 'sale_price_starts', - 'sale_price_ends', - ] - supported_fields = simple_fields + regular_price_fields + sale_price_fields + [ - 'brand_name', - 'department_number', - 'subdepartment_number', - 'category_number', - 'family_code', - 'report_code', - 'deposit_link_code', - 'tax_code', - 'vendor_id', - 'vendor_item_code', - 'vendor_case_cost', - ] - - def setup(self): - if 'brand_name' in self.fields: - self.brands = self.cache_model(model.Brand, 'name') - if 'department_number' in self.fields: - self.departments = self.cache_model(model.Department, 'number') - if 'subdepartment_number' in self.fields: - self.subdepartments = self.cache_model(model.Subdepartment, 'number') - if 'category_number' in self.fields: - self.categories = self.cache_model(model.Category, 'number') - if 'family_code' in self.fields: - self.families = self.cache_model(model.Family, 'code') - if 'report_code' in self.fields: - self.reportcodes = self.cache_model(model.ReportCode, 'code') - if 'deposit_link_code' in self.fields: - self.depositlinks = self.cache_model(model.DepositLink, 'code') - if 'tax_code' in self.fields: - self.taxes = self.cache_model(model.Tax, 'code') - if 'vendor_id' in self.fields: - self.vendors = self.cache_model(model.Vendor, 'id') - - def cache_query_options(self): - options = [] - if 'brand_name' in self.fields: - options.append(orm.joinedload(model.Product.brand)) - if 'department_number' in self.fields: - options.append(orm.joinedload(model.Product.department)) - if 'subdepartment_number' in self.fields: - options.append(orm.joinedload(model.Product.subdepartment)) - if 'category_number' in self.fields: - options.append(orm.joinedload(model.Product.category)) - if 'family_code' in self.fields: - options.append(orm.joinedload(model.Product.family)) - if 'report_code' in self.fields: - options.append(orm.joinedload(model.Product.report_code)) - if 'deposit_link_code' in self.fields: - options.append(orm.joinedload(model.Product.deposit_link)) - if 'tax_code' in self.fields: - options.append(orm.joinedload(model.Product.tax)) - joined_prices = False - for field in self.regular_price_fields: - if field in self.fields: - options.append(orm.joinedload(model.Product.prices)) - options.append(orm.joinedload(model.Product.regular_price)) - joined_prices = True - break - for field in self.sale_price_fields: - if field in self.fields: - if not joined_prices: - options.append(orm.joinedload(model.Product.prices)) - options.append(orm.joinedload(model.Product.current_price)) - break - if ('vendor_id' in self.fields - or 'vendor_item_code' in self.fields - or 'vendor_case_cost' in self.fields): - options.append(orm.joinedload(model.Product.cost)) - return options - - def normalize_record(self, data): - self.string_or_null(data, 'brand_name', 'description', 'unit_of_measure', - 'vendor_id', 'vendor_item_code') - self.int_or_null(data, 'family_code', 'report_code', 'deposit_link_code', 'tax_code') - - def normalize_instance(self, product): - data = super(ProductImporter, self).normalize_instance(product) - if 'brand_name' in self.fields: - data['brand_name'] = product.brand.name if product.brand else None - if 'department_number' in self.fields: - data['department_number'] = product.department.number if product.department else None - if 'subdepartment_number' in self.fields: - data['subdepartment_number'] = product.subdepartment.number if product.subdepartment else None - if 'category_number' in self.fields: - data['category_number'] = product.category.number if product.category else None - if 'family_code' in self.fields: - data['family_code'] = product.family.code if product.family else None - if 'report_code' in self.fields: - data['report_code'] = product.report_code.code if product.report_code else None - if 'deposit_link_code' in self.fields: - data['deposit_link_code'] = product.deposit_link.code if product.deposit_link else None - if 'tax_code' in self.fields: - data['tax_code'] = product.tax.code if product.tax else None - - for field in self.regular_price_fields: - if field in self.fields: - price = product.regular_price - if 'regular_price_price' in self.fields: - data['regular_price_price'] = price.price if price else None - if 'regular_price_multiple' in self.fields: - data['regular_price_multiple'] = price.multiple if price else None - if 'regular_price_pack_price' in self.fields: - data['regular_price_pack_price'] = price.pack_price if price else None - if 'regular_price_pack_multiple' in self.fields: - data['regular_price_pack_multiple'] = price.pack_multiple if price else None - if 'regular_price_type' in self.fields: - data['regular_price_type'] = price.type if price else None - if 'regular_price_level' in self.fields: - data['regular_price_level'] = price.level if price else None - if 'regular_price_starts' in self.fields: - data['regular_price_starts'] = price.starts if price else None - if 'regular_price_ends' in self.fields: - data['regular_price_ends'] = price.ends if price else None - break - - for field in self.sale_price_fields: - if field in self.fields: - price = product.current_price - if 'sale_price_price' in self.fields: - data['sale_price_price'] = price.price if price else None - if 'sale_price_multiple' in self.fields: - data['sale_price_multiple'] = price.multiple if price else None - if 'sale_price_pack_price' in self.fields: - data['sale_price_pack_price'] = price.pack_price if price else None - if 'sale_price_pack_multiple' in self.fields: - data['sale_price_pack_multiple'] = price.pack_multiple if price else None - if 'sale_price_type' in self.fields: - data['sale_price_type'] = price.type if price else None - if 'sale_price_level' in self.fields: - data['sale_price_level'] = price.level if price else None - if 'sale_price_starts' in self.fields: - data['sale_price_starts'] = price.starts if price else None - if 'sale_price_ends' in self.fields: - data['sale_price_ends'] = price.ends if price else None - break - - if 'vendor_id' in self.fields or 'vendor_item_code' in self.fields or 'vendor_case_cost' in self.fields: - cost = product.cost - if 'vendor_id' in self.fields: - data['vendor_id'] = cost.vendor.id if cost else None - if 'vendor_item_code' in self.fields: - data['vendor_item_code'] = cost.code if cost else None - if 'vendor_case_cost' in self.fields: - data['vendor_case_cost'] = cost.case_cost if cost else None - - return data - - def update_instance(self, product, data, inst_data=None): - super(ProductImporter, self).update_instance(product, data, inst_data) - - if 'brand_name' in data: - name = data['brand_name'] - if name: - brand = self.brands.get(name) - if not brand: - brand = model.Brand() - brand.name = name - self.session.add(brand) - self.brands[brand.name] = brand - product.brand = brand - else: - if product.brand: - product.brand = None - - if 'department_number' in data: - number = data['department_number'] - if number: - dept = self.departments.get(number) - if not dept: - dept = model.Department() - dept.number = number - dept.name = "(auto-created)" - self.session.add(dept) - self.departments[dept.number] = dept - product.department = dept - else: - if product.department: - product.department = None - - if 'subdepartment_number' in data: - number = data['subdepartment_number'] - if number: - sub = self.subdepartments.get(number) - if not sub: - sub = model.Subdepartment() - sub.number = number - sub.name = "(auto-created)" - self.session.add(sub) - self.subdepartments[number] = sub - product.subdepartment = sub - else: - if product.subdepartment: - product.subdepartment = None - - if 'category_number' in data: - number = data['category_number'] - if number: - cat = self.categories.get(number) - if not cat: - cat = model.Category() - cat.number = number - cat.name = "(auto-created)" - self.session.add(cat) - self.categories[number] = cat - product.category = cat - else: - if product.category: - product.category = None - - if 'family_code' in data: - code = data['family_code'] - if code: - family = self.families.get(code) - if not family: - family = model.Family() - family.code = code - family.name = "(auto-created)" - self.session.add(family) - self.families[family.code] = family - product.family = family - else: - if product.family: - product.family = None - - if 'report_code' in data: - code = data['report_code'] - if code: - rc = self.reportcodes.get(code) - if not rc: - rc = model.ReportCode() - rc.code = code - rc.name = "(auto-created)" - self.session.add(rc) - self.reportcodes[rc.code] = rc - product.report_code = rc - else: - if product.report_code: - product.report_code = None - - if 'deposit_link_code' in data: - code = data['deposit_link_code'] - if code: - link = self.depositlinks.get(code) - if not link: - link = model.DepositLink() - link.code = code - link.description = "(auto-created)" - self.session.add(link) - self.depositlinks[link.code] = link - product.deposit_link = link - else: - if product.deposit_link: - product.deposit_link = None - - if 'tax_code' in data: - code = data['tax_code'] - if code: - tax = self.taxes.get(code) - if not tax: - tax = model.Tax() - tax.code = code - tax.description = "(auto-created)" - tax.rate = 0 - self.session.add(tax) - self.taxes[tax.code] = tax - product.tax = tax - elif product.tax: - product.tax = None - - create = False - delete = False - for field in self.regular_price_fields: - if field in data: - delete = True - if data[field] is not None: - create = True - break - if create: - price = product.regular_price - if not price: - price = model.ProductPrice() - product.prices.append(price) - product.regular_price = price - if 'regular_price_price' in data: - price.price = data['regular_price_price'] - if 'regular_price_multiple' in data: - price.multiple = data['regular_price_multiple'] - if 'regular_price_pack_price' in data: - price.pack_price = data['regular_price_pack_price'] - if 'regular_price_pack_multiple' in data: - price.pack_multiple = data['regular_price_pack_multiple'] - if 'regular_price_type' in data: - price.type = data['regular_price_type'] - if 'regular_price_level' in data: - price.level = data['regular_price_level'] - if 'regular_price_starts' in data: - price.starts = data['regular_price_starts'] - if 'regular_price_ends' in data: - price.ends = data['regular_price_ends'] - elif delete: - if product.regular_price: - product.regular_price = None - - create = False - delete = False - for field in self.sale_price_fields: - if field in data: - delete = True - if data[field]: - create = True - break - if create: - price = product.current_price - if not price: - price = model.ProductPrice() - product.prices.append(price) - product.current_price = price - if 'sale_price_price' in data: - price.price = data['sale_price_price'] - if 'sale_price_multiple' in data: - price.multiple = data['sale_price_multiple'] - if 'sale_price_pack_price' in data: - price.pack_price = data['sale_price_pack_price'] - if 'sale_price_pack_multiple' in data: - price.pack_multiple = data['sale_price_pack_multiple'] - if 'sale_price_type' in data: - price.type = data['sale_price_type'] - if 'sale_price_level' in data: - price.level = data['sale_price_level'] - if 'sale_price_starts' in data: - price.starts = data['sale_price_starts'] - if 'sale_price_ends' in data: - price.ends = data['sale_price_ends'] - elif delete: - if product.current_price: - product.current_price = None - - if 'vendor_id' in data: - id_ = data['vendor_id'] - if id_: - vendor = self.vendors.get(id_) - if not vendor: - vendor = model.Vendor() - vendor.id = id_ - vendor.name = "(auto-created)" - self.session.add(vendor) - self.vendors[id_] = vendor - if product.cost: - if product.cost.vendor is not vendor: - cost = product.cost_for_vendor(vendor) - if not cost: - cost = model.ProductCost() - cost.vendor = vendor - product.costs.insert(0, cost) - else: - cost = model.ProductCost() - cost.vendor = vendor - product.costs.append(cost) - # TODO: This seems heavy-handed, but also seems necessary - # to populate the `Product.cost` relationship... - self.session.add(product) - self.session.flush() - self.session.refresh(product) - else: - if product.cost: - del product.costs[:] - - if 'vendor_item_code' in self.fields: - code = data['vendor_item_code'] - if data.get('vendor_id'): - if product.cost: - product.cost.code = code - else: - log.warning("product has no cost, so can't set vendor_item_code: {0}".format(product)) - - if 'vendor_case_cost' in self.fields: - cost = data['vendor_case_cost'] - if data.get('vendor_id'): - if product.cost: - product.cost.case_cost = cost - else: - log.warning("product has no cost, so can't set vendor_case_cost: {0}".format(product)) - - -class ProductCodeImporter(Importer): - """ - Data importer for :class:`rattail.db.model.ProductCode`. - """ - model_class = model.ProductCode - simple_fields = [ - 'uuid', - 'product_uuid', - 'ordinal', - 'code', - ] - supported_fields = simple_fields + [ - 'product_upc', - 'primary', - ] - - def setup(self): - if 'product_upc' in self.fields: - self.products = self.cache_model(model.Product, 'upc') - - def cache_query_options(self): - if 'product_upc' in self.fields: - return [orm.joinedload(model.ProductCode.product)] - - def normalize_record(self, data): - self.string_or_null(data, 'code') - - def normalize_instance(self, code): - data = super(ProductCodeImporter, self).normalize_instance(code) - if 'product_upc' in self.fields: - data['product_upc'] = code.product.upc - if 'primary' in self.fields: - data['primary'] = code.ordinal == 1 - return data - - def update_instance(self, code, data, inst_data=None): - super(ProductCodeImporter, self).update_instance(code, data, inst_data) - - if 'product_upc' in data and 'product_uuid' not in data: - upc = data['product_upc'] - assert upc, "Source data has no product_upc value: {0}".format(repr(data)) - product = self.products.get(upc) - if not product: - product = model.Product() - product.upc = upc - product.description = "(auto-created)" - self.session.add(product) - self.products[product.upc] = product - product._codes.append(code) - else: - if code not in product._codes: - product._codes.append(code) - - if 'primary' in data: - if data['primary']: - if code.ordinal != 1: - product = code.product - product._codes.remove(code) - product._codes.insert(0, code) - product._codes.reorder() - elif code.ordinal == 1: - product = code.product - if len(product._codes) > 1: - product._codes.remove(code) - product._codes.append(code) - product._codes.reorder() - - -class ProductCostImporter(Importer): - """ - Data importer for :class:`rattail.db.model.ProductCost`. - """ - model_class = model.ProductCost - simple_fields = [ - 'uuid', - 'product_uuid', - 'vendor_uuid', - 'preference', - 'code', - 'case_size', - 'case_cost', - 'pack_size', - 'pack_cost', - 'unit_cost', - 'effective', - ] - supported_fields = simple_fields + [ - 'product_upc', - 'vendor_id', - 'preferred', - ] - - def setup(self): - if 'product_upc' in self.fields: - self.products = self.cache_model(model.Product, 'upc') - if 'vendor_id' in self.fields: - self.vendors = self.cache_model(model.Vendor, 'id') - - def cache_query_options(self): - options = [] - if 'product_upc' in self.fields: - options.append(orm.joinedload(model.ProductCost.product)) - if 'vendor_id' in self.fields: - options.append(orm.joinedload(model.ProductCost.vendor)) - return options - - def normalize_instance(self, cost): - data = super(ProductCostImporter, self).normalize_instance(cost) - if 'product_upc' in self.fields: - data['product_upc'] = cost.product.upc - if 'vendor_id' in self.fields: - data['vendor_id'] = cost.vendor.id - data['preferred'] = cost.preference == 1 - return data - - def update_instance(self, cost, data, inst_data=None): - super(ProductCostImporter, self).update_instance(cost, data, inst_data) - - if 'product_upc' in data and 'product_uuid' not in data: - upc = data['product_upc'] - assert upc, "Source data has no product_upc value: {0}".format(repr(data)) - product = self.products.get(upc) - if not product: - product = model.Product() - product.upc = upc - product.description = "(auto-created)" - self.session.add(product) - self.products[product.upc] = product - if not cost.product: - product.costs.append(cost) - elif cost.product is not product: - log.warning("duplicate products detected for UPC {0}".format(upc.pretty())) - - if 'vendor_id' in data and 'vendor_uuid' not in data: - id_ = data['vendor_id'] - assert id_, "Source data has no vendor_id value: {0}".format(repr(data)) - vendor = self.vendors.get(id_) - if not vendor: - vendor = model.Vendor() - vendor.id = id_ - vendor.name = "(auto-created)" - self.session.add(vendor) - self.vendors[vendor.id] = vendor - cost.vendor = vendor - - if 'preferred' in data: - if data['preferred']: - if cost.preference != 1: - product = cost.product - product.costs.remove(cost) - product.costs.insert(0, cost) - else: - if cost.preference == 1: - product = cost.product - if len(product.costs) > 1: - product.costs.remove(cost) - product.costs.append(cost) - product.costs.reorder() diff --git a/rattail/db/importing/normal.py b/rattail/db/importing/normal.py deleted file mode 100644 index 12985e65..00000000 --- a/rattail/db/importing/normal.py +++ /dev/null @@ -1,164 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Rattail Data Normalization -""" - -from __future__ import unicode_literals, absolute_import - - -class Normalizer(object): - """ - Base class for data normalizers. - """ - - def normalize(self, instance): - raise NotImplementedError - - -class UserNormalizer(Normalizer): - """ - Normalizer for user data. - """ - # Must set this to the administrator Role instance. - admin = None - - def normalize(self, user): - return { - 'uuid': user.uuid, - 'username': user.username, - 'password': user.password, - 'salt': user.salt, - 'person_uuid': user.person_uuid, - 'active': user.active, - 'admin': self.admin in user.roles, - } - - -class DepartmentNormalizer(Normalizer): - """ - Normalizer for department data. - """ - - def normalize(self, department): - return { - 'uuid': department.uuid, - 'number': department.number, - 'name': department.name, - } - - -class EmployeeNormalizer(Normalizer): - """ - Normalizer for employee data. - """ - - def normalize(self, employee): - person = employee.person - customer = person.customers[0] if person.customers else None - data = { - 'uuid': employee.uuid, - 'id': employee.id, - 'person_uuid': person.uuid, - 'customer_id': customer.id if customer else None, - 'status': employee.status, - 'first_name': person.first_name, - 'last_name': person.last_name, - 'display_name': employee.display_name, - 'person_display_name': person.display_name, - } - - data['phone_number'] = None - for phone in employee.phones: - if phone.type == 'Home': - data['phone_number'] = phone.number - break - - data['phone_number_2'] = None - first = False - for phone in employee.phones: - if phone.type == 'Home': - if first: - data['phone_number_2'] = phone.number - break - first = True - - email = employee.email - data['email_address'] = email.address if email else None - - return data - - -class EmployeeStoreNormalizer(Normalizer): - """ - Normalizer for employee_x_store data. - """ - - def normalize(self, emp_store): - return { - 'uuid': emp_store.uuid, - 'employee_uuid': emp_store.employee_uuid, - 'store_uuid': emp_store.store_uuid, - } - - -class EmployeeDepartmentNormalizer(Normalizer): - """ - Normalizer for employee_x_department data. - """ - - def normalize(self, emp_dept): - return { - 'uuid': emp_dept.uuid, - 'employee_uuid': emp_dept.employee_uuid, - 'department_uuid': emp_dept.department_uuid, - } - - -class MessageNormalizer(Normalizer): - """ - Normalizer for message data. - """ - - def normalize(self, message): - return { - 'uuid': message.uuid, - 'sender_uuid': message.sender_uuid, - 'subject': message.subject, - 'body': message.body, - 'sent': message.sent, - } - - -class MessageRecipientNormalizer(Normalizer): - """ - Normalizer for message recipient data. - """ - - def normalize(self, recip): - return { - 'uuid': recip.uuid, - 'message_uuid': recip.message_uuid, - 'recipient_uuid': recip.recipient_uuid, - 'status': recip.status, - } diff --git a/rattail/db/importing/providers/__init__.py b/rattail/db/importing/providers/__init__.py deleted file mode 100644 index cd55e655..00000000 --- a/rattail/db/importing/providers/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2015 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Import Data Providers -""" - -from .core import DataProvider, QueryProvider diff --git a/rattail/db/importing/providers/core.py b/rattail/db/importing/providers/core.py deleted file mode 100644 index 5da3f633..00000000 --- a/rattail/db/importing/providers/core.py +++ /dev/null @@ -1,182 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2015 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Import Data Providers -""" - -from __future__ import unicode_literals - -import datetime - -from rattail.core import Object -from rattail.db.cache import cache_model - - -class DataProvider(Object): - """ - Base class for import data providers. - """ - importer_class = None - normalize_progress_message = "Normalizing source data" - progress = None - - def __init__(self, config, session, importer=None, importer_kwargs={}, **kwargs): - self.config = config - self.session = session - if importer is None: - self.importer = self.importer_class(config, session, **importer_kwargs) - else: - self.importer = importer - super(DataProvider, self).__init__(**kwargs) - self.importer.provider = self - - @property - def model_class(self): - return self.importer.model_class - - @property - def key(self): - """ - Key by which records should be matched between the source data and - Rattail. - """ - raise NotImplementedError("Please define the `key` for your data provider.") - - @property - def model_name(self): - return self.model_class.__name__ - - def cache_model(self, model_class, key, **kwargs): - """ - Convenience method for caching a model. - """ - kwargs.setdefault('progress', self.progress) - return cache_model(self.session, model_class, key=key, **kwargs) - - def setup(self): - """ - Perform any setup necessary, e.g. cache lookups for existing data. - """ - - def get_data(self, progress=None, normalize_progress_message=None): - """ - Return the full set of normalized data which is to be imported. - """ - self.now = datetime.datetime.utcnow() - self.progress = progress - self.setup() - source_data = self.get_source_data(progress=progress) - data = self.normalize_source_data(source_data, progress=progress) - self.teardown() - return data - - def teardown(self): - """ - Perform any cleanup necessary, after the main data run. - """ - - def get_source_data(self, progress=None): - """ - Return the data which is to be imported. - """ - return [] - - def normalize_source_data(self, source_data, progress=None): - """ - Return a normalized version of the full set of source data. - """ - data = [] - count = len(source_data) - if count == 0: - return data - prog = None - if progress: - prog = progress(self.normalize_progress_message, count) - for i, record in enumerate(source_data, 1): - record = self.normalize(record) - if record: - data.append(record) - if prog: - prog.update(i) - if prog: - prog.destroy() - return data - - def normalize(self, data): - """ - Normalize a source data record. Generally this is where the provider - may massage the record in any way necessary, so that its values are - more "native" and can be used for direct comparison with, and - assignment to, the target model instance. - - Note that if you override this, your method must return the data to be - imported. If your method returns ``None`` then that particular record - would be skipped and not imported. - """ - return data - - def int_(self, value): - """ - Coerce ``value`` to an integer, or return ``None`` if that can't be - done cleanly. - """ - try: - return int(value) - except (TypeError, ValueError): - return None - - -class QueryDataProxy(object): - """ - Simple proxy to wrap a SQLAlchemy query and make it sort of behave like a - normal sequence, as much as needed to make a ``DataProvider`` happy. - """ - - def __init__(self, query): - self.query = query - - def __len__(self): - return self.query.count() - - def __iter__(self): - return iter(self.query) - - -class QueryProvider(DataProvider): - """ - Data provider whose data source is a SQLAlchemy query. Note that this - needn't be a Rattail database query; any database will work as long as a - SQLAlchemy query is behind it. - """ - - def query(self): - """ - Return the query which will define the data set. - """ - raise NotImplementedError - - def get_source_data(self, progress=None): - """ - Return the data which is to be imported. - """ - return QueryDataProxy(self.query()) diff --git a/rattail/db/importing/providers/csv.py b/rattail/db/importing/providers/csv.py deleted file mode 100644 index 23aa70f8..00000000 --- a/rattail/db/importing/providers/csv.py +++ /dev/null @@ -1,177 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2015 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -CSV Import Data Providers -""" - -from __future__ import unicode_literals - -import datetime -from decimal import Decimal - -from .core import DataProvider -from rattail.db import model -from rattail.gpc import GPC -from rattail.db.importing import models -from rattail.util import load_object -from rattail.csvutil import UnicodeDictReader -from rattail.db.util import maxlen -from rattail.time import localtime, make_utc - - -def make_provider(config, session, spec, **kwargs): - """ - Create a provider instance according to the given spec. For now..see the - source code for more details. - """ - provider = None - if '.' not in spec and ':' not in spec: - from rattail.db.importing.providers import csv - if hasattr(csv, spec): - provider = getattr(csv, spec) - elif hasattr(csv, '{0}Provider'.format(spec)): - provider = getattr(csv, '{0}Provider'.format(spec)) - else: - provider = load_object(spec) - if provider: - return provider(config, session, **kwargs) - - -class CsvProvider(DataProvider): - """ - Base class for CSV data providers. - """ - time_format = '%Y-%m-%d %H:%M:%S' - - def get_source_data(self, progress=None): - with open(self.data_path, 'rb') as f: - reader = UnicodeDictReader(f) - return list(reader) - - def make_utc(self, time): - if time is None: - return None - return make_utc(localtime(self.config, time)) - - def make_time(self, value): - if not value: - return None - time = datetime.datetime.strptime(value, self.time_format) - return self.make_utc(time) - - -class ProductProvider(CsvProvider): - """ - CSV product data provider. - """ - importer_class = models.ProductImporter - supported_fields = [ - 'uuid', - 'upc', - 'description', - 'size', - 'department_uuid', - 'subdepartment_uuid', - 'category_uuid', - 'brand_uuid', - 'regular_price', - 'sale_price', - 'sale_starts', - 'sale_ends', - ] - maxlen_description = maxlen(model.Product.description) - maxlen_size = maxlen(model.Product.size) - - def normalize(self, data): - - if 'upc' in data: - upc = data['upc'] - data['upc'] = GPC(upc) if upc else None - - # Warn about truncation until Rattail schema is addressed. - if 'description' in data: - description = data['description'] or '' - if self.maxlen_description and len(description) > self.maxlen_description: - log.warning("product description is more than {} chars and will be truncated: {}".format( - self.maxlen_description, repr(description))) - description = description[:self.maxlen_description] - data['description'] = description - - # Warn about truncation until Rattail schema is addressed. - if 'size' in data: - size = data['size'] or '' - if self.maxlen_size and len(size) > self.maxlen_size: - log.warning("product size is more than {} chars and will be truncated: {}".format( - self.maxlen_size, repr(size))) - size = size[:self.maxlen_size] - data['size'] = size - - if 'department_uuid' in data: - data['department_uuid'] = data['department_uuid'] or None - - if 'subdepartment_uuid' in data: - data['subdepartment_uuid'] = data['subdepartment_uuid'] or None - - if 'category_uuid' in data: - data['category_uuid'] = data['category_uuid'] or None - - if 'brand_uuid' in data: - data['brand_uuid'] = data['brand_uuid'] or None - - if 'regular_price' in data: - price = data['regular_price'] - data['regular_price'] = Decimal(price) if price else None - - # Determine if sale price is currently active; if it is not then we - # will declare None for all sale fields. - if 'sale_starts' in data: - data['sale_starts'] = self.make_time(data['sale_starts']) - if 'sale_ends' in data: - data['sale_ends'] = self.make_time(data['sale_ends']) - if 'sale_price' in data: - price = data['sale_price'] - data['sale_price'] = Decimal(price) if price else None - if data['sale_price']: - sale_starts = data.get('sale_starts') - sale_ends = data.get('sale_ends') - active = False - if sale_starts and sale_ends: - if sale_starts <= self.now <= sale_ends: - active = True - elif sale_starts: - if sale_starts <= self.now: - active = True - elif sale_ends: - if self.now <= sale_ends: - active = True - else: - active = True - if not active: - data['sale_price'] = None - data['sale_starts'] = None - data['sale_ends'] = None - else: - data['sale_starts'] = None - data['sale_ends'] = None - - return data diff --git a/rattail/db/newimporting/__init__.py b/rattail/db/newimporting/__init__.py deleted file mode 100644 index f99c292b..00000000 --- a/rattail/db/newimporting/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -(New) Data Importing Framework -""" - -from __future__ import unicode_literals, absolute_import - -from .importers import Importer, QueryImporter, SQLAlchemyImporter, BulkPostgreSQLImporter -from .handlers import ImportHandler, SQLAlchemyImportHandler, BulkPostgreSQLImportHandler -from . import model diff --git a/rattail/db/newimporting/handlers.py b/rattail/db/newimporting/handlers.py deleted file mode 100644 index 8a9229a3..00000000 --- a/rattail/db/newimporting/handlers.py +++ /dev/null @@ -1,323 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Import Handlers -""" - -from __future__ import unicode_literals, absolute_import - -import sys -import datetime -import logging - -import humanize - -from rattail.util import OrderedDict -from rattail.mail import send_email - - -log = logging.getLogger(__name__) - - -class ImportHandler(object): - """ - Base class for all import handlers. - """ - local_title = "Rattail" - host_title = "Host/Other" - session = None - progress = None - dry_run = False - - def __init__(self, config=None, **kwargs): - self.config = config - self.importers = self.get_importers() - for key, value in kwargs.iteritems(): - setattr(self, key, value) - - def get_importers(self): - """ - Returns a dict of all available importers, where the values are - importer factories. All subclasses will want to override this. Note - that if you return an ``OrderedDict`` instance, you can affect the - ordering of keys in the command line help system, etc. - """ - return {} - - def get_importer_keys(self): - """ - Returns a list of keys corresponding to the available importers. - """ - return list(self.importers.iterkeys()) - - def get_default_keys(self): - """ - Returns a list of keys corresponding to the default importers. - Override this if you wish certain importers to be excluded by default, - e.g. when first testing them out etc. - """ - return self.get_importer_keys() - - def get_importer(self, key): - """ - Returns an importer instance corresponding to the given key. - """ - kwargs = self.get_importer_kwargs(key) - kwargs['config'] = self.config - kwargs['session'] = self.session - importer = self.importers[key](**kwargs) - importer.handler = self - return importer - - def get_importer_kwargs(self, key): - """ - Return a dict of kwargs to be used when construcing an importer with - the given key. - """ - kwargs = {} - if hasattr(self, 'host_session'): - kwargs['host_session'] = self.host_session - return kwargs - - def import_data(self, keys, args): - """ - Import all data for the given importer keys. - """ - self.now = datetime.datetime.utcnow() - self.dry_run = args.dry_run - self.begin_transaction() - self.setup() - changes = OrderedDict() - - for key in keys: - importer = self.get_importer(key) - if not importer: - log.warning("skipping unknown importer: {}".format(key)) - continue - - created, updated, deleted = importer.import_data(args, progress=self.progress) - - changed = bool(created or updated or deleted) - logger = log.warning if changed and args.warnings else log.info - logger("{} -> {}: added {}, updated {}, deleted {} {} records".format( - self.host_title, self.local_title, len(created), len(updated), len(deleted), key)) - if changed: - changes[key] = created, updated, deleted - - if changes: - self.process_changes(changes, args) - - if self.dry_run: - self.rollback_transaction() - else: - self.commit_transaction() - - self.teardown() - return changes - - def begin_transaction(self): - self.begin_host_transaction() - self.begin_local_transaction() - - def begin_host_transaction(self): - if hasattr(self, 'make_host_session'): - self.host_session = self.make_host_session() - - def begin_local_transaction(self): - pass - - def setup(self): - """ - Perform any setup necessary, prior to running the import task(s). - """ - - def rollback_transaction(self): - self.rollback_host_transaction() - self.rollback_local_transaction() - - def rollback_host_transaction(self): - if hasattr(self, 'host_session'): - self.host_session.rollback() - self.host_session.close() - self.host_session = None - - def rollback_local_transaction(self): - pass - - def commit_transaction(self): - self.commit_host_transaction() - self.commit_local_transaction() - - def commit_host_transaction(self): - if hasattr(self, 'host_session'): - self.host_session.commit() - self.host_session.close() - self.host_session = None - - def commit_local_transaction(self): - pass - - def teardown(self): - """ - Perform any cleanup necessary, after running the import task(s). - """ - - def process_changes(self, changes, args): - """ - This method is called any time changes occur, regardless of whether the - import is running in "warnings" mode. Default implementation however - is to do nothing unless warnings mode is in effect, in which case an - email notification will be sent. - """ - # TODO: This whole thing needs a re-write...but for now, waiting until - # the old importer has really gone away, so we can share its email - # template instead of bothering with something more complicated. - - if not args.warnings: - return - - data = { - 'local_title': self.local_title, - 'host_title': self.host_title, - 'argv': sys.argv, - 'runtime': humanize.naturaldelta(datetime.datetime.utcnow() - self.now), - 'changes': changes, - 'dry_run': args.dry_run, - 'render_record': RecordRenderer(self.config), - 'max_display': 15, - } - - command = getattr(self, 'command', None) - if command: - data['command'] = '{} {}'.format(command.parent.name, command.name) - else: - data['command'] = None - - if command: - key = '{}_{}_updates'.format(command.parent.name, command.name) - key = key.replace('-', '_') - else: - key = 'rattail_import_updates' - - send_email(self.config, key, fallback_key='rattail_import_updates', data=data) - - -class SQLAlchemyImportHandler(ImportHandler): - """ - Handler for imports for which the host data source is represented by a - SQLAlchemy engine and ORM. - """ - host_session = None - - def make_host_session(self): - raise NotImplementedError - - -class BulkPostgreSQLImportHandler(ImportHandler): - """ - Handler for bulk imports which target PostgreSQL on the local side. - """ - - def import_data(self, keys, args): - """ - Import all data for the given importer keys. - """ - self.now = datetime.datetime.utcnow() - self.dry_run = args.dry_run - self.begin_transaction() - self.setup() - - for key in keys: - importer = self.get_importer(key) - if not importer: - log.warning("skipping unknown importer: {}".format(key)) - continue - - created = importer.import_data(args, progress=self.progress) - log.info("{} -> {}: added {}, updated 0, deleted 0 {} records".format( - self.host_title, self.local_title, created, key)) - - if self.dry_run: - self.rollback_transaction() - else: - self.commit_transaction() - - self.teardown() - - -class RecordRenderer(object): - """ - Record renderer for email notifications sent from data import jobs. - """ - - def __init__(self, config): - self.config = config - - def __call__(self, record): - return self.render(record) - - def render(self, record): - """ - Render the given record. - """ - key = record.__class__.__name__.lower() - renderer = getattr(self, 'render_{}'.format(key), None) - if renderer: - return renderer(record) - - label = self.get_label(record) - url = self.get_url(record) - if url: - return '{}'.format(url, label) - return label - - def get_label(self, record): - key = record.__class__.__name__.lower() - label = getattr(self, 'label_{}'.format(key), self.label) - return label(record) - - def label(self, record): - return unicode(record) - - def get_url(self, record): - """ - Fetch / generate a URL for the given data record. You should *not* - override this method, but do :meth:`url()` instead. - """ - key = record.__class__.__name__.lower() - url = getattr(self, 'url_{}'.format(key), self.url) - return url(record) - - def url(self, record): - """ - Fetch / generate a URL for the given data record. - """ - if hasattr(record, 'uuid'): - url = self.config.get('tailbone', 'url') - if url: - url = url.rstrip('/') - name = '{}s'.format(record.__class__.__name__.lower()) - if name == 'persons': # FIXME, obviously this is a hack - name = 'people' - url = '{}/{}/{{uuid}}'.format(url, name) - return url.format(uuid=record.uuid) diff --git a/rattail/db/newimporting/importers.py b/rattail/db/newimporting/importers.py deleted file mode 100644 index 32233e5c..00000000 --- a/rattail/db/newimporting/importers.py +++ /dev/null @@ -1,640 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Data Importers -""" - -from __future__ import unicode_literals, absolute_import - -import datetime -import logging - -from sqlalchemy import orm -from sqlalchemy.orm.exc import NoResultFound - -from rattail.db import cache -from rattail.time import make_utc - - -log = logging.getLogger(__name__) - - -class Importer(object): - """ - Base class for all data importers. - """ - key = 'uuid' - cached_instances = None - allow_create = True - allow_update = True - allow_delete = True - dry_run = False - - def __init__(self, config=None, session=None, fields=None, key=None, **kwargs): - self.config = config - self.session = session - self.fields = fields or self.supported_fields - if key: - self.key = key - if isinstance(self.key, basestring): - self.key = (self.key,) - for key, value in kwargs.iteritems(): - setattr(self, key, value) - - @property - def model_class(self): - """ - This should return a reference to the model class which the importer - "targets" so to speak. - """ - raise NotImplementedError - - @property - def model_name(self): - """ - Returns the string 'name' of the model class which the importer targets. - """ - return self.model_class.__name__ - - @property - def model_mapper(self): - """ - This should return the SQLAlchemy mapper for the model class. - """ - return orm.class_mapper(self.model_class) - - @property - def model_table(self): - """ - Returns the underlying table used by the primary local data model class. - """ - tables = self.model_mapper.tables - assert len(tables) == 1 - return tables[0] - - @property - def simple_fields(self): - """ - The list of field names which may be considered "simple" and therefore - treated as such, i.e. with basic getattr/setattr calls. Note that this - only applies to the local / target side, it has no effect on the - upstream / foreign side. - """ - return list(self.model_mapper.columns.keys()) - - @property - def supported_fields(self): - """ - The list of field names which are supported in general by the importer. - Note that this only applies to the local / target side, it has no - effect on the upstream / foreign side. - """ - return self.simple_fields - - @property - def normalize_progress_message(self): - return "Reading {} data from {}".format(self.model_name, self.handler.host_title) - - def setup(self): - """ - Perform any setup necessary, e.g. cache lookups for existing data. - """ - - def teardown(self): - """ - Perform any cleanup after import, if necessary. - """ - - def _setup(self, args, progress): - self.now = datetime.datetime.utcnow() - self.allow_create = self.allow_create and args.create - self.allow_update = self.allow_update and args.update - self.allow_delete = self.allow_delete and args.delete - self.dry_run = args.dry_run - self.args = args - self.progress = progress - self.setup() - - def import_data(self, args, progress=None): - """ - Import some data! This is the core body of logic for that, regardless - of where data is coming from or where it's headed. Note that this - method handles deletions as well as adds/updates. - """ - self._setup(args, progress) - created = updated = deleted = [] - - data = self.normalize_source_data() - self.cached_instances = self.cache_instance_data(data) - - # Normalize source data set in order to prune duplicate keys. This is - # for the sake of sanity since duplicates typically lead to a ping-pong - # effect, where a "clean" (change-less) import is impossible. - unique = {} - for record in data: - key = self.get_key(record) - if key in unique: - log.warning("duplicate records detected from {} for key: {}".format( - self.handler.host_title, key)) - unique[key] = record - data = [] - for key in sorted(unique): - data.append(unique[key]) - - if self.allow_create or self.allow_update: - created, updated = self._import_create_update(data, args) - - if self.allow_delete: - changes = len(created) + len(updated) - if args.max_total and changes >= args.max_total: - log.warning("max of {} total changes already reached; skipping deletions".format(args.max_total)) - else: - deleted = self._import_delete(data, args, host_keys=set(unique), changes=changes) - - self.teardown() - return created, updated, deleted - - def _import_create_update(self, data, args): - """ - Import the given data; create and/or update records as needed and - according to the args provided. - """ - created = [] - updated = [] - count = len(data) - if not count: - return created, updated - - prog = None - if self.progress: - prog = self.progress("Importing {} data".format(self.model_name), count) - - keys_seen = set() - for i, source_data in enumerate(data, 1): - - # Get what should be the unique key for the current 'host' data - # record, but warn if we find it to be not unique. Note that we - # will still wind up importing both records however. - key = self.get_key(source_data) - if key in keys_seen: - log.warning("duplicate records from {}:{} for key: {}".format( - self.__class__.__module__, self.__class__.__name__, key)) - else: - keys_seen.add(key) - - # Fetch local instance, using key from host record. - instance = self.get_instance(key) - - # If we have a local instance, but its data differs from host, update it. - if instance and self.allow_update: - instance_data = self.normalize_instance(instance) - diffs = self.data_diffs(instance_data, source_data) - if diffs: - log.debug("fields '{}' differed for local data: {}, host data: {}".format( - ','.join(diffs), instance_data, source_data)) - instance = self.update_instance(instance, source_data, instance_data) - updated.append((instance, instance_data, source_data)) - if args.max_update and len(updated) >= args.max_update: - log.warning("max of {} *updated* records has been reached; stopping now".format(args.max_update)) - break - if args.max_total and (len(created) + len(updated)) >= args.max_total: - log.warning("max of {} *total changes* has been reached; stopping now".format(args.max_total)) - break - - # If we did not yet have a local instance, create it using host data. - elif not instance and self.allow_create: - instance = self.create_instance(key, source_data) - log.debug("created new {} {}: {}".format(self.model_name, key, instance)) - created.append((instance, source_data)) - if self.cached_instances is not None: - self.cached_instances[key] = {'instance': instance, 'data': self.normalize_instance(instance)} - if args.max_create and len(created) >= args.max_create: - log.warning("max of {} *created* records has been reached; stopping now".format(args.max_create)) - break - if args.max_total and (len(created) + len(updated)) >= args.max_total: - log.warning("max of {} *total changes* has been reached; stopping now".format(args.max_total)) - break - - if prog: - prog.update(i) - if prog: - prog.destroy() - - return created, updated - - def get_deletion_keys(self): - """ - Return a set of keys from the *local* data set, which are eligible for - deletion. By default this will be all keys from the local (cached) - data set. - """ - return set(self.cached_instances) - - def _import_delete(self, data, args, host_keys=None, changes=0): - """ - Import deletions for the given data set. - """ - if host_keys is None: - host_keys = set([self.get_key(rec) for rec in data]) - - deleted = [] - deleting = self.get_deletion_keys() - host_keys - count = len(deleting) - log.debug("found {} instances to delete".format(count)) - if count: - - prog = None - if self.progress: - prog = self.progress("Deleting {} data".format(self.model_name), count) - - for i, key in enumerate(sorted(deleting), 1): - - instance = self.cached_instances.pop(key)['instance'] - if self.delete_instance(instance): - deleted.append((instance, self.normalize_instance(instance))) - - if args.max_delete and len(deleted) >= args.max_delete: - log.warning("max of {} *deleted* records has been reached; stopping now".format(args.max_delete)) - break - if args.max_total and (changes + len(deleted)) >= args.max_total: - log.warning("max of {} *total changes* has been reached; stopping now".format(args.max_total)) - break - - if prog: - prog.update(i) - if prog: - prog.destroy() - - return deleted - - def delete_instance(self, instance): - """ - Process a deletion for the given instance. The default implementation - really does delete the instance from the local session, so you must - override this if you need something else to happen. - - This method must return a boolean indicating whether or not the - deletion was performed. This implies for example that you may simply - do nothing, and return ``False``, to effectively disable deletion - altogether for an importer. - """ - self.session.delete(instance) - self.session.flush() - self.session.expunge(instance) - return True - - def get_source_data(self): - """ - Return the "raw" (as-is, not normalized) data which is to be imported. - This may be any sequence-like object, which has a ``len()`` value and - responds to iteration etc. The objects contained within it may be of - any type, no assumptions are made there. (That is the job of the - :meth:`normalize_source_data()` method.) - """ - return [] - - def normalize_source_data(self): - """ - Return a normalized version of the full set of source data. Note that - this calls :meth:`get_source_data()` to obtain the initial data set, - and then normalizes each record. the normalization process may filter - out some records from the set, in which case the return value will be - smaller than the original data set. - """ - source_data = self.get_source_data() - normalized = [] - count = len(source_data) - if count == 0: - return normalized - prog = None - if self.progress: - prog = self.progress(self.normalize_progress_message, count) - for i, data in enumerate(source_data, 1): - data = self.normalize_source_record(data) - if data: - normalized.append(data) - if prog: - prog.update(i) - if prog: - prog.destroy() - return normalized - - def get_key(self, data): - """ - Return the key value for the given data record. - """ - return tuple(data[k] for k in self.key) - - def int_(self, value): - """ - Coerce ``value`` to an integer, or return ``None`` if that can't be - done cleanly. - """ - try: - return int(value) - except (TypeError, ValueError): - return None - - def prioritize_2(self, data, field): - """ - Prioritize the data values for the pair of fields implied by the given - fieldname. I.e., if only one non-empty value is present, make sure - it's in the first slot. - """ - field2 = '{}_2'.format(field) - if field in data and field2 in data: - if data[field2] and not data[field]: - data[field], data[field2] = data[field2], None - - def normalize_source_record(self, record): - """ - Normalize a source data record. Generally this is where the importer - may massage the record in any way necessary, so that its values are - more "native" and can be used for direct comparison with, and - assignment to, the target model instance. - - Note that if you override this, your method must return the data to be - imported. If your method returns ``None`` then that particular record - would be skipped and not imported. - """ - return record - - def cache_model(self, model, **kwargs): - """ - Convenience method which invokes :func:`rattail.db.cache.cache_model()` - with the given model and keyword arguments. It will provide the - ``session`` and ``progress`` parameters by default, setting them to the - importer's attributes of the same names. - """ - session = kwargs.pop('session', self.session) - kwargs.setdefault('progress', self.progress) - return cache.cache_model(session, model, **kwargs) - - def cache_instance_data(self, data=None): - """ - Cache all existing model instances as normalized data. - """ - return cache.cache_model(self.session, self.model_class, - key=self.get_cache_key, - omit_duplicates=True, - query_options=self.cache_query_options(), - normalizer=self.normalize_cache_instance, - progress=self.progress) - - def cache_query_options(self): - """ - Return a list of options to apply to the cache query, if needed. - """ - - def get_cache_key(self, instance, normalized): - """ - Get the primary model cache key for a given instance/data object. - """ - return tuple(normalized['data'].get(k) for k in self.key) - - def normalize_cache_instance(self, instance): - """ - Normalizer for cache data. This adds the instance to the cache in - addition to its normalized data. This is so that if lots of updates - are required, we don't we have to constantly fetch them. - """ - return {'instance': instance, 'data': self.normalize_instance(instance)} - - def get_instance(self, key): - """ - Must return the local object corresponding to the given key, or None. - Default behavior here will be to check the cache if one is in effect, - otherwise return the value from :meth:`get_single_instance()`. - """ - if self.cached_instances is not None: - data = self.cached_instances.get(key) - return data['instance'] if data else None - return self.get_single_instance(key) - - def get_single_instance(self, key): - """ - Must return the local object corresponding to the given key, or None. - This method should not consult the cache; that is handled within the - :meth:`get_instance()` method. - """ - query = self.session.query(self.model_class) - for i, k in enumerate(self.key): - query = query.filter(getattr(self.model_class, k) == key[i]) - - try: - return query.one() - except NoResultFound: - pass - - def normalize_instance(self, instance): - """ - Normalize a model instance. - """ - data = {} - for field in self.simple_fields: - if field in self.fields: - data[field] = getattr(instance, field) - return data - - def newval(self, data, field, value): - """ - Assign a "new" field value to the given data record. In other words - don't try to be smart about not overwriting it if the existing data - already matches etc. However the main point of this is to skip fields - which are not included in the current task. - """ - if field in self.fields: - data[field] = value - - def data_diffs(self, local_data, host_data): - """ - Find all (relevant) fields which differ between the model and host data - for a given record. - """ - diffs = [] - for field in self.fields: - if local_data[field] != host_data[field]: - diffs.append(field) - return diffs - - def create_instance(self, key, data): - instance = self.new_instance(key) - if instance: - instance = self.update_instance(instance, data) - if instance: - self.session.add(instance) - return instance - - def new_instance(self, key): - """ - Return a new model instance to correspond to the given key. - """ - instance = self.model_class() - for i, k in enumerate(self.key): - if hasattr(instance, k): - setattr(instance, k, key[i]) - return instance - - def update_instance(self, instance, data, instance_data=None): - """ - Update the given model instance with the given data. - """ - for field in self.simple_fields: - if field in self.fields: - if not instance_data or instance_data[field] != data[field]: - setattr(instance, field, data[field]) - return instance - - -class QueryDataProxy(object): - """ - Simple proxy to wrap a SQLAlchemy (or Django) query and make it sort of - behave like a normal sequence, as much as needed to make an importer happy. - """ - - def __init__(self, query): - self.query = query - - def __len__(self): - return self.query.count() - - def __iter__(self): - return iter(self.query) - - -class QueryImporter(Importer): - """ - Base class for importers whose raw external data source is a SQLAlchemy (or - Django) query. - """ - - def query(self): - """ - Must return the primary query which will define the data set. - """ - raise NotImplementedError - - def get_source_data(self, progress=None): - return QueryDataProxy(self.query()) - - -class SQLAlchemyImporter(QueryImporter): - """ - Base class for importers whose external data source is a SQLAlchemy query. - """ - host_session = None - - @property - def host_model_class(self): - """ - For default behavior, set this to a model class to be used in - generating the host (source) data query. - """ - raise NotImplementedError - - def query(self): - """ - Must return the primary query which will define the data set. Default - behavior is to leverage :attr:`host_session` and generate a query for - the class defined by :attr:`host_model_class`. - """ - return self.host_session.query(self.host_model_class) - - -class BulkPostgreSQLImporter(Importer): - """ - Base class for bulk data importers which target PostgreSQL on the local side. - """ - - def import_data(self, args, progress=None): - self._setup(args, progress) - self.open_data_buffers() - data = self.normalize_source_data() - created = self._import_create(data, args) - self.teardown() - return created - - def open_data_buffers(self): - self.data_buffer = open(self.data_path, 'wb') - - def teardown(self): - self.data_buffer.close() - - def _import_create(self, data, args): - count = len(data) - if not count: - return 0 - created = count - - prog = None - if self.progress: - prog = self.progress("Importing {} data".format(self.model_name), count) - - for i, source_data in enumerate(data, 1): - - key = self.get_key(source_data) - self.create_instance(key, source_data) - if args.max_create and i >= args.max_create: - log.warning("max of {} *created* records has been reached; stopping now".format(args.max_create)) - created = i - break - - if prog: - prog.update(i) - if prog: - prog.destroy() - - self.commit_create() - return created - - def commit_create(self): - log.info("copying {} data from buffer to PostgreSQL".format(self.model_name)) - self.seek_data_buffers() - cursor = self.session.connection().connection.cursor() - cursor.copy_from(self.data_buffer, self.model_table.name, columns=self.fields) - log.debug("PostgreSQL data copy completed") - - def seek_data_buffers(self): - self.data_buffer.close() - self.data_buffer = open(self.data_path, 'rb') - - def create_instance(self, key, data): - self.prep_data_for_postgres(data) - self.data_buffer.write('{}\n'.format('\t'.join([data[field] for field in self.fields]))) - - def prep_data_for_postgres(self, data): - for key, value in data.iteritems(): - if value is None: - value = '\\N' - elif value is True: - value = 't' - elif value is False: - value = 'f' - elif isinstance(value, datetime.datetime): - value = make_utc(value) - elif isinstance(value, basestring): - value = value.replace('\\', '\\\\') - value = value.replace('\r\n', '\n') - value = value.replace('\r', '\\r') - value = value.replace('\n', '\\n') - data[key] = unicode(value) diff --git a/rattail/db/newimporting/model.py b/rattail/db/newimporting/model.py deleted file mode 100644 index c06460d3..00000000 --- a/rattail/db/newimporting/model.py +++ /dev/null @@ -1,1575 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# -# Rattail -- Retail Software Framework -# Copyright © 2010-2016 Lance Edgar -# -# This file is part of Rattail. -# -# Rattail is free software: you can redistribute it and/or modify it under the -# terms of the GNU Affero General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Rattail. If not, see . -# -################################################################################ -""" -Rattail Model Importers -""" - -from __future__ import unicode_literals, absolute_import - -import logging - -from sqlalchemy import orm - -from rattail.db import model, cache, auth -from rattail.db.newimporting import Importer -from rattail.db.util import normalize_phone_number, format_phone_number - - -log = logging.getLogger(__name__) - - -class PersonImporter(Importer): - """ - Person data importer. - """ - model_class = model.Person - - -class PersonEmailAddressImporter(Importer): - """ - Person email address data importer. - """ - model_class = model.PersonEmailAddress - - @property - def supported_fields(self): - return self.simple_fields + [ - 'preferred', - ] - - def normalize_instance(self, email): - data = super(PersonEmailAddressImporter, self).normalize_instance(email) - data['preferred'] = email.preference == 1 - return data - - def update_instance(self, email, data, inst_data=None): - email = super(PersonEmailAddressImporter, self).update_instance(email, data, inst_data) - if 'preferred' in self.fields: - if data['preferred']: - if email.preference != 1: - person = email.person - if not person: - person = self.session.query(model.Person).get(email.parent_uuid) - assert person, "hm: " + email.parent_uuid - if email in person.emails: - person.emails.remove(email) - person.emails.insert(0, email) - person.emails.reorder() - else: - if email.preference == 1: - person = email.person - if not person: - person = self.session.query(model.Person).get(email.parent_uuid) - assert person, "hm: " + email.parent_uuid - if len(person.emails) > 1: - person.emails.remove(email) - person.emails.append(email) - person.emails.reorder() - - # If this is a new record, we may still need to establish its preference. - if email.preference is None: - person = email.person - if not person: - person = self.session.query(model.Person).get(email.parent_uuid) - assert person, "hm: " + email.parent_uuid - if email not in person.emails: - person.emails.append(email) - person.emails.reorder() - - return email - - -class PersonPhoneNumberImporter(Importer): - """ - Person phone number data importer. - """ - model_class = model.PersonPhoneNumber - - @property - def supported_fields(self): - return self.simple_fields + [ - 'normalized_number', - 'preferred', - ] - - def normalize_number(self, number): - return normalize_phone_number(number) - - def format_number(self, number): - return format_phone_number(number) - - def normalize_instance(self, phone): - data = super(PersonPhoneNumberImporter, self).normalize_instance(phone) - if 'normalized_number' in self.fields: - data['normalized_number'] = self.normalize_number(phone.number) - if 'preferred' in self.fields: - data['preferred'] = phone.preference == 1 - return data - - def update_instance(self, phone, data, inst_data=None): - phone = super(PersonPhoneNumberImporter, self).update_instance(phone, data, inst_data) - if 'preferred' in self.fields: - if data['preferred']: - if phone.preference != 1: - person = phone.person - if not person: - person = self.session.query(model.Person).get(phone.parent_uuid) - assert person, "hm: " + phone.parent_uuid - if phone in person.phones: - person.phones.remove(phone) - person.phones.insert(0, phone) - person.phones.reorder() - else: - if phone.preference == 1: - person = phone.person - if not person: - person = self.session.query(model.Person).get(phone.parent_uuid) - assert person, "hm: " + phone.parent_uuid - if len(person.phones) > 1: - person.phones.remove(phone) - person.phones.append(phone) - person.phones.reorder() - - # If this is a new record, we may still need to establish its preference. - if phone.preference is None: - person = phone.person - if not person: - person = self.session.query(model.Person).get(phone.parent_uuid) - assert person, "hm: " + phone.parent_uuid - if phone not in person.phones: - person.phones.append(phone) - person.phones.reorder() - - return phone - - -class PersonMailingAddressImporter(Importer): - """ - Person mailing address data importer. - """ - model_class = model.PersonMailingAddress - - -class UserImporter(Importer): - """ - User data importer. - """ - model_class = model.User - - @property - def supported_fields(self): - return self.simple_fields + [ - 'admin', - ] - - def setup(self): - self.admin = auth.administrator_role(self.session) - - def normalize_instance(self, user): - data = super(UserImporter, self).normalize_instance(user) - if 'admin' in self.fields: - data['admin'] = self.admin in user.roles - return data - - def update_instance(self, user, data, inst_data=None): - user = super(UserImporter, self).update_instance(user, data, inst_data) - if 'admin' in self.fields: - if data['admin']: - if self.admin not in user.roles: - user.roles.append(self.admin) - else: - if self.admin in user.roles: - user.roles.remove(self.admin) - return user - - -class MessageImporter(Importer): - """ - User message data importer. - """ - model_class = model.Message - - -class MessageRecipientImporter(Importer): - """ - User message recipient data importer. - """ - model_class = model.MessageRecipient - - -class StoreImporter(Importer): - """ - Store data importer. - """ - model_class = model.Store - - @property - def supported_fields(self): - return self.simple_fields + [ - 'phone_number', - 'fax_number', - ] - - def cache_query_options(self): - if 'phone_number' in self.fields or 'fax_number' in self.fields: - return [orm.joinedload(model.Store.phones)] - - def normalize_instance(self, store): - data = super(StoreImporter, self).normalize_instance(store) - - if 'phone_number' in self.fields: - data['phone_number'] = None - for phone in store.phones: - if phone.type == 'Voice': - data['phone_number'] = phone.number - break - - if 'fax_number' in self.fields: - data['fax_number'] = None - for phone in store.phones: - if phone.type == 'Fax': - data['fax_number'] = phone.number - break - - return data - - def update_instance(self, store, data, inst_data=None): - store = super(StoreImporter, self).update_instance(store, data, inst_data) - - if 'phone_number' in self.fields: - number = data['phone_number'] or None - if number: - found = False - for phone in store.phones: - if phone.type == 'Voice': - if phone.number != number: - phone.number = number - found = True - break - if not found: - store.add_phone_number(number, type='Voice') - else: - for phone in list(store.phones): - if phone.type == 'Voice': - store.phones.remove(phone) - - if 'fax_number' in self.fields: - number = data['fax_number'] or None - if number: - found = False - for phone in store.phones: - if phone.type == 'Fax': - if phone.number != number: - phone.number = number - found = True - break - if not found: - store.add_phone_number(number, type='Fax') - else: - for phone in list(store.phones): - if phone.type == 'Fax': - store.phones.remove(phone) - - return store - - -class StorePhoneNumberImporter(Importer): - """ - Store phone data importer. - """ - model_class = model.StorePhoneNumber - - -class EmployeeImporter(Importer): - """ - Employee data importer. - """ - model_class = model.Employee - - @property - def supported_fields(self): - return self.simple_fields + [ - 'customer_id', - 'first_name', - 'last_name', - 'full_name', - 'phone_number', - 'phone_number_2', - 'email_address', - ] - - def setup(self): - if 'customer_id' in self.fields: - self.customers = cache.cache_model(self.session, model.Customer, key='id', - progress=self.progress) - - def cache_query_options(self): - return [ - orm.joinedload(model.Employee.person).joinedload(model.Person._customers), - orm.joinedload(model.Employee.phones), - orm.joinedload(model.Employee.email), - ] - - def normalize_instance(self, employee): - data = super(EmployeeImporter, self).normalize_instance(employee) - - if set(self.fields) | set(['customer_id', 'first_name', 'last_name', 'full_name']): - if not employee.person: - self.session.flush() - assert employee.person - person = employee.person - self.newval(data, 'first_name', person.first_name) - self.newval(data, 'last_name', person.last_name) - self.newval(data, 'full_name', person.display_name) - if 'customer_id' in self.fields: - customer = person.customers[0] if person.customers else None - data['customer_id'] = customer.id if customer else None - - if 'phone_number' in self.fields: - data['phone_number'] = None - for phone in employee.phones: - if phone.type == 'Home': - data['phone_number'] = phone.number - break - - if 'phone_number_2' in self.fields: - data['phone_number_2'] = None - first = False - for phone in employee.phones: - if phone.type == 'Home': - if first: - data['phone_number_2'] = phone.number - break - first = True - - if 'email_address' in self.fields: - email = employee.email - data['email_address'] = email.address if email else None - - return data - - def update_instance(self, employee, data, inst_data=None): - employee = super(EmployeeImporter, self).update_instance(employee, data, inst_data) - person = employee.person - - if 'first_name' in self.fields: - employee.first_name = data['first_name'] - if 'last_name' in self.fields: - employee.last_name = data['last_name'] - - if 'full_name' in self.fields: - if person.display_name != data['full_name']: - person.display_name = data['full_name'] - - if 'customer_id' in self.fields: - id_ = data['customer_id'] - if id_: - customer = self.customers.get(id_) - if not customer: - customer = model.Customer() - customer.id = id_ - customer.name = employee.display_name - self.session.add(customer) - self.customers[customer.id] = customer - if person not in customer.people: - customer.people.append(person) - else: - for customer in list(person.customers): - if len(customer.people) > 1: - if person in customer.people: - customer.people.remove(person) - - if 'phone_number' in self.fields: - number = data['phone_number'] - if number: - found = False - for phone in employee.phones: - if phone.type == 'Home': - if phone.number != number: - phone.number = number - found = True - break - if not found: - employee.add_phone_number(number, type='Home') - else: - for phone in list(employee.phones): - if phone.type == 'Home': - employee.phones.remove(phone) - - if 'phone_number_2' in self.fields: - number = data['phone_number_2'] - if number: - found = False - first = False - for phone in employee.phones: - if phone.type == 'Home': - if first: - if phone.number != number: - phone.number = number - found = True - break - first = True - if not found: - employee.add_phone_number(number, type='Home') - else: - first = False - for phone in list(employee.phones): - if phone.type == 'Home': - if first: - employee.phones.remove(phone) - break - first = True - - if 'email_address' in self.fields: - address = data['email_address'] - if address: - if employee.email: - if employee.email.address != address: - employee.email.address = address - else: - employee.add_email_address(address) - else: - if len(employee.emails): - del employee.emails[:] - - return employee - - -class EmployeeStoreImporter(Importer): - """ - Employee/store data importer. - """ - model_class = model.EmployeeStore - - -class EmployeeDepartmentImporter(Importer): - """ - Employee/department data importer. - """ - model_class = model.EmployeeDepartment - - -class EmployeeEmailAddressImporter(Importer): - """ - Employee email data importer. - """ - model_class = model.EmployeeEmailAddress - - -class EmployeePhoneNumberImporter(Importer): - """ - Employee phone data importer. - """ - model_class = model.EmployeePhoneNumber - - -class ScheduledShiftImporter(Importer): - """ - Imports employee scheduled shifts. - """ - model_class = model.ScheduledShift - - -class WorkedShiftImporter(Importer): - """ - Imports shifts worked by employees. - """ - model_class = model.WorkedShift - - -class CustomerImporter(Importer): - """ - Customer data importer. - """ - model_class = model.Customer - - @property - def supported_fields(self): - return self.simple_fields + [ - 'first_name', - 'last_name', - 'phone_number', - 'phone_number_2', - 'email_address', - 'group_id', - 'group_id_2', - ] - - def setup(self): - if 'group_id' in self.fields or 'group_id_2' in self.fields: - self.groups = cache.cache_model(self.session, model.CustomerGroup, key='id', - progress=self.progress) - - def cache_query_options(self): - options = [] - if 'first_name' in self.fields or 'last_name' in self.fields: - options.append(orm.joinedload(model.Customer._people).joinedload(model.CustomerPerson.person)) - if 'phone_number' in self.fields or 'phone_number_2' in self.fields: - options.append(orm.joinedload(model.Customer.phones)) - if 'email_address' in self.fields: - options.append(orm.joinedload(model.Customer.email)) - if 'group_id' in self.fields or 'group_id_2' in self.fields: - options.append(orm.joinedload_all(model.Customer._groups, model.CustomerGroupAssignment.group)) - return options - - def normalize_instance(self, customer): - data = super(CustomerImporter, self).normalize_instance(customer) - if 'first_name' in self.fields or 'last_name' in self.fields: - person = customer.people[0] if customer.people else None - self.newval(data, 'first_name', person.first_name if person else None) - self.newval(data, 'last_name', person.last_name if person else None) - - if 'phone_number' in self.fields or 'phone_number_2' in self.fields: - phones = filter(lambda p: p.type == 'Voice', customer.phones) - self.newval(data, 'phone_number', phones[0].number if phones else None) - self.newval(data, 'phone_number_2', phones[1].number if len(phones) > 1 else None) - - if 'email_address' in self.fields: - email = customer.email - data['email_address'] = email.address if email else None - - if 'group_id' in self.fields: - group = customer.groups[0] if customer.groups else None - data['group_id'] = group.id if group else None - - if 'group_id_2' in self.fields: - group = customer.groups[1] if customer.groups and len(customer.groups) > 1 else None - data['group_id_2'] = group.id if group else None - - return data - - def update_instance(self, customer, data, instance_data=None): - customer = super(CustomerImporter, self).update_instance(customer, data, instance_data) - - if 'first_name' in self.fields or 'last_name' in self.fields: - first_name = data['first_name'] - last_name = data['last_name'] - if not customer.people: - customer.people.append(model.Person()) - person = customer.people[0] - if 'first_name' in self.fields and person.first_name != first_name: - person.first_name = first_name - if 'last_name' in self.fields and person.last_name != last_name: - person.last_name = last_name - - if 'phone_number' in self.fields: - phones = filter(lambda p: p.type == 'Voice', customer.phones) - number = data['phone_number'] - if number: - if phones: - phone = phones[0] - if phone.number != number: - phone.number = number - else: - customer.add_phone_number(number, type='Voice') - else: - for phone in phones: - customer.phones.remove(phone) - - if 'phone_number_2' in self.fields: - phones = filter(lambda p: p.type == 'Voice', customer.phones) - number = data['phone_number_2'] - if number: - if len(phones) > 1: - phone = phones[1] - if phone.number != number: - phone.number = number - else: - customer.add_phone_number(number, 'Voice') - else: - for phone in phones[1:]: - customer.phones.remove(phone) - - if 'email_address' in self.fields: - address = data['email_address'] - if address: - if customer.email: - if customer.email.address != address: - customer.email.address = address - else: - customer.add_email_address(address) - else: - if len(customer.emails): - del customer.emails[:] - - if 'group_id' in self.fields: - id_ = data['group_id'] - if id_: - group = self.groups.get(id_) - if not group: - group = model.CustomerGroup() - group.id = id_ - group.name = "(auto-created)" - self.session.add(group) - self.groups[group.id] = group - if group in customer.groups: - if group is not customer.groups[0]: - customer.groups.remove(group) - customer.groups.insert(0, group) - else: - customer.groups.insert(0, group) - else: - if customer.groups: - del customer.groups[:] - - if 'group_id_2' in self.fields: - id_ = data['group_id_2'] - if id_: - group = self.groups.get(id_) - if not group: - group = model.CustomerGroup() - group.id_ = id_ - group.name = "(auto-created)" - self.session.add(group) - self.groups[group.id] = group - if group in customer.groups: - if len(customer.groups) > 1: - if group is not customer.groups[1]: - customer.groups.remove(group) - customer.groups.insert(1, group) - else: - if len(customer.groups) > 1: - customer.groups.insert(1, group) - else: - customer.groups.append(group) - else: - if len(customer.groups) > 1: - del customer.groups[1:] - - return customer - - -class CustomerGroupImporter(Importer): - """ - CustomerGroup data importer. - """ - model_class = model.CustomerGroup - - -class CustomerGroupAssignmentImporter(Importer): - """ - CustomerGroupAssignment data importer. - """ - model_class = model.CustomerGroupAssignment - - -class CustomerPersonImporter(Importer): - """ - CustomerPerson data importer. - """ - model_class = model.CustomerPerson - - -class CustomerEmailAddressImporter(Importer): - """ - Customer email address data importer. - """ - model_class = model.CustomerEmailAddress - - -class CustomerPhoneNumberImporter(Importer): - """ - Customer phone number data importer. - """ - model_class = model.CustomerPhoneNumber - - -class VendorImporter(Importer): - """ - Vendor data importer. - """ - model_class = model.Vendor - - phone_fields = [ - 'phone_number', - 'phone_number_2', - 'fax_number', - 'fax_number_2', - ] - contact_fields = [ - 'contact_name', - 'contact_name_2', - ] - complex_fields = [ - 'email_address', - ] - - @property - def supported_fields(self): - return (self.simple_fields + self.phone_fields + self.contact_fields - + self.complex_fields) - - def cache_query_options(self): - options = [] - for field in self.phone_fields: - if field in self.fields: - options.append(orm.joinedload(model.Vendor.phones)) - break - for field in self.contact_fields: - if field in self.fields: - options.append(orm.joinedload(model.Vendor._contacts)) - break - if 'email_address' in self.fields: - options.append(orm.joinedload(model.Vendor.email)) - return options - - def normalize_instance(self, vendor): - data = super(VendorImporter, self).normalize_instance(vendor) - - if 'phone_number' in self.fields: - data['phone_number'] = None - for phone in vendor.phones: - if phone.type == 'Voice': - data['phone_number'] = phone.number - break - - if 'phone_number_2' in self.fields: - data['phone_number_2'] = None - first = False - for phone in vendor.phones: - if phone.type == 'Voice': - if first: - data['phone_number_2'] = phone.number - break - first = True - - if 'fax_number' in self.fields: - data['fax_number'] = None - for phone in vendor.phones: - if phone.type == 'Fax': - data['fax_number'] = phone.number - break - - if 'fax_number_2' in self.fields: - data['fax_number_2'] = None - first = False - for phone in vendor.phones: - if phone.type == 'Fax': - if first: - data['fax_number_2'] = phone.number - break - first = True - - if 'contact_name' in self.fields: - contact = vendor.contact - data['contact_name'] = contact.display_name if contact else None - - if 'contact_name_2' in self.fields: - contact = vendor.contacts[1] if len(vendor.contacts) > 1 else None - data['contact_name_2'] = contact.display_name if contact else None - - if 'email_address' in self.fields: - email = vendor.email - data['email_address'] = email.address if email else None - - return data - - def update_instance(self, vendor, data, inst_data=None): - vendor = super(VendorImporter, self).update_instance(vendor, data, inst_data) - - if 'phone_number' in self.fields: - number = data['phone_number'] or None - if number: - found = False - for phone in vendor.phones: - if phone.type == 'Voice': - if phone.number != number: - phone.number = number - found = True - break - if not found: - vendor.add_phone_number(number, type='Voice') - else: - for phone in list(vendor.phones): - if phone.type == 'Voice': - vendor.phones.remove(phone) - - if 'phone_number_2' in self.fields: - number = data['phone_number_2'] or None - if number: - found = False - first = False - for phone in vendor.phones: - if phone.type == 'Voice': - if first: - if phone.number != number: - phone.number = number - found = True - break - first = True - if not found: - vendor.add_phone_number(number, type='Voice') - else: - first = False - for phone in list(vendor.phones): - if phone.type == 'Voice': - if first: - vendor.phones.remove(phone) - break - first = True - - if 'fax_number' in self.fields: - number = data['fax_number'] or None - if number: - found = False - for phone in vendor.phones: - if phone.type == 'Fax': - if phone.number != number: - phone.number = number - found = True - break - if not found: - vendor.add_phone_number(number, type='Fax') - else: - for phone in list(vendor.phones): - if phone.type == 'Fax': - vendor.phones.remove(phone) - - if 'fax_number_2' in self.fields: - number = data['fax_number_2'] or None - if number: - found = False - first = False - for phone in vendor.phones: - if phone.type == 'Fax': - if first: - if phone.number != number: - phone.number = number - found = True - break - first = True - if not found: - vendor.add_phone_number(number, type='Fax') - else: - first = False - for phone in list(vendor.phones): - if phone.type == 'Fax': - if first: - vendor.phones.remove(phone) - break - first = True - - if 'contact_name' in self.fields: - if data['contact_name']: - contact = vendor.contact - if not contact: - contact = model.Person() - self.session.add(contact) - vendor.contacts.append(contact) - contact.display_name = data['contact_name'] - else: - if len(vendor.contacts): - del vendor.contacts[:] - - if 'contact_name_2' in self.fields: - if data['contact_name_2']: - contact = vendor.contacts[1] if len(vendor.contacts) > 1 else None - if not contact: - contact = model.Person() - self.session.add(contact) - vendor.contacts.append(contact) - contact.display_name = data['contact_name_2'] - else: - if len(vendor.contacts) > 1: - del vendor.contacts[1:] - - if 'email_address' in self.fields: - address = data['email_address'] or None - if address: - if vendor.email: - if vendor.email.address != address: - vendor.email.address = address - else: - vendor.add_email_address(address) - else: - if len(vendor.emails): - del vendor.emails[:] - - return vendor - - -class VendorEmailAddressImporter(Importer): - """ - Vendor email data importer. - """ - model_class = model.VendorEmailAddress - - -class VendorPhoneNumberImporter(Importer): - """ - Vendor phone data importer. - """ - model_class = model.VendorPhoneNumber - - -class VendorContactImporter(Importer): - """ - Vendor contact data importer. - """ - model_class = model.VendorContact - - -class DepartmentImporter(Importer): - """ - Department data importer. - """ - model_class = model.Department - - -class SubdepartmentImporter(Importer): - """ - Subdepartment data importer. - """ - model_class = model.Subdepartment - - @property - def supported_fields(self): - return self.simple_fields + [ - 'department_number', - ] - - def setup(self): - self.departments = cache.cache_model(self.session, model.Department, key='number', - progress=self.progress) - - def cache_query_options(self): - if 'department_number' in self.fields: - return [orm.joinedload(model.Subdepartment.department)] - - def normalize_instance(self, subdepartment): - data = super(SubdepartmentImporter, self).normalize_instance(subdepartment) - if 'department_number' in self.fields: - dept = subdepartment.department - data['department_number'] = dept.number if dept else None - return data - - def update_instance(self, subdepartment, data, inst_data=None): - subdepartment = super(SubdepartmentImporter, self).update_instance(subdepartment, data, inst_data) - if 'department_number' in self.fields: - dept = self.departments.get(data['department_number']) - if not dept: - dept = model.Department() - dept.number = data['department_number'] - dept.name = "(auto-created)" - self.session.add(dept) - self.departments[dept.number] = dept - subdepartment.department = dept - return subdepartment - - -class CategoryImporter(Importer): - """ - Category data importer. - """ - model_class = model.Category - - -class FamilyImporter(Importer): - """ - Family data importer. - """ - model_class = model.Family - - -class ReportCodeImporter(Importer): - """ - ReportCode data importer. - """ - model_class = model.ReportCode - - -class DepositLinkImporter(Importer): - """ - Deposit link data importer. - """ - model_class = model.DepositLink - - -class TaxImporter(Importer): - """ - Tax data importer. - """ - model_class = model.Tax - - -class BrandImporter(Importer): - """ - Brand data importer. - """ - model_class = model.Brand - - -class ProductImporter(Importer): - """ - Data importer for :class:`rattail.db.model.Product`. - """ - model_class = model.Product - - regular_price_fields = [ - 'regular_price_price', - 'regular_price_multiple', - 'regular_price_pack_price', - 'regular_price_pack_multiple', - 'regular_price_type', - 'regular_price_level', - 'regular_price_starts', - 'regular_price_ends', - ] - sale_price_fields = [ - 'sale_price_price', - 'sale_price_multiple', - 'sale_price_pack_price', - 'sale_price_pack_multiple', - 'sale_price_type', - 'sale_price_level', - 'sale_price_starts', - 'sale_price_ends', - ] - - @property - def supported_fields(self): - return self.simple_fields + self.regular_price_fields + self.sale_price_fields + [ - 'brand_name', - 'department_number', - 'subdepartment_number', - 'category_number', - 'family_code', - 'report_code', - 'deposit_link_code', - 'tax_code', - 'vendor_id', - 'vendor_item_code', - 'vendor_case_cost', - ] - - def setup(self): - if 'brand_name' in self.fields: - self.brands = cache.cache_model(self.session, model.Brand, key='name', progress=self.progress) - if 'department_number' in self.fields: - self.departments = cache.cache_model(self.session, model.Department, key='number', progress=self.progress) - if 'subdepartment_number' in self.fields: - self.subdepartments = cache.cache_model(self.session, model.Subdepartment, key='number', progress=self.progress) - if 'category_number' in self.fields: - self.categories = cache.cache_model(self.session, model.Category, key='number', progress=self.progress) - if 'family_code' in self.fields: - self.families = cache.cache_model(self.session, model.Family, key='code', progress=self.progress) - if 'report_code' in self.fields: - self.reportcodes = cache.cache_model(self.session, model.ReportCode, key='code', progress=self.progress) - if 'deposit_link_code' in self.fields: - self.depositlinks = cache.cache_model(self.session, model.DepositLink, key='code', progress=self.progress) - if 'tax_code' in self.fields: - self.taxes = cache.cache_model(self.session, model.Tax, key='code', progress=self.progress) - if 'vendor_id' in self.fields: - self.vendors = cache.cache_model(self.session, model.Vendor, key='id', progress=self.progress) - - def cache_query_options(self): - options = [] - if 'brand_name' in self.fields: - options.append(orm.joinedload(model.Product.brand)) - if 'department_number' in self.fields: - options.append(orm.joinedload(model.Product.department)) - if 'subdepartment_number' in self.fields: - options.append(orm.joinedload(model.Product.subdepartment)) - if 'category_number' in self.fields: - options.append(orm.joinedload(model.Product.category)) - if 'family_code' in self.fields: - options.append(orm.joinedload(model.Product.family)) - if 'report_code' in self.fields: - options.append(orm.joinedload(model.Product.report_code)) - if 'deposit_link_code' in self.fields: - options.append(orm.joinedload(model.Product.deposit_link)) - if 'tax_code' in self.fields: - options.append(orm.joinedload(model.Product.tax)) - joined_prices = False - for field in self.regular_price_fields: - if field in self.fields: - options.append(orm.joinedload(model.Product.prices)) - options.append(orm.joinedload(model.Product.regular_price)) - joined_prices = True - break - for field in self.sale_price_fields: - if field in self.fields: - if not joined_prices: - options.append(orm.joinedload(model.Product.prices)) - options.append(orm.joinedload(model.Product.current_price)) - break - if set(self.fields) | set(['vendor_id', 'vendor_item_code', 'vendor_case_cost']): - options.append(orm.joinedload(model.Product.cost)) - # options.append(orm.joinedload(model.Product.costs)) - return options - - def normalize_instance(self, product): - data = super(ProductImporter, self).normalize_instance(product) - - if 'brand_name' in self.fields: - data['brand_name'] = product.brand.name if product.brand else None - if 'department_number' in self.fields: - data['department_number'] = product.department.number if product.department else None - if 'subdepartment_number' in self.fields: - data['subdepartment_number'] = product.subdepartment.number if product.subdepartment else None - if 'category_number' in self.fields: - data['category_number'] = product.category.number if product.category else None - if 'family_code' in self.fields: - data['family_code'] = product.family.code if product.family else None - if 'report_code' in self.fields: - data['report_code'] = product.report_code.code if product.report_code else None - if 'deposit_link_code' in self.fields: - data['deposit_link_code'] = product.deposit_link.code if product.deposit_link else None - if 'tax_code' in self.fields: - data['tax_code'] = product.tax.code if product.tax else None - - for field in self.regular_price_fields: - if field in self.fields: - price = product.regular_price - self.newval(data, 'regular_price_price', price.price if price else None) - self.newval(data, 'regular_price_multiple', price.multiple if price else None) - self.newval(data, 'regular_price_pack_price', price.pack_price if price else None) - self.newval(data, 'regular_price_pack_multiple', price.pack_multiple if price else None) - self.newval(data, 'regular_price_type', price.type if price else None) - self.newval(data, 'regular_price_level', price.level if price else None) - self.newval(data, 'regular_price_starts', price.starts if price else None) - self.newval(data, 'regular_price_ends', price.ends if price else None) - break - - for field in self.sale_price_fields: - if field in self.fields: - price = product.current_price - self.newval(data, 'sale_price_price', price.price if price else None) - self.newval(data, 'sale_price_multiple', price.multiple if price else None) - self.newval(data, 'sale_price_pack_price', price.pack_price if price else None) - self.newval(data, 'sale_price_pack_multiple', price.pack_multiple if price else None) - self.newval(data, 'sale_price_type', price.type if price else None) - self.newval(data, 'sale_price_level', price.level if price else None) - self.newval(data, 'sale_price_starts', price.starts if price else None) - self.newval(data, 'sale_price_ends', price.ends if price else None) - break - - if set(self.fields) | set(['vendor_id', 'vendor_item_code', 'vendor_case_cost']): - cost = product.cost - self.newval(data, 'vendor_id', cost.vendor.id if cost else None) - self.newval(data, 'vendor_item_code', cost.code if cost else None) - self.newval(data, 'vendor_case_cost', cost.case_cost if cost else None) - - return data - - def update_instance(self, product, data, inst_data=None): - product = super(ProductImporter, self).update_instance(product, data, inst_data) - - if 'brand_name' in self.fields: - name = data['brand_name'] - if name: - brand = self.brands.get(name) - if not brand: - brand = model.Brand() - brand.name = name - self.session.add(brand) - self.brands[brand.name] = brand - product.brand = brand - else: - if product.brand: - product.brand = None - - if 'department_number' in self.fields: - number = data['department_number'] - if number: - dept = self.departments.get(number) - if not dept: - dept = model.Department() - dept.number = number - dept.name = "(auto-created)" - self.session.add(dept) - self.departments[dept.number] = dept - product.department = dept - else: - if product.department: - product.department = None - - if 'subdepartment_number' in self.fields: - number = data['subdepartment_number'] - if number: - sub = self.subdepartments.get(number) - if not sub: - sub = model.Subdepartment() - sub.number = number - sub.name = "(auto-created)" - self.session.add(sub) - self.subdepartments[number] = sub - product.subdepartment = sub - else: - if product.subdepartment: - product.subdepartment = None - - if 'category_number' in self.fields: - number = data['category_number'] - if number: - cat = self.categories.get(number) - if not cat: - cat = model.Category() - cat.number = number - cat.name = "(auto-created)" - self.session.add(cat) - self.categories[number] = cat - product.category = cat - else: - if product.category: - product.category = None - - if 'family_code' in self.fields: - code = data['family_code'] - if code: - family = self.families.get(code) - if not family: - family = model.Family() - family.code = code - family.name = "(auto-created)" - self.session.add(family) - self.families[family.code] = family - product.family = family - else: - if product.family: - product.family = None - - if 'report_code' in self.fields: - code = data['report_code'] - if code: - rc = self.reportcodes.get(code) - if not rc: - rc = model.ReportCode() - rc.code = code - rc.name = "(auto-created)" - self.session.add(rc) - self.reportcodes[rc.code] = rc - product.report_code = rc - else: - if product.report_code: - product.report_code = None - - if 'deposit_link_code' in self.fields: - code = data['deposit_link_code'] - if code: - link = self.depositlinks.get(code) - if not link: - link = model.DepositLink() - link.code = code - link.description = "(auto-created)" - self.session.add(link) - self.depositlinks[link.code] = link - product.deposit_link = link - else: - if product.deposit_link: - product.deposit_link = None - - if 'tax_code' in self.fields: - code = data['tax_code'] - if code: - tax = self.taxes.get(code) - if not tax: - tax = model.Tax() - tax.code = code - tax.description = "(auto-created)" - tax.rate = 0 - self.session.add(tax) - self.taxes[tax.code] = tax - product.tax = tax - elif product.tax: - product.tax = None - - create = False - delete = False - for field in self.regular_price_fields: - if field in self.fields: - delete = True - if data[field] is not None: - create = True - break - if create: - price = product.regular_price - if not price: - price = model.ProductPrice() - product.prices.append(price) - product.regular_price = price - if 'regular_price_price' in self.fields: - price.price = data['regular_price_price'] - if 'regular_price_multiple' in self.fields: - price.multiple = data['regular_price_multiple'] - if 'regular_price_pack_price' in self.fields: - price.pack_price = data['regular_price_pack_price'] - if 'regular_price_pack_multiple' in self.fields: - price.pack_multiple = data['regular_price_pack_multiple'] - if 'regular_price_type' in self.fields: - price.type = data['regular_price_type'] - if 'regular_price_level' in self.fields: - price.level = data['regular_price_level'] - if 'regular_price_starts' in self.fields: - price.starts = data['regular_price_starts'] - if 'regular_price_ends' in self.fields: - price.ends = data['regular_price_ends'] - elif delete: - if product.regular_price: - product.regular_price = None - - create = False - delete = False - for field in self.sale_price_fields: - if field in self.fields: - delete = True - if data[field]: - create = True - break - if create: - price = product.current_price - if not price: - price = model.ProductPrice() - product.prices.append(price) - product.current_price = price - if 'sale_price_price' in self.fields: - price.price = data['sale_price_price'] - if 'sale_price_multiple' in self.fields: - price.multiple = data['sale_price_multiple'] - if 'sale_price_pack_price' in self.fields: - price.pack_price = data['sale_price_pack_price'] - if 'sale_price_pack_multiple' in self.fields: - price.pack_multiple = data['sale_price_pack_multiple'] - if 'sale_price_type' in self.fields: - price.type = data['sale_price_type'] - if 'sale_price_level' in self.fields: - price.level = data['sale_price_level'] - if 'sale_price_starts' in self.fields: - price.starts = data['sale_price_starts'] - if 'sale_price_ends' in self.fields: - price.ends = data['sale_price_ends'] - elif delete: - if product.current_price: - product.current_price = None - - if 'vendor_id' in self.fields: - id_ = data['vendor_id'] - if id_: - vendor = self.vendors.get(id_) - if not vendor: - vendor = model.Vendor() - vendor.id = id_ - vendor.name = "(auto-created)" - self.session.add(vendor) - self.vendors[id_] = vendor - if product.cost: - if product.cost.vendor is not vendor: - cost = product.cost_for_vendor(vendor) - if not cost: - cost = model.ProductCost() - cost.vendor = vendor - product.costs.insert(0, cost) - else: - cost = model.ProductCost() - cost.vendor = vendor - product.costs.append(cost) - # TODO: This seems heavy-handed, but also seems necessary - # to populate the `Product.cost` relationship... - self.session.add(product) - self.session.flush() - self.session.refresh(product) - else: - if product.cost: - del product.costs[:] - - if 'vendor_item_code' in self.fields: - code = data['vendor_item_code'] - if data.get('vendor_id'): - if product.cost: - product.cost.code = code - else: - log.warning("product has no cost, so can't set vendor_item_code: {0}".format(product)) - - if 'vendor_case_cost' in self.fields: - cost = data['vendor_case_cost'] - if data.get('vendor_id'): - if product.cost: - product.cost.case_cost = cost - else: - log.warning("product has no cost, so can't set vendor_case_cost: {0}".format(product)) - - return product - - -class ProductCodeImporter(Importer): - """ - Data importer for :class:`rattail.db.model.ProductCode`. - """ - model_class = model.ProductCode - - @property - def supported_fields(self): - return self.simple_fields + [ - 'product_upc', - 'primary', - ] - - def setup(self): - if 'product_upc' in self.fields: - self.products = self.cache_model(model.Product, key='upc') - - def cache_query_options(self): - if 'product_upc' in self.fields: - return [orm.joinedload(model.ProductCode.product)] - - def normalize_instance(self, code): - data = super(ProductCodeImporter, self).normalize_instance(code) - if 'product_upc' in self.fields: - data['product_upc'] = code.product.upc - self.newval(data, 'primary', code.ordinal == 1) - return data - - def new_instance(self, key): - code = super(ProductCodeImporter, self).new_instance(key) - if 'product_upc' in self.key: - i = list(self.key).index('product_upc') - product = self.products[key[i]] - product._codes.append(code) - return code - - def update_instance(self, code, data, instance_data=None): - code = super(ProductCodeImporter, self).update_instance(code, data, instance_data) - - if 'product_upc' in self.fields and 'product_uuid' not in self.fields: - upc = data['product_upc'] - assert upc, "Source data has no product_upc value: {0}".format(repr(data)) - product = self.products.get(upc) - if not product: - product = model.Product() - product.upc = upc - product.description = "(auto-created)" - self.session.add(product) - self.products[product.upc] = product - product._codes.append(code) - else: - if code not in product._codes: - product._codes.append(code) - - if 'primary' in self.fields: - if data['primary']: - if code.ordinal != 1: - product = code.product - product._codes.remove(code) - product._codes.insert(0, code) - product._codes.reorder() - elif code.ordinal == 1: - product = code.product - if len(product._codes) > 1: - product._codes.remove(code) - product._codes.append(code) - product._codes.reorder() - - return code - - -class ProductCostImporter(Importer): - """ - Data importer for :class:`rattail.db.model.ProductCost`. - """ - model_class = model.ProductCost - - @property - def supported_fields(self): - return self.simple_fields + [ - 'product_upc', - 'vendor_id', - 'preferred', - ] - - def setup(self): - if 'product_upc' in self.fields: - self.products = cache.cache_model(self.session, model.Product, key='upc', - progress=self.progress) - if 'vendor_id' in self.fields: - self.vendors = cache.cache_model(self.session, model.Vendor, key='id', - progress=self.progress) - - def cache_query_options(self): - options = [] - if 'product_upc' in self.fields: - options.append(orm.joinedload(model.ProductCost.product)) - if 'vendor_id' in self.fields: - options.append(orm.joinedload(model.ProductCost.vendor)) - return options - - def normalize_instance(self, cost): - data = super(ProductCostImporter, self).normalize_instance(cost) - if 'product_upc' in self.fields: - data['product_upc'] = cost.product.upc - if 'vendor_id' in self.fields: - data['vendor_id'] = cost.vendor.id - self.newval(data, 'preferred', cost.preference == 1) - return data - - def update_instance(self, cost, data, instance_data=None): - cost = super(ProductCostImporter, self).update_instance(cost, data, instance_data) - - if 'product_upc' in self.fields and 'product_uuid' not in self.fields: - upc = data['product_upc'] - assert upc, "Source data has no product_upc value: {0}".format(repr(data)) - product = self.products.get(upc) - if not product: - product = model.Product() - product.upc = upc - product.description = "(auto-created)" - self.session.add(product) - self.products[product.upc] = product - if not cost.product: - product.costs.append(cost) - elif cost.product is not product: - log.warning("duplicate products detected for UPC {0}".format(upc.pretty())) - - if 'vendor_id' in self.fields and 'vendor_uuid' not in self.fields: - id_ = data['vendor_id'] - assert id_, "Source data has no vendor_id value: {0}".format(repr(data)) - vendor = self.vendors.get(id_) - if not vendor: - vendor = model.Vendor() - vendor.id = id_ - vendor.name = "(auto-created)" - self.session.add(vendor) - self.vendors[vendor.id] = vendor - cost.vendor = vendor - - if 'preferred' in self.fields: - if data['preferred']: - if cost.preference != 1: - product = cost.product - product.costs.remove(cost) - product.costs.insert(0, cost) - else: - if cost.preference == 1: - product = cost.product - if len(product.costs) > 1: - product.costs.remove(cost) - product.costs.append(cost) - product.costs.reorder() - - return cost - - -class ProductPriceImporter(Importer): - """ - Data importer for :class:`rattail.db.model.ProductPrice`. - """ - model_class = model.ProductPrice