From 8f9f77b6b7d439129fe50611f119c1a7370b65a7 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 16 Mar 2020 19:45:54 -0500 Subject: [PATCH] Add proper importing for Customer/Person data from CORE API includes datasync support. i think it even works right, but we'll see --- rattail_corepos/datasync/rattail.py | 38 ++++ rattail_corepos/importing/corepos/api.py | 215 +++++++++++++++++++++-- rattail_corepos/importing/model.py | 19 ++ 3 files changed, 256 insertions(+), 16 deletions(-) diff --git a/rattail_corepos/datasync/rattail.py b/rattail_corepos/datasync/rattail.py index 867397c..0a8fd4c 100644 --- a/rattail_corepos/datasync/rattail.py +++ b/rattail_corepos/datasync/rattail.py @@ -46,6 +46,44 @@ class FromCOREAPIToRattail(NewDataSyncImportConsumer): url = self.config.require('corepos.api', 'url') self.api = CoreWebAPI(url) + def process_changes(self, session, changes): + if self.runas_username: + session.set_continuum_user(self.runas_username) + + # update all importers with current Rattail session + for importer in self.importers.values(): + importer.session = session + # also establish the API client for each! + importer.establish_api() + + # sync all Customer-related changes + types = [ + 'Customer', + ] + for change in [c for c in changes if c.payload_type in types]: + if change.deletion: + # normal logic works fine for this (maybe?) + self.invoke_importer(session, change) + else: + # import customer data from API, into various Rattail tables + customer = self.get_host_object(session, change) + self.process_change(session, self.importers['Customer'], + host_object=customer) + people = self.importers['Person'].get_person_objects_for_customer(customer) + for person in people: + self.process_change(session, self.importers['Person'], + host_object=person) + + # process all remaining supported models with typical logic + types = [ + 'Department', + 'Subdepartment', + 'Vendor', + 'Product', + ] + for change in [c for c in changes if c.payload_type in types]: + self.invoke_importer(session, change) + def get_host_object(self, session, change): if change.payload_type == 'Customer': return self.api.get_customer(change.payload_key) diff --git a/rattail_corepos/importing/corepos/api.py b/rattail_corepos/importing/corepos/api.py index 29f12ff..57ec9c8 100644 --- a/rattail_corepos/importing/corepos/api.py +++ b/rattail_corepos/importing/corepos/api.py @@ -27,10 +27,14 @@ CORE POS (API) -> Rattail data importing import decimal import logging +from sqlalchemy import orm +from sqlalchemy.orm.exc import NoResultFound + from corepos.api import CoreWebAPI from corepos.db.office_op import Session as CoreSession, model as corepos from rattail import importing +from rattail.db import model from rattail.gpc import GPC from rattail.util import OrderedDict from rattail.db.util import normalize_full_name, short_session @@ -49,6 +53,8 @@ class FromCOREPOSToRattail(importing.ToRattailHandler): def get_importers(self): importers = OrderedDict() importers['Customer'] = CustomerImporter + importers['Person'] = PersonImporter + importers['CustomerPerson'] = CustomerPersonImporter importers['Department'] = DepartmentImporter importers['Subdepartment'] = SubdepartmentImporter importers['Vendor'] = VendorImporter @@ -69,21 +75,7 @@ class FromCOREPOSAPI(importing.Importer): url = self.config.require('corepos.api', 'url') self.api = CoreWebAPI(url) - -class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter): - """ - Importer for customer data from CORE POS API. - """ - key = 'number' - supported_fields = [ - 'id', - 'number', - 'name', - 'first_name', - 'last_name', - ] - - def get_host_objects(self): + def get_core_customers(self): # TODO: ideally could do this, but API doesn't let us fetch "all" # return self.api.get_customers() @@ -107,9 +99,24 @@ class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter): dbcust.card_number) self.progress_loop(fetch, db_customers, - message="Fetching Customer data from CORE-POS API") + message="Fetching Customer data from CORE API") return list(customers.values()) + +class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter): + """ + Importer for customer data from CORE POS API. + """ + key = 'number' + supported_fields = [ + 'id', + 'number', + 'name', + ] + + def get_host_objects(self): + return self.get_core_customers() + def normalize_host_object(self, customer): # figure out the "account holder" person for the customer @@ -128,8 +135,184 @@ class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter): 'number': customer['cardNo'], 'name': normalize_full_name(person['firstName'], person['lastName']), + } + + +class PersonImporter(FromCOREPOSAPI, corepos_importing.model.PersonImporter): + """ + Importer for person data from CORE POS API. + """ + key = 'corepos_customer_id' + supported_fields = [ + 'corepos_customer_id', + 'first_name', + 'last_name', + 'display_name', + 'customer_uuid', + 'customer_person_ordinal', + ] + + def setup(self): + super(PersonImporter, self).setup() + model = self.config.get_model() + + self.customers = self.cache_model(model.Customer, key='id') + + def get_host_objects(self): + + # first get all customer data from CORE API + customers = self.get_core_customers() + normalized = [] + + # then collect all the "person" records + def normalize(customer, i): + normalized.extend(self.get_person_objects_for_customer(customer)) + + self.progress_loop(normalize, customers, + message="Collecting Person data from CORE") + return normalized + + def get_person_objects_for_customer(self, customer): + """ + Return a list of Person data objects for the given Customer. This + logic is split out separately so that datasync can leverage it too. + """ + records = [] + + # make sure we put the account holder first in the list! + people = sorted(customer['customers'], + key=lambda cust: 1 if cust['accountHolder'] else 0, + reverse=True) + + for i, person in enumerate(people, 1): + person = dict(person) + person['customer_person_ordinal'] = i + records.append(person) + + return records + + def get_customer(self, id): + if hasattr(self, 'customers'): + return self.customers.get(id) + + try: + return self.session.query(model.Customer)\ + .filter(model.Customer.id == id)\ + .one() + except NoResultFound: + pass + + def normalize_host_object(self, person): + + customer = self.get_customer(person['customerAccountID']) + if not customer: + log.warning("Rattail customer not found for customerAccountID: %s", + person['customerAccountID']) + return + + return { + 'corepos_customer_id': int(person['customerID']), 'first_name': person['firstName'], 'last_name': person['lastName'], + 'display_name': normalize_full_name(person['firstName'], + person['lastName']), + 'customer_uuid': customer.uuid, + 'customer_person_ordinal': person['customer_person_ordinal'], + } + + +class CustomerPersonImporter(FromCOREPOSAPI, importing.model.CustomerPersonImporter): + """ + Importer for customer-person linkage data from CORE POS API. + + Note that we don't use this one in datasync, it's just for nightly + double-check. + """ + key = ('customer_uuid', 'person_uuid') + supported_fields = [ + 'customer_uuid', + 'person_uuid', + 'ordinal', + ] + + def setup(self): + super(CustomerPersonImporter, self).setup() + model = self.config.get_model() + + self.customers = self.cache_model(model.Customer, key='id') + + query = self.session.query(model.Person)\ + .join(model.CorePerson)\ + .filter(model.CorePerson.corepos_customer_id != None) + self.people = self.cache_model(model.Person, query=query, + key='corepos_customer_id', + query_options=[orm.joinedload(model.Person._corepos)]) + + def get_host_objects(self): + + # first get all customer data from CORE API + customers = self.get_core_customers() + normalized = [] + + # then collect all customer/person combination records + def normalize(customer, i): + # make sure we put the account holder first in the list! + people = sorted(customer['customers'], + key=lambda cust: 1 if cust['accountHolder'] else 0, + reverse=True) + for i, person in enumerate(people, 1): + normalized.append({ + 'customer_account_id': customer['customerAccountID'], + 'person_customer_id': person['customerID'], + 'ordinal': i, + }) + + self.progress_loop(normalize, customers, + message="Collecting CustomerPerson data from CORE") + return normalized + + def get_customer(self, id): + if hasattr(self, 'customers'): + return self.customers.get(id) + + try: + return self.session.query(model.Customer)\ + .filter(model.Customer.id == id)\ + .one() + except NoResultFound: + pass + + def get_person(self, corepos_customer_id): + if hasattr(self, 'people'): + return self.people.get(corepos_customer_id) + + model = self.config.get_model() + try: + return self.session.query(model.Person)\ + .join(model.CorePerson)\ + .filter(model.CorePerson.corepos_customer_id == corepos_customer_id)\ + .one() + except NoResultFound: + pass + + def normalize_host_object(self, cp): + + customer = self.get_customer(cp['customer_account_id']) + if not customer: + log.warning("Rattail customer not found for customerAccountID: %s", + cp['customer_account_id']) + return + + person = self.get_person(int(cp['person_customer_id'])) + if not person: + log.warning("Rattail person not found for customerID: %s", + cp['person_customer_id']) + return + + return { + 'customer_uuid': customer.uuid, + 'person_uuid': person.uuid, + 'ordinal': cp['ordinal'], } diff --git a/rattail_corepos/importing/model.py b/rattail_corepos/importing/model.py index 261d026..450ec31 100644 --- a/rattail_corepos/importing/model.py +++ b/rattail_corepos/importing/model.py @@ -31,6 +31,25 @@ from rattail import importing # core importer overrides ############################## +class PersonImporter(importing.model.PersonImporter): + + extension_attr = '_corepos' + extension_fields = [ + 'corepos_customer_id', + ] + + def cache_query(self): + query = super(PersonImporter, self).cache_query() + model = self.config.get_model() + + # we want to ignore people with no CORE ID, if that's (part of) our key + if 'corepos_customer_id' in self.key: + query = query.join(model.CorePerson)\ + .filter(model.CorePerson.corepos_customer_id != None) + + return query + + class VendorImporter(importing.model.VendorImporter): extension_attr = '_corepos'