From cb63644c7dfbf7ea6b194b52ad4f5bbecdfad6a2 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Tue, 17 Mar 2020 16:29:35 -0500 Subject: [PATCH] Add support for Rattail -> CORE export/sync for Member data also refactor CORE -> Rattail logic to use `api.set_member()` etc. --- rattail_corepos/corepos/importing/model.py | 141 +++++++++++++++++++ rattail_corepos/corepos/importing/rattail.py | 37 +++++ rattail_corepos/corepos/util.py | 37 +++++ rattail_corepos/datasync/corepos.py | 78 ++++++++-- rattail_corepos/datasync/rattail.py | 2 +- rattail_corepos/importing/corepos/api.py | 110 ++++++--------- 6 files changed, 326 insertions(+), 79 deletions(-) diff --git a/rattail_corepos/corepos/importing/model.py b/rattail_corepos/corepos/importing/model.py index 2199ef9..96bcea4 100644 --- a/rattail_corepos/corepos/importing/model.py +++ b/rattail_corepos/corepos/importing/model.py @@ -27,6 +27,8 @@ CORE-POS model importers (webservices API) from corepos.api import CoreWebAPI from rattail import importing +from rattail.util import data_diffs +from rattail_corepos.corepos.util import get_core_members class ToCoreAPI(importing.Importer): @@ -74,6 +76,145 @@ class ToCoreAPI(importing.Importer): data[field] = '' +class MemberImporter(ToCoreAPI): + """ + Member model importer for CORE-POS + """ + model_name = 'Member' + key = 'cardNo' + supported_fields = [ + 'cardNo' + 'customerAccountID', + 'customers', + # 'memberStatus', + # 'activeStatus', + # 'customerTypeID', + # 'chargeBalance', + # 'chargeLimit', + # 'idCardUPC', + # 'startDate', + # 'endDate', + # 'addressFirstLine', + # 'addressSecondLine', + # 'city', + # 'state', + # 'zip', + # 'contactAllowed', + # 'contactMethod', + # 'modified', + ] + supported_customer_fields = [ + 'customerID', + # 'customerAccountID', + # 'cardNo', + 'firstName', + 'lastName', + # 'chargeAllowed', + # 'checksAllowed', + # 'discount', + 'accountHolder', + # 'staff', + # 'phone', + # 'altPhone', + # 'email', + # 'memberPricingAllowed', + # 'memberCouponsAllowed', + # 'lowIncomeBenefits', + # 'modified', + ] + + def get_local_objects(self, host_data=None): + return get_core_members(self.api, progress=self.progress) + + def get_single_local_object(self, key): + assert len(self.key) == 1 + assert self.key[0] == 'cardNo' + return self.api.get_member(key[0]) + + def normalize_local_object(self, member): + data = dict(member) + return data + + def data_diffs(self, local_data, host_data): + diffs = super(MemberImporter, self).data_diffs(local_data, host_data) + + # the 'customers' field requires a more granular approach, since the + # data coming from API may have different fields than our local data + if 'customers' in self.fields and 'customers' in diffs: + if not self.customer_data_differs(local_data, host_data): + diffs.remove('customers') + + return diffs + + def customer_data_differs(self, local_data, host_data): + local_customers = local_data['customers'] + host_customers = host_data['customers'] + + # if both are empty, we're good + if not local_customers and not host_customers: + return False + + # obviously we differ if record count doesn't match + if len(local_customers) != len(host_customers): + return True + + # okay then, let's traverse the "new" list + for host_customer in host_customers: + + # we differ if can't locate corresponding "old" local record + local_customer = self.find_local_customer(local_customers, host_customer) + if not local_customer: + return True + + # we differ if old and new records differ + if data_diffs(local_customer, host_customer, + fields=self.supported_customer_fields): + return True + + # okay, now let's traverse the "old" list + for local_customer in local_customers: + + # we differ if can't locate corresponding "new" host record + host_customer = self.find_host_customer(host_customers, local_customer) + if not host_customer: + return True + + # guess we don't differ after all + return False + + def find_local_customer(self, local_customers, host_customer): + assert 'customerID' in self.supported_customer_fields + + if not host_customer['customerID']: + return # new customer + + for local_customer in local_customers: + if local_customer['customerID'] == host_customer['customerID']: + return local_customer + + def find_host_customer(self, host_customers, local_customer): + assert 'customerID' in self.supported_customer_fields + + for host_customer in host_customers: + if host_customer['customerID'] == local_customer['customerID']: + return host_customer + + def create_object(self, key, data): + # we can get away with using the same logic for both here + return self.update_object(None, data) + + def update_object(self, member, data, local_data=None): + """ + Push an update for the member, via the CORE API. + """ + if self.dry_run: + return data + + cardNo = data.pop('cardNo') + member = self.api.set_member(cardNo, **data) + return member + + class DepartmentImporter(ToCoreAPI): """ Department model importer for CORE-POS diff --git a/rattail_corepos/corepos/importing/rattail.py b/rattail_corepos/corepos/importing/rattail.py index d6e0395..612120b 100644 --- a/rattail_corepos/corepos/importing/rattail.py +++ b/rattail_corepos/corepos/importing/rattail.py @@ -47,6 +47,7 @@ class FromRattailToCore(importing.FromRattailHandler): def get_importers(self): importers = OrderedDict() + importers['Member'] = MemberImporter importers['Department'] = DepartmentImporter importers['Subdepartment'] = SubdepartmentImporter importers['Vendor'] = VendorImporter @@ -60,6 +61,42 @@ class FromRattail(importing.FromSQLAlchemy): """ +class MemberImporter(FromRattail, corepos_importing.model.MemberImporter): + """ + Member data exporter + """ + host_model_class = model.Customer + key = 'cardNo' + supported_fields = [ + 'cardNo', + 'customerAccountID', + 'customers', + ] + supported_person_fields = [ + 'customerID', + 'firstName', + 'lastName', + 'accountHolder', + ] + + def normalize_host_object(self, customer): + + people = [] + for i, person in enumerate(customer.people, 1): + people.append({ + 'customerID': str(person.corepos_customer_id), + 'firstName': person.first_name, + 'lastName': person.last_name, + 'accountHolder': i == 1, + }) + + return { + 'cardNo': customer.number, + 'customerAccountID': customer.id, + 'customers': people, + } + + class DepartmentImporter(FromRattail, corepos_importing.model.DepartmentImporter): """ Department data exporter diff --git a/rattail_corepos/corepos/util.py b/rattail_corepos/corepos/util.py index cd5e34e..fd25030 100644 --- a/rattail_corepos/corepos/util.py +++ b/rattail_corepos/corepos/util.py @@ -24,11 +24,48 @@ CORE-POS misc. utilities """ +import logging + import sqlalchemy as sa from corepos.db.office_op import Session as CoreSession, model as corepos from rattail.db.util import short_session +from rattail.util import OrderedDict, progress_loop + + +log = logging.getLogger(__name__) + + +def get_core_members(api, progress=None): + """ + Shared logic for fetching *all* customer accounts from CORE-POS API. + """ + # TODO: ideally could do this, but API doesn't let us fetch "all" + # return api.get_members() + + # first we fetch all customer records from CORE DB + with short_session(Session=CoreSession) as s: + db_customers = s.query(corepos.Customer).all() + s.expunge_all() + + # now we must fetch each customer account individually from API + members = OrderedDict() + + def fetch(dbcust, i): + if dbcust.card_number in members: + return # already fetched this one + member = api.get_member(dbcust.card_number) + if member: + members[dbcust.card_number] = member + else: + logger = log.warning if dbcust.account_holder else log.debug + logger("could not fetch member from CORE API: %s", + dbcust.card_number) + + progress_loop(fetch, db_customers, progress, + message="Fetching Member data from CORE API") + return list(members.values()) def get_max_existing_vendor_id(session=None): diff --git a/rattail_corepos/datasync/corepos.py b/rattail_corepos/datasync/corepos.py index 22ab69e..8419707 100644 --- a/rattail_corepos/datasync/corepos.py +++ b/rattail_corepos/datasync/corepos.py @@ -152,21 +152,23 @@ class FromRattailToCore(NewDataSyncImportConsumer): # also establish the API client for each! importer.establish_api() - # sync all Department changes + # sync all Customer changes types = [ - 'Department', + 'Customer', + 'Person', + 'CustomerPerson', ] for change in [c for c in changes if c.payload_type in types]: - if change.payload_type == 'Department' and change.deletion: - # TODO: we have no way (yet) to delete a CORE department via API + if change.payload_type == 'Customer' and change.deletion: # # just do default logic for this one # self.invoke_importer(session, change) + # TODO: we have no way to delete a CORE customer via API, right? pass else: # we consider this an "add/update" - department = self.get_department(session, change) - if department: - self.process_change(session, self.importers['Department'], - host_object=department) + customers = self.get_customers(session, change) + for customer in customers: + self.process_change(session, self.importers['Member'], + host_object=customer) # sync all Vendor changes types = [ @@ -186,9 +188,53 @@ class FromRattailToCore(NewDataSyncImportConsumer): self.process_change(session, self.importers['Vendor'], host_object=vendor) - def get_department(self, session, change): - if change.payload_type == 'Department': - return session.query(model.Department).get(change.payload_key) + # sync all Product changes + types = [ + 'Product', + 'ProductPrice', + ] + for change in [c for c in changes if c.payload_type in types]: + if change.payload_type == 'Product' and change.deletion: + # # just do default logic for this one + # self.invoke_importer(session, change) + # TODO: we have no way to delete a CORE product via API, right? + pass + else: # we consider this an "add/update" + product = self.get_product(session, change) + if product: + self.process_change(session, self.importers['Product'], + host_object=product) + + # process all remaining supported models with typical logic + types = [ + 'Department', + 'Subdepartment', + ] + for change in [c for c in changes if c.payload_type in types]: + self.invoke_importer(session, change) + + def get_host_object(self, session, change): + return session.query(getattr(model, change.payload_type))\ + .get(change.payload_key) + + def get_customers(self, session, change): + + if change.payload_type == 'Customer': + customer = session.query(model.Customer).get(change.payload_key) + if customer: + return [customer] + + if change.payload_type == 'CustomerPerson': + cp = session.query(model.CustomerPerson).get(change.payload_key) + if cp: + return [cp.customer] + + if change.payload_type == 'Person': + person = session.query(model.Person).get(change.payload_key) + if person: + return person.customers + + return [] def get_vendor(self, session, change): @@ -204,3 +250,13 @@ class FromRattailToCore(NewDataSyncImportConsumer): email = session.query(model.VendorEmailAddress).get(change.payload_key) if email: return email.vendor + + def get_product(self, session, change): + + if change.payload_type == 'Product': + return session.query(model.Product).get(change.payload_key) + + if change.payload_type == 'ProductPrice': + price = session.query(model.ProductPrice).get(change.payload_key) + if price: + return price.product diff --git a/rattail_corepos/datasync/rattail.py b/rattail_corepos/datasync/rattail.py index 0a8fd4c..37a5d7a 100644 --- a/rattail_corepos/datasync/rattail.py +++ b/rattail_corepos/datasync/rattail.py @@ -86,7 +86,7 @@ class FromCOREAPIToRattail(NewDataSyncImportConsumer): def get_host_object(self, session, change): if change.payload_type == 'Customer': - return self.api.get_customer(change.payload_key) + return self.api.get_member(change.payload_key) if change.payload_type == 'Department': return self.api.get_department(change.payload_key) if change.payload_type == 'Subdepartment': diff --git a/rattail_corepos/importing/corepos/api.py b/rattail_corepos/importing/corepos/api.py index 57ec9c8..d0a2fc1 100644 --- a/rattail_corepos/importing/corepos/api.py +++ b/rattail_corepos/importing/corepos/api.py @@ -31,14 +31,14 @@ from sqlalchemy import orm from sqlalchemy.orm.exc import NoResultFound from corepos.api import CoreWebAPI -from corepos.db.office_op import Session as CoreSession, model as corepos from rattail import importing from rattail.db import model from rattail.gpc import GPC from rattail.util import OrderedDict -from rattail.db.util import normalize_full_name, short_session +from rattail.db.util import normalize_full_name from rattail_corepos import importing as corepos_importing +from rattail_corepos.corepos.util import get_core_members log = logging.getLogger(__name__) @@ -75,32 +75,8 @@ class FromCOREPOSAPI(importing.Importer): url = self.config.require('corepos.api', 'url') self.api = CoreWebAPI(url) - def get_core_customers(self): - # TODO: ideally could do this, but API doesn't let us fetch "all" - # return self.api.get_customers() - - # first we fetch all customer records from CORE DB - with short_session(Session=CoreSession) as s: - db_customers = s.query(corepos.Customer).all() - s.expunge_all() - - # now we must fetch each customer account individually from API - customers = OrderedDict() - - def fetch(dbcust, i): - if dbcust.card_number in customers: - return # already fetched this one - customer = self.api.get_customer(dbcust.card_number) - if customer: - customers[dbcust.card_number] = customer - else: - logger = log.warning if dbcust.account_holder else log.debug - logger("could not fetch customer from CORE API: %s", - dbcust.card_number) - - self.progress_loop(fetch, db_customers, - message="Fetching Customer data from CORE API") - return list(customers.values()) + def get_core_members(self): + return get_core_members(self.api, progress=self.progress) class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter): @@ -115,26 +91,26 @@ class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter): ] def get_host_objects(self): - return self.get_core_customers() + return self.get_core_members() - def normalize_host_object(self, customer): + def normalize_host_object(self, member): - # figure out the "account holder" person for the customer - people = customer['customers'] - account_holders = [person for person in people - if person['accountHolder']] + # figure out the "account holder" customer for the member + customers = member['customers'] + account_holders = [customer for customer in customers + if customer['accountHolder']] if len(account_holders) > 1: - log.warning("customer %s has %s account holders in CORE: %s", - customer['cardNo'], len(account_holders), customer) + log.warning("member %s has %s account holders in CORE: %s", + member['cardNo'], len(account_holders), member) elif not account_holders: - raise NotImplementedError("TODO: how to handle customer with no account holders?") - person = account_holders[0] + raise NotImplementedError("TODO: how to handle member with no account holders?") + customer = account_holders[0] return { - 'id': customer['customerAccountID'], - 'number': customer['cardNo'], - 'name': normalize_full_name(person['firstName'], - person['lastName']), + 'id': member['customerAccountID'], + 'number': member['cardNo'], + 'name': normalize_full_name(customer['firstName'], + customer['lastName']), } @@ -160,36 +136,36 @@ class PersonImporter(FromCOREPOSAPI, corepos_importing.model.PersonImporter): def get_host_objects(self): - # first get all customer data from CORE API - customers = self.get_core_customers() + # first get all member data from CORE API + members = self.get_core_members() normalized = [] # then collect all the "person" records - def normalize(customer, i): - normalized.extend(self.get_person_objects_for_customer(customer)) + def normalize(member, i): + normalized.extend(self.get_person_objects_for_member(member)) - self.progress_loop(normalize, customers, + self.progress_loop(normalize, members, message="Collecting Person data from CORE") return normalized - def get_person_objects_for_customer(self, customer): + def get_person_objects_for_member(self, member): """ - Return a list of Person data objects for the given Customer. This + Return a list of Person data objects for the given Member. This logic is split out separately so that datasync can leverage it too. """ - records = [] + people = [] # make sure we put the account holder first in the list! - people = sorted(customer['customers'], - key=lambda cust: 1 if cust['accountHolder'] else 0, - reverse=True) + customers = sorted(member['customers'], + key=lambda cust: 1 if cust['accountHolder'] else 0, + reverse=True) - for i, person in enumerate(people, 1): - person = dict(person) + for i, customer in enumerate(customers, 1): + person = dict(customer) person['customer_person_ordinal'] = i - records.append(person) + people.append(person) - return records + return people def get_customer(self, id): if hasattr(self, 'customers'): @@ -250,24 +226,24 @@ class CustomerPersonImporter(FromCOREPOSAPI, importing.model.CustomerPersonImpor def get_host_objects(self): - # first get all customer data from CORE API - customers = self.get_core_customers() + # first get all member data from CORE API + members = self.get_core_members() normalized = [] # then collect all customer/person combination records - def normalize(customer, i): + def normalize(member, i): # make sure we put the account holder first in the list! - people = sorted(customer['customers'], - key=lambda cust: 1 if cust['accountHolder'] else 0, - reverse=True) - for i, person in enumerate(people, 1): + customers = sorted(member['customers'], + key=lambda cust: 1 if cust['accountHolder'] else 0, + reverse=True) + for i, customer in enumerate(customers, 1): normalized.append({ - 'customer_account_id': customer['customerAccountID'], - 'person_customer_id': person['customerID'], + 'customer_account_id': member['customerAccountID'], + 'person_customer_id': customer['customerID'], 'ordinal': i, }) - self.progress_loop(normalize, customers, + self.progress_loop(normalize, members, message="Collecting CustomerPerson data from CORE") return normalized