Add support for Rattail -> CORE export/sync for Member data
also refactor CORE -> Rattail logic to use `api.set_member()` etc.
This commit is contained in:
Lance Edgar
parent
8d47a1449c
commit
cb63644c7d
|
|
@ -27,6 +27,8 @@ CORE-POS model importers (webservices API)
|
|||
from corepos.api import CoreWebAPI
|
||||
|
||||
from rattail import importing
|
||||
from rattail.util import data_diffs
|
||||
from rattail_corepos.corepos.util import get_core_members
|
||||
|
||||
|
||||
class ToCoreAPI(importing.Importer):
|
||||
|
|
@ -74,6 +76,145 @@ class ToCoreAPI(importing.Importer):
|
|||
data[field] = ''
|
||||
|
||||
|
||||
class MemberImporter(ToCoreAPI):
|
||||
"""
|
||||
Member model importer for CORE-POS
|
||||
"""
|
||||
model_name = 'Member'
|
||||
key = 'cardNo'
|
||||
supported_fields = [
|
||||
'cardNo'
|
||||
'customerAccountID',
|
||||
'customers',
|
||||
# 'memberStatus',
|
||||
# 'activeStatus',
|
||||
# 'customerTypeID',
|
||||
# 'chargeBalance',
|
||||
# 'chargeLimit',
|
||||
# 'idCardUPC',
|
||||
# 'startDate',
|
||||
# 'endDate',
|
||||
# 'addressFirstLine',
|
||||
# 'addressSecondLine',
|
||||
# 'city',
|
||||
# 'state',
|
||||
# 'zip',
|
||||
# 'contactAllowed',
|
||||
# 'contactMethod',
|
||||
# 'modified',
|
||||
]
|
||||
supported_customer_fields = [
|
||||
'customerID',
|
||||
# 'customerAccountID',
|
||||
# 'cardNo',
|
||||
'firstName',
|
||||
'lastName',
|
||||
# 'chargeAllowed',
|
||||
# 'checksAllowed',
|
||||
# 'discount',
|
||||
'accountHolder',
|
||||
# 'staff',
|
||||
# 'phone',
|
||||
# 'altPhone',
|
||||
# 'email',
|
||||
# 'memberPricingAllowed',
|
||||
# 'memberCouponsAllowed',
|
||||
# 'lowIncomeBenefits',
|
||||
# 'modified',
|
||||
]
|
||||
|
||||
def get_local_objects(self, host_data=None):
|
||||
return get_core_members(self.api, progress=self.progress)
|
||||
|
||||
def get_single_local_object(self, key):
|
||||
assert len(self.key) == 1
|
||||
assert self.key[0] == 'cardNo'
|
||||
return self.api.get_member(key[0])
|
||||
|
||||
def normalize_local_object(self, member):
|
||||
data = dict(member)
|
||||
return data
|
||||
|
||||
def data_diffs(self, local_data, host_data):
|
||||
diffs = super(MemberImporter, self).data_diffs(local_data, host_data)
|
||||
|
||||
# the 'customers' field requires a more granular approach, since the
|
||||
# data coming from API may have different fields than our local data
|
||||
if 'customers' in self.fields and 'customers' in diffs:
|
||||
if not self.customer_data_differs(local_data, host_data):
|
||||
diffs.remove('customers')
|
||||
|
||||
return diffs
|
||||
|
||||
def customer_data_differs(self, local_data, host_data):
|
||||
local_customers = local_data['customers']
|
||||
host_customers = host_data['customers']
|
||||
|
||||
# if both are empty, we're good
|
||||
if not local_customers and not host_customers:
|
||||
return False
|
||||
|
||||
# obviously we differ if record count doesn't match
|
||||
if len(local_customers) != len(host_customers):
|
||||
return True
|
||||
|
||||
# okay then, let's traverse the "new" list
|
||||
for host_customer in host_customers:
|
||||
|
||||
# we differ if can't locate corresponding "old" local record
|
||||
local_customer = self.find_local_customer(local_customers, host_customer)
|
||||
if not local_customer:
|
||||
return True
|
||||
|
||||
# we differ if old and new records differ
|
||||
if data_diffs(local_customer, host_customer,
|
||||
fields=self.supported_customer_fields):
|
||||
return True
|
||||
|
||||
# okay, now let's traverse the "old" list
|
||||
for local_customer in local_customers:
|
||||
|
||||
# we differ if can't locate corresponding "new" host record
|
||||
host_customer = self.find_host_customer(host_customers, local_customer)
|
||||
if not host_customer:
|
||||
return True
|
||||
|
||||
# guess we don't differ after all
|
||||
return False
|
||||
|
||||
def find_local_customer(self, local_customers, host_customer):
|
||||
assert 'customerID' in self.supported_customer_fields
|
||||
|
||||
if not host_customer['customerID']:
|
||||
return # new customer
|
||||
|
||||
for local_customer in local_customers:
|
||||
if local_customer['customerID'] == host_customer['customerID']:
|
||||
return local_customer
|
||||
|
||||
def find_host_customer(self, host_customers, local_customer):
|
||||
assert 'customerID' in self.supported_customer_fields
|
||||
|
||||
for host_customer in host_customers:
|
||||
if host_customer['customerID'] == local_customer['customerID']:
|
||||
return host_customer
|
||||
|
||||
def create_object(self, key, data):
|
||||
# we can get away with using the same logic for both here
|
||||
return self.update_object(None, data)
|
||||
|
||||
def update_object(self, member, data, local_data=None):
|
||||
"""
|
||||
Push an update for the member, via the CORE API.
|
||||
"""
|
||||
if self.dry_run:
|
||||
return data
|
||||
|
||||
cardNo = data.pop('cardNo')
|
||||
member = self.api.set_member(cardNo, **data)
|
||||
return member
|
||||
|
||||
|
||||
class DepartmentImporter(ToCoreAPI):
|
||||
"""
|
||||
Department model importer for CORE-POS
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ class FromRattailToCore(importing.FromRattailHandler):
|
|||
|
||||
def get_importers(self):
|
||||
importers = OrderedDict()
|
||||
importers['Member'] = MemberImporter
|
||||
importers['Department'] = DepartmentImporter
|
||||
importers['Subdepartment'] = SubdepartmentImporter
|
||||
importers['Vendor'] = VendorImporter
|
||||
|
|
@ -60,6 +61,42 @@ class FromRattail(importing.FromSQLAlchemy):
|
|||
"""
|
||||
|
||||
|
||||
class MemberImporter(FromRattail, corepos_importing.model.MemberImporter):
|
||||
"""
|
||||
Member data exporter
|
||||
"""
|
||||
host_model_class = model.Customer
|
||||
key = 'cardNo'
|
||||
supported_fields = [
|
||||
'cardNo',
|
||||
'customerAccountID',
|
||||
'customers',
|
||||
]
|
||||
supported_person_fields = [
|
||||
'customerID',
|
||||
'firstName',
|
||||
'lastName',
|
||||
'accountHolder',
|
||||
]
|
||||
|
||||
def normalize_host_object(self, customer):
|
||||
|
||||
people = []
|
||||
for i, person in enumerate(customer.people, 1):
|
||||
people.append({
|
||||
'customerID': str(person.corepos_customer_id),
|
||||
'firstName': person.first_name,
|
||||
'lastName': person.last_name,
|
||||
'accountHolder': i == 1,
|
||||
})
|
||||
|
||||
return {
|
||||
'cardNo': customer.number,
|
||||
'customerAccountID': customer.id,
|
||||
'customers': people,
|
||||
}
|
||||
|
||||
|
||||
class DepartmentImporter(FromRattail, corepos_importing.model.DepartmentImporter):
|
||||
"""
|
||||
Department data exporter
|
||||
|
|
|
|||
|
|
@ -24,11 +24,48 @@
|
|||
CORE-POS misc. utilities
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from corepos.db.office_op import Session as CoreSession, model as corepos
|
||||
|
||||
from rattail.db.util import short_session
|
||||
from rattail.util import OrderedDict, progress_loop
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_core_members(api, progress=None):
|
||||
"""
|
||||
Shared logic for fetching *all* customer accounts from CORE-POS API.
|
||||
"""
|
||||
# TODO: ideally could do this, but API doesn't let us fetch "all"
|
||||
# return api.get_members()
|
||||
|
||||
# first we fetch all customer records from CORE DB
|
||||
with short_session(Session=CoreSession) as s:
|
||||
db_customers = s.query(corepos.Customer).all()
|
||||
s.expunge_all()
|
||||
|
||||
# now we must fetch each customer account individually from API
|
||||
members = OrderedDict()
|
||||
|
||||
def fetch(dbcust, i):
|
||||
if dbcust.card_number in members:
|
||||
return # already fetched this one
|
||||
member = api.get_member(dbcust.card_number)
|
||||
if member:
|
||||
members[dbcust.card_number] = member
|
||||
else:
|
||||
logger = log.warning if dbcust.account_holder else log.debug
|
||||
logger("could not fetch member from CORE API: %s",
|
||||
dbcust.card_number)
|
||||
|
||||
progress_loop(fetch, db_customers, progress,
|
||||
message="Fetching Member data from CORE API")
|
||||
return list(members.values())
|
||||
|
||||
|
||||
def get_max_existing_vendor_id(session=None):
|
||||
|
|
|
|||
|
|
@ -152,21 +152,23 @@ class FromRattailToCore(NewDataSyncImportConsumer):
|
|||
# also establish the API client for each!
|
||||
importer.establish_api()
|
||||
|
||||
# sync all Department changes
|
||||
# sync all Customer changes
|
||||
types = [
|
||||
'Department',
|
||||
'Customer',
|
||||
'Person',
|
||||
'CustomerPerson',
|
||||
]
|
||||
for change in [c for c in changes if c.payload_type in types]:
|
||||
if change.payload_type == 'Department' and change.deletion:
|
||||
# TODO: we have no way (yet) to delete a CORE department via API
|
||||
if change.payload_type == 'Customer' and change.deletion:
|
||||
# # just do default logic for this one
|
||||
# self.invoke_importer(session, change)
|
||||
# TODO: we have no way to delete a CORE customer via API, right?
|
||||
pass
|
||||
else: # we consider this an "add/update"
|
||||
department = self.get_department(session, change)
|
||||
if department:
|
||||
self.process_change(session, self.importers['Department'],
|
||||
host_object=department)
|
||||
customers = self.get_customers(session, change)
|
||||
for customer in customers:
|
||||
self.process_change(session, self.importers['Member'],
|
||||
host_object=customer)
|
||||
|
||||
# sync all Vendor changes
|
||||
types = [
|
||||
|
|
@ -186,9 +188,53 @@ class FromRattailToCore(NewDataSyncImportConsumer):
|
|||
self.process_change(session, self.importers['Vendor'],
|
||||
host_object=vendor)
|
||||
|
||||
def get_department(self, session, change):
|
||||
if change.payload_type == 'Department':
|
||||
return session.query(model.Department).get(change.payload_key)
|
||||
# sync all Product changes
|
||||
types = [
|
||||
'Product',
|
||||
'ProductPrice',
|
||||
]
|
||||
for change in [c for c in changes if c.payload_type in types]:
|
||||
if change.payload_type == 'Product' and change.deletion:
|
||||
# # just do default logic for this one
|
||||
# self.invoke_importer(session, change)
|
||||
# TODO: we have no way to delete a CORE product via API, right?
|
||||
pass
|
||||
else: # we consider this an "add/update"
|
||||
product = self.get_product(session, change)
|
||||
if product:
|
||||
self.process_change(session, self.importers['Product'],
|
||||
host_object=product)
|
||||
|
||||
# process all remaining supported models with typical logic
|
||||
types = [
|
||||
'Department',
|
||||
'Subdepartment',
|
||||
]
|
||||
for change in [c for c in changes if c.payload_type in types]:
|
||||
self.invoke_importer(session, change)
|
||||
|
||||
def get_host_object(self, session, change):
|
||||
return session.query(getattr(model, change.payload_type))\
|
||||
.get(change.payload_key)
|
||||
|
||||
def get_customers(self, session, change):
|
||||
|
||||
if change.payload_type == 'Customer':
|
||||
customer = session.query(model.Customer).get(change.payload_key)
|
||||
if customer:
|
||||
return [customer]
|
||||
|
||||
if change.payload_type == 'CustomerPerson':
|
||||
cp = session.query(model.CustomerPerson).get(change.payload_key)
|
||||
if cp:
|
||||
return [cp.customer]
|
||||
|
||||
if change.payload_type == 'Person':
|
||||
person = session.query(model.Person).get(change.payload_key)
|
||||
if person:
|
||||
return person.customers
|
||||
|
||||
return []
|
||||
|
||||
def get_vendor(self, session, change):
|
||||
|
||||
|
|
@ -204,3 +250,13 @@ class FromRattailToCore(NewDataSyncImportConsumer):
|
|||
email = session.query(model.VendorEmailAddress).get(change.payload_key)
|
||||
if email:
|
||||
return email.vendor
|
||||
|
||||
def get_product(self, session, change):
|
||||
|
||||
if change.payload_type == 'Product':
|
||||
return session.query(model.Product).get(change.payload_key)
|
||||
|
||||
if change.payload_type == 'ProductPrice':
|
||||
price = session.query(model.ProductPrice).get(change.payload_key)
|
||||
if price:
|
||||
return price.product
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ class FromCOREAPIToRattail(NewDataSyncImportConsumer):
|
|||
|
||||
def get_host_object(self, session, change):
|
||||
if change.payload_type == 'Customer':
|
||||
return self.api.get_customer(change.payload_key)
|
||||
return self.api.get_member(change.payload_key)
|
||||
if change.payload_type == 'Department':
|
||||
return self.api.get_department(change.payload_key)
|
||||
if change.payload_type == 'Subdepartment':
|
||||
|
|
|
|||
|
|
@ -31,14 +31,14 @@ from sqlalchemy import orm
|
|||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from corepos.api import CoreWebAPI
|
||||
from corepos.db.office_op import Session as CoreSession, model as corepos
|
||||
|
||||
from rattail import importing
|
||||
from rattail.db import model
|
||||
from rattail.gpc import GPC
|
||||
from rattail.util import OrderedDict
|
||||
from rattail.db.util import normalize_full_name, short_session
|
||||
from rattail.db.util import normalize_full_name
|
||||
from rattail_corepos import importing as corepos_importing
|
||||
from rattail_corepos.corepos.util import get_core_members
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -75,32 +75,8 @@ class FromCOREPOSAPI(importing.Importer):
|
|||
url = self.config.require('corepos.api', 'url')
|
||||
self.api = CoreWebAPI(url)
|
||||
|
||||
def get_core_customers(self):
|
||||
# TODO: ideally could do this, but API doesn't let us fetch "all"
|
||||
# return self.api.get_customers()
|
||||
|
||||
# first we fetch all customer records from CORE DB
|
||||
with short_session(Session=CoreSession) as s:
|
||||
db_customers = s.query(corepos.Customer).all()
|
||||
s.expunge_all()
|
||||
|
||||
# now we must fetch each customer account individually from API
|
||||
customers = OrderedDict()
|
||||
|
||||
def fetch(dbcust, i):
|
||||
if dbcust.card_number in customers:
|
||||
return # already fetched this one
|
||||
customer = self.api.get_customer(dbcust.card_number)
|
||||
if customer:
|
||||
customers[dbcust.card_number] = customer
|
||||
else:
|
||||
logger = log.warning if dbcust.account_holder else log.debug
|
||||
logger("could not fetch customer from CORE API: %s",
|
||||
dbcust.card_number)
|
||||
|
||||
self.progress_loop(fetch, db_customers,
|
||||
message="Fetching Customer data from CORE API")
|
||||
return list(customers.values())
|
||||
def get_core_members(self):
|
||||
return get_core_members(self.api, progress=self.progress)
|
||||
|
||||
|
||||
class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter):
|
||||
|
|
@ -115,26 +91,26 @@ class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter):
|
|||
]
|
||||
|
||||
def get_host_objects(self):
|
||||
return self.get_core_customers()
|
||||
return self.get_core_members()
|
||||
|
||||
def normalize_host_object(self, customer):
|
||||
def normalize_host_object(self, member):
|
||||
|
||||
# figure out the "account holder" person for the customer
|
||||
people = customer['customers']
|
||||
account_holders = [person for person in people
|
||||
if person['accountHolder']]
|
||||
# figure out the "account holder" customer for the member
|
||||
customers = member['customers']
|
||||
account_holders = [customer for customer in customers
|
||||
if customer['accountHolder']]
|
||||
if len(account_holders) > 1:
|
||||
log.warning("customer %s has %s account holders in CORE: %s",
|
||||
customer['cardNo'], len(account_holders), customer)
|
||||
log.warning("member %s has %s account holders in CORE: %s",
|
||||
member['cardNo'], len(account_holders), member)
|
||||
elif not account_holders:
|
||||
raise NotImplementedError("TODO: how to handle customer with no account holders?")
|
||||
person = account_holders[0]
|
||||
raise NotImplementedError("TODO: how to handle member with no account holders?")
|
||||
customer = account_holders[0]
|
||||
|
||||
return {
|
||||
'id': customer['customerAccountID'],
|
||||
'number': customer['cardNo'],
|
||||
'name': normalize_full_name(person['firstName'],
|
||||
person['lastName']),
|
||||
'id': member['customerAccountID'],
|
||||
'number': member['cardNo'],
|
||||
'name': normalize_full_name(customer['firstName'],
|
||||
customer['lastName']),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -160,36 +136,36 @@ class PersonImporter(FromCOREPOSAPI, corepos_importing.model.PersonImporter):
|
|||
|
||||
def get_host_objects(self):
|
||||
|
||||
# first get all customer data from CORE API
|
||||
customers = self.get_core_customers()
|
||||
# first get all member data from CORE API
|
||||
members = self.get_core_members()
|
||||
normalized = []
|
||||
|
||||
# then collect all the "person" records
|
||||
def normalize(customer, i):
|
||||
normalized.extend(self.get_person_objects_for_customer(customer))
|
||||
def normalize(member, i):
|
||||
normalized.extend(self.get_person_objects_for_member(member))
|
||||
|
||||
self.progress_loop(normalize, customers,
|
||||
self.progress_loop(normalize, members,
|
||||
message="Collecting Person data from CORE")
|
||||
return normalized
|
||||
|
||||
def get_person_objects_for_customer(self, customer):
|
||||
def get_person_objects_for_member(self, member):
|
||||
"""
|
||||
Return a list of Person data objects for the given Customer. This
|
||||
Return a list of Person data objects for the given Member. This
|
||||
logic is split out separately so that datasync can leverage it too.
|
||||
"""
|
||||
records = []
|
||||
people = []
|
||||
|
||||
# make sure we put the account holder first in the list!
|
||||
people = sorted(customer['customers'],
|
||||
key=lambda cust: 1 if cust['accountHolder'] else 0,
|
||||
reverse=True)
|
||||
customers = sorted(member['customers'],
|
||||
key=lambda cust: 1 if cust['accountHolder'] else 0,
|
||||
reverse=True)
|
||||
|
||||
for i, person in enumerate(people, 1):
|
||||
person = dict(person)
|
||||
for i, customer in enumerate(customers, 1):
|
||||
person = dict(customer)
|
||||
person['customer_person_ordinal'] = i
|
||||
records.append(person)
|
||||
people.append(person)
|
||||
|
||||
return records
|
||||
return people
|
||||
|
||||
def get_customer(self, id):
|
||||
if hasattr(self, 'customers'):
|
||||
|
|
@ -250,24 +226,24 @@ class CustomerPersonImporter(FromCOREPOSAPI, importing.model.CustomerPersonImpor
|
|||
|
||||
def get_host_objects(self):
|
||||
|
||||
# first get all customer data from CORE API
|
||||
customers = self.get_core_customers()
|
||||
# first get all member data from CORE API
|
||||
members = self.get_core_members()
|
||||
normalized = []
|
||||
|
||||
# then collect all customer/person combination records
|
||||
def normalize(customer, i):
|
||||
def normalize(member, i):
|
||||
# make sure we put the account holder first in the list!
|
||||
people = sorted(customer['customers'],
|
||||
key=lambda cust: 1 if cust['accountHolder'] else 0,
|
||||
reverse=True)
|
||||
for i, person in enumerate(people, 1):
|
||||
customers = sorted(member['customers'],
|
||||
key=lambda cust: 1 if cust['accountHolder'] else 0,
|
||||
reverse=True)
|
||||
for i, customer in enumerate(customers, 1):
|
||||
normalized.append({
|
||||
'customer_account_id': customer['customerAccountID'],
|
||||
'person_customer_id': person['customerID'],
|
||||
'customer_account_id': member['customerAccountID'],
|
||||
'person_customer_id': customer['customerID'],
|
||||
'ordinal': i,
|
||||
})
|
||||
|
||||
self.progress_loop(normalize, customers,
|
||||
self.progress_loop(normalize, members,
|
||||
message="Collecting CustomerPerson data from CORE")
|
||||
return normalized
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue