rattail/rattail-corepos
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add support for Rattail -> CORE export/sync for Member data

Browse source 
also refactor CORE -> Rattail logic to use `api.set_member()` etc.
This commit is contained in:
Lance Edgar 2020-03-17 16:29:35 -05:00
parent 8d47a1449c
commit cb63644c7d
6 changed files with 326 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
rattail_corepos/corepos/importing/model.py
View file

@ -27,6 +27,8 @@ CORE-POS model importers (webservices API)
from corepos.api import CoreWebAPI
from rattail import importing
from rattail.util import data_diffs
from rattail_corepos.corepos.util import get_core_members
class ToCoreAPI(importing.Importer):
@ -74,6 +76,145 @@ class ToCoreAPI(importing.Importer):
data[field] = ''
class MemberImporter(ToCoreAPI):
"""
Member model importer for CORE-POS
"""
model_name = 'Member'
key = 'cardNo'
supported_fields = [
'cardNo'
'customerAccountID',
'customers',
# 'memberStatus',
# 'activeStatus',
# 'customerTypeID',
# 'chargeBalance',
# 'chargeLimit',
# 'idCardUPC',
# 'startDate',
# 'endDate',
# 'addressFirstLine',
# 'addressSecondLine',
# 'city',
# 'state',
# 'zip',
# 'contactAllowed',
# 'contactMethod',
# 'modified',
]
supported_customer_fields = [
'customerID',
# 'customerAccountID',
# 'cardNo',
'firstName',
'lastName',
# 'chargeAllowed',
# 'checksAllowed',
# 'discount',
'accountHolder',
# 'staff',
# 'phone',
# 'altPhone',
# 'email',
# 'memberPricingAllowed',
# 'memberCouponsAllowed',
# 'lowIncomeBenefits',
# 'modified',
]
def get_local_objects(self, host_data=None):
return get_core_members(self.api, progress=self.progress)
def get_single_local_object(self, key):
assert len(self.key) == 1
assert self.key[0] == 'cardNo'
return self.api.get_member(key[0])
def normalize_local_object(self, member):
data = dict(member)
return data
def data_diffs(self, local_data, host_data):
diffs = super(MemberImporter, self).data_diffs(local_data, host_data)
# the 'customers' field requires a more granular approach, since the
# data coming from API may have different fields than our local data
if 'customers' in self.fields and 'customers' in diffs:
if not self.customer_data_differs(local_data, host_data):
diffs.remove('customers')
return diffs
def customer_data_differs(self, local_data, host_data):
local_customers = local_data['customers']
host_customers = host_data['customers']
# if both are empty, we're good
if not local_customers and not host_customers:
return False
# obviously we differ if record count doesn't match
if len(local_customers) != len(host_customers):
return True
# okay then, let's traverse the "new" list
for host_customer in host_customers:
# we differ if can't locate corresponding "old" local record
local_customer = self.find_local_customer(local_customers, host_customer)
if not local_customer:
return True
# we differ if old and new records differ
if data_diffs(local_customer, host_customer,
fields=self.supported_customer_fields):
return True
# okay, now let's traverse the "old" list
for local_customer in local_customers:
# we differ if can't locate corresponding "new" host record
host_customer = self.find_host_customer(host_customers, local_customer)
if not host_customer:
return True
# guess we don't differ after all
return False
def find_local_customer(self, local_customers, host_customer):
assert 'customerID' in self.supported_customer_fields
if not host_customer['customerID']:
return # new customer
for local_customer in local_customers:
if local_customer['customerID'] == host_customer['customerID']:
return local_customer
def find_host_customer(self, host_customers, local_customer):
assert 'customerID' in self.supported_customer_fields
for host_customer in host_customers:
if host_customer['customerID'] == local_customer['customerID']:
return host_customer
def create_object(self, key, data):
# we can get away with using the same logic for both here
return self.update_object(None, data)
def update_object(self, member, data, local_data=None):
"""
Push an update for the member, via the CORE API.
"""
if self.dry_run:
return data
cardNo = data.pop('cardNo')
member = self.api.set_member(cardNo, **data)
return member
class DepartmentImporter(ToCoreAPI):
"""
Department model importer for CORE-POS

37
rattail_corepos/corepos/importing/rattail.py
View file

@ -47,6 +47,7 @@ class FromRattailToCore(importing.FromRattailHandler):
def get_importers(self):
importers = OrderedDict()
importers['Member'] = MemberImporter
importers['Department'] = DepartmentImporter
importers['Subdepartment'] = SubdepartmentImporter
importers['Vendor'] = VendorImporter
@ -60,6 +61,42 @@ class FromRattail(importing.FromSQLAlchemy):
"""
class MemberImporter(FromRattail, corepos_importing.model.MemberImporter):
"""
Member data exporter
"""
host_model_class = model.Customer
key = 'cardNo'
supported_fields = [
'cardNo',
'customerAccountID',
'customers',
]
supported_person_fields = [
'customerID',
'firstName',
'lastName',
'accountHolder',
]
def normalize_host_object(self, customer):
people = []
for i, person in enumerate(customer.people, 1):
people.append({
'customerID': str(person.corepos_customer_id),
'firstName': person.first_name,
'lastName': person.last_name,
'accountHolder': i == 1,
})
return {
'cardNo': customer.number,
'customerAccountID': customer.id,
'customers': people,
}
class DepartmentImporter(FromRattail, corepos_importing.model.DepartmentImporter):
"""
Department data exporter

37
rattail_corepos/corepos/util.py
View file

@ -24,11 +24,48 @@
CORE-POS misc. utilities
"""
import logging
import sqlalchemy as sa
from corepos.db.office_op import Session as CoreSession, model as corepos
from rattail.db.util import short_session
from rattail.util import OrderedDict, progress_loop
log = logging.getLogger(__name__)
def get_core_members(api, progress=None):
"""
Shared logic for fetching *all* customer accounts from CORE-POS API.
"""
# TODO: ideally could do this, but API doesn't let us fetch "all"
# return api.get_members()
# first we fetch all customer records from CORE DB
with short_session(Session=CoreSession) as s:
db_customers = s.query(corepos.Customer).all()
s.expunge_all()
# now we must fetch each customer account individually from API
members = OrderedDict()
def fetch(dbcust, i):
if dbcust.card_number in members:
return # already fetched this one
member = api.get_member(dbcust.card_number)
if member:
members[dbcust.card_number] = member
else:
logger = log.warning if dbcust.account_holder else log.debug
logger("could not fetch member from CORE API: %s",
dbcust.card_number)
progress_loop(fetch, db_customers, progress,
message="Fetching Member data from CORE API")
return list(members.values())
def get_max_existing_vendor_id(session=None):

78
rattail_corepos/datasync/corepos.py
View file

@ -152,21 +152,23 @@ class FromRattailToCore(NewDataSyncImportConsumer):
# also establish the API client for each!
importer.establish_api()
# sync all Department changes
# sync all Customer changes
types = [
'Department',
'Customer',
'Person',
'CustomerPerson',
]
for change in [c for c in changes if c.payload_type in types]:
if change.payload_type == 'Department' and change.deletion:
# TODO: we have no way (yet) to delete a CORE department via API
if change.payload_type == 'Customer' and change.deletion:
# # just do default logic for this one
# self.invoke_importer(session, change)
# TODO: we have no way to delete a CORE customer via API, right?
pass
else: # we consider this an "add/update"
department = self.get_department(session, change)
if department:
self.process_change(session, self.importers['Department'],
host_object=department)
customers = self.get_customers(session, change)
for customer in customers:
self.process_change(session, self.importers['Member'],
host_object=customer)
# sync all Vendor changes
types = [
@ -186,9 +188,53 @@ class FromRattailToCore(NewDataSyncImportConsumer):
self.process_change(session, self.importers['Vendor'],
host_object=vendor)
def get_department(self, session, change):
if change.payload_type == 'Department':
return session.query(model.Department).get(change.payload_key)
# sync all Product changes
types = [
'Product',
'ProductPrice',
]
for change in [c for c in changes if c.payload_type in types]:
if change.payload_type == 'Product' and change.deletion:
# # just do default logic for this one
# self.invoke_importer(session, change)
# TODO: we have no way to delete a CORE product via API, right?
pass
else: # we consider this an "add/update"
product = self.get_product(session, change)
if product:
self.process_change(session, self.importers['Product'],
host_object=product)
# process all remaining supported models with typical logic
types = [
'Department',
'Subdepartment',
]
for change in [c for c in changes if c.payload_type in types]:
self.invoke_importer(session, change)
def get_host_object(self, session, change):
return session.query(getattr(model, change.payload_type))\
.get(change.payload_key)
def get_customers(self, session, change):
if change.payload_type == 'Customer':
customer = session.query(model.Customer).get(change.payload_key)
if customer:
return [customer]
if change.payload_type == 'CustomerPerson':
cp = session.query(model.CustomerPerson).get(change.payload_key)
if cp:
return [cp.customer]
if change.payload_type == 'Person':
person = session.query(model.Person).get(change.payload_key)
if person:
return person.customers
return []
def get_vendor(self, session, change):
@ -204,3 +250,13 @@ class FromRattailToCore(NewDataSyncImportConsumer):
email = session.query(model.VendorEmailAddress).get(change.payload_key)
if email:
return email.vendor
def get_product(self, session, change):
if change.payload_type == 'Product':
return session.query(model.Product).get(change.payload_key)
if change.payload_type == 'ProductPrice':
price = session.query(model.ProductPrice).get(change.payload_key)
if price:
return price.product

2
rattail_corepos/datasync/rattail.py
View file

@ -86,7 +86,7 @@ class FromCOREAPIToRattail(NewDataSyncImportConsumer):
def get_host_object(self, session, change):
if change.payload_type == 'Customer':
return self.api.get_customer(change.payload_key)
return self.api.get_member(change.payload_key)
if change.payload_type == 'Department':
return self.api.get_department(change.payload_key)
if change.payload_type == 'Subdepartment':

102
rattail_corepos/importing/corepos/api.py
View file

@ -31,14 +31,14 @@ from sqlalchemy import orm
from sqlalchemy.orm.exc import NoResultFound
from corepos.api import CoreWebAPI
from corepos.db.office_op import Session as CoreSession, model as corepos
from rattail import importing
from rattail.db import model
from rattail.gpc import GPC
from rattail.util import OrderedDict
from rattail.db.util import normalize_full_name, short_session
from rattail.db.util import normalize_full_name
from rattail_corepos import importing as corepos_importing
from rattail_corepos.corepos.util import get_core_members
log = logging.getLogger(__name__)
@ -75,32 +75,8 @@ class FromCOREPOSAPI(importing.Importer):
url = self.config.require('corepos.api', 'url')
self.api = CoreWebAPI(url)
def get_core_customers(self):
# TODO: ideally could do this, but API doesn't let us fetch "all"
# return self.api.get_customers()
# first we fetch all customer records from CORE DB
with short_session(Session=CoreSession) as s:
db_customers = s.query(corepos.Customer).all()
s.expunge_all()
# now we must fetch each customer account individually from API
customers = OrderedDict()
def fetch(dbcust, i):
if dbcust.card_number in customers:
return # already fetched this one
customer = self.api.get_customer(dbcust.card_number)
if customer:
customers[dbcust.card_number] = customer
else:
logger = log.warning if dbcust.account_holder else log.debug
logger("could not fetch customer from CORE API: %s",
dbcust.card_number)
self.progress_loop(fetch, db_customers,
message="Fetching Customer data from CORE API")
return list(customers.values())
def get_core_members(self):
return get_core_members(self.api, progress=self.progress)
class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter):
@ -115,26 +91,26 @@ class CustomerImporter(FromCOREPOSAPI, importing.model.CustomerImporter):
]
def get_host_objects(self):
return self.get_core_customers()
return self.get_core_members()
def normalize_host_object(self, customer):
def normalize_host_object(self, member):
# figure out the "account holder" person for the customer
people = customer['customers']
account_holders = [person for person in people
if person['accountHolder']]
# figure out the "account holder" customer for the member
customers = member['customers']
account_holders = [customer for customer in customers
if customer['accountHolder']]
if len(account_holders) > 1:
log.warning("customer %s has %s account holders in CORE: %s",
customer['cardNo'], len(account_holders), customer)
log.warning("member %s has %s account holders in CORE: %s",
member['cardNo'], len(account_holders), member)
elif not account_holders:
raise NotImplementedError("TODO: how to handle customer with no account holders?")
person = account_holders[0]
raise NotImplementedError("TODO: how to handle member with no account holders?")
customer = account_holders[0]
return {
'id': customer['customerAccountID'],
'number': customer['cardNo'],
'name': normalize_full_name(person['firstName'],
person['lastName']),
'id': member['customerAccountID'],
'number': member['cardNo'],
'name': normalize_full_name(customer['firstName'],
customer['lastName']),
}
@ -160,36 +136,36 @@ class PersonImporter(FromCOREPOSAPI, corepos_importing.model.PersonImporter):
def get_host_objects(self):
# first get all customer data from CORE API
customers = self.get_core_customers()
# first get all member data from CORE API
members = self.get_core_members()
normalized = []
# then collect all the "person" records
def normalize(customer, i):
normalized.extend(self.get_person_objects_for_customer(customer))
def normalize(member, i):
normalized.extend(self.get_person_objects_for_member(member))
self.progress_loop(normalize, customers,
self.progress_loop(normalize, members,
message="Collecting Person data from CORE")
return normalized
def get_person_objects_for_customer(self, customer):
def get_person_objects_for_member(self, member):
"""
Return a list of Person data objects for the given Customer. This
Return a list of Person data objects for the given Member. This
logic is split out separately so that datasync can leverage it too.
"""
records = []
people = []
# make sure we put the account holder first in the list!
people = sorted(customer['customers'],
customers = sorted(member['customers'],
key=lambda cust: 1 if cust['accountHolder'] else 0,
reverse=True)
for i, person in enumerate(people, 1):
person = dict(person)
for i, customer in enumerate(customers, 1):
person = dict(customer)
person['customer_person_ordinal'] = i
records.append(person)
people.append(person)
return records
return people
def get_customer(self, id):
if hasattr(self, 'customers'):
@ -250,24 +226,24 @@ class CustomerPersonImporter(FromCOREPOSAPI, importing.model.CustomerPersonImpor
def get_host_objects(self):
# first get all customer data from CORE API
customers = self.get_core_customers()
# first get all member data from CORE API
members = self.get_core_members()
normalized = []
# then collect all customer/person combination records
def normalize(customer, i):
def normalize(member, i):
# make sure we put the account holder first in the list!
people = sorted(customer['customers'],
customers = sorted(member['customers'],
key=lambda cust: 1 if cust['accountHolder'] else 0,
reverse=True)
for i, person in enumerate(people, 1):
for i, customer in enumerate(customers, 1):
normalized.append({
'customer_account_id': customer['customerAccountID'],
'person_customer_id': person['customerID'],
'customer_account_id': member['customerAccountID'],
'person_customer_id': customer['customerID'],
'ordinal': i,
})
self.progress_loop(normalize, customers,
self.progress_loop(normalize, members,
message="Collecting CustomerPerson data from CORE")
return normalized