From bfc52a6fb3f6a56f5b5e5e5f3f46d6f67aa66b9c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Mon, 5 Jun 2023 20:45:45 -0500 Subject: [PATCH] Make `card_number` more central for CORE API -> Rattail importers let's track that as (effectively) `Customer.corepos_card_number` and use that when possible for importer key --- .../corepos/office/importing/db/csv.py | 6 +- rattail_corepos/corepos/util.py | 4 +- .../ae74c537ea51_add_customer_card_number.py | 39 ++++ rattail_corepos/db/model/people.py | 9 +- rattail_corepos/importing/corepos/api.py | 183 ++++++++++++------ rattail_corepos/importing/model.py | 3 +- 6 files changed, 179 insertions(+), 65 deletions(-) create mode 100644 rattail_corepos/db/alembic/versions/ae74c537ea51_add_customer_card_number.py diff --git a/rattail_corepos/corepos/office/importing/db/csv.py b/rattail_corepos/corepos/office/importing/db/csv.py index f2f77fb..753fe61 100644 --- a/rattail_corepos/corepos/office/importing/db/csv.py +++ b/rattail_corepos/corepos/office/importing/db/csv.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2020 Lance Edgar +# Copyright © 2010-2023 Lance Edgar # # This file is part of Rattail. # @@ -28,8 +28,8 @@ from corepos.db.office_op import model as corepos, Session as CoreSession from rattail.importing.handlers import FromFileHandler from rattail.importing.csv import FromCSVToSQLAlchemyMixin -from rattail_corepos.corepos.importing.db.model import ToCore -from rattail_corepos.corepos.importing.db.corepos import ToCoreHandler +from rattail_corepos.corepos.office.importing.db.model import ToCore +from rattail_corepos.corepos.office.importing.db.corepos import ToCoreHandler class FromCSVToCore(FromCSVToSQLAlchemyMixin, FromFileHandler, ToCoreHandler): diff --git a/rattail_corepos/corepos/util.py b/rattail_corepos/corepos/util.py index 6885066..14762ad 100644 --- a/rattail_corepos/corepos/util.py +++ b/rattail_corepos/corepos/util.py @@ -46,7 +46,9 @@ def get_core_members(config, api, progress=None): # first we fetch all customer records from CORE DB with app.short_session(factory=CoreSession) as s: - db_customers = s.query(corepos.CustData).all() + db_customers = s.query(corepos.CustData)\ + .order_by(corepos.CustData.card_number)\ + .all() s.expunge_all() # now we must fetch each customer account individually from API diff --git a/rattail_corepos/db/alembic/versions/ae74c537ea51_add_customer_card_number.py b/rattail_corepos/db/alembic/versions/ae74c537ea51_add_customer_card_number.py new file mode 100644 index 0000000..7f568c9 --- /dev/null +++ b/rattail_corepos/db/alembic/versions/ae74c537ea51_add_customer_card_number.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8; -*- +"""add customer.card_number + +Revision ID: ae74c537ea51 +Revises: d6a0f21a6a94 +Create Date: 2023-06-05 19:04:25.574077 + +""" + +# revision identifiers, used by Alembic. +revision = 'ae74c537ea51' +down_revision = 'd6a0f21a6a94' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +import rattail.db.types + + + +def upgrade(): + + # corepos_customer + op.alter_column('corepos_customer', 'corepos_account_id', + existing_type=sa.INTEGER(), + nullable=True) + op.add_column('corepos_customer', sa.Column('corepos_card_number', sa.Integer(), nullable=True)) + op.add_column('corepos_customer_version', sa.Column('corepos_card_number', sa.Integer(), autoincrement=False, nullable=True)) + + +def downgrade(): + + # corepos_customer + op.drop_column('corepos_customer_version', 'corepos_card_number') + op.drop_column('corepos_customer', 'corepos_card_number') + op.alter_column('corepos_customer', 'corepos_account_id', + existing_type=sa.INTEGER(), + nullable=False) diff --git a/rattail_corepos/db/model/people.py b/rattail_corepos/db/model/people.py index d63b4fa..a769d62 100644 --- a/rattail_corepos/db/model/people.py +++ b/rattail_corepos/db/model/people.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2020 Lance Edgar +# Copyright © 2010-2023 Lance Edgar # # This file is part of Rattail. # @@ -90,14 +90,19 @@ class CoreCustomer(model.Base): Reference to the CORE-POS extension record for this customer. """)) - corepos_account_id = sa.Column(sa.Integer(), nullable=False, doc=""" + corepos_account_id = sa.Column(sa.Integer(), nullable=True, doc=""" ``Customers.customerAccountID`` value for this customer, within CORE-POS. """) + corepos_card_number = sa.Column(sa.Integer(), nullable=True, doc=""" + ``custdata.CardNo`` value for this customer, within CORE-POS. + """) + def __str__(self): return str(self.customer) CoreCustomer.make_proxy(model.Customer, '_corepos', 'corepos_account_id') +CoreCustomer.make_proxy(model.Customer, '_corepos', 'corepos_card_number') class CoreMember(model.Base): diff --git a/rattail_corepos/importing/corepos/api.py b/rattail_corepos/importing/corepos/api.py index 2fb2c32..3e3fadd 100644 --- a/rattail_corepos/importing/corepos/api.py +++ b/rattail_corepos/importing/corepos/api.py @@ -70,6 +70,12 @@ class FromCOREPOSToRattail(importing.ToRattailHandler): def get_default_keys(self): keys = super(FromCOREPOSToRattail, self).get_default_keys() + + # normally this one is redundant, but it can be used for + # double-check if desired + if 'CustomerPerson' in keys: + keys.remove('CustomerPerson') + if 'ProductMovement' in keys: keys.remove('ProductMovement') return keys @@ -99,10 +105,9 @@ class CustomerImporter(FromCOREPOSAPI, corepos_importing.model.CustomerImporter) """ Importer for customer data from CORE POS API. """ - key = 'corepos_account_id' + key = 'corepos_card_number' supported_fields = [ - 'corepos_account_id', - 'id', + 'corepos_card_number', 'number', 'name', 'address_street', @@ -116,11 +121,7 @@ class CustomerImporter(FromCOREPOSAPI, corepos_importing.model.CustomerImporter) return self.get_core_members() def normalize_host_object(self, member): - - if member['customerAccountID'] == 0: - log.debug("member %s has customerAccountID of 0: %s", - member['cardNo'], member) - return + card_number = int(member['cardNo']) # figure out the "account holder" customer for the member. note that # we only use this to determine the `Customer.name` in Rattail @@ -141,9 +142,8 @@ class CustomerImporter(FromCOREPOSAPI, corepos_importing.model.CustomerImporter) raise NotImplementedError("TODO: how to handle member with no customers?") return { - 'corepos_account_id': int(member['customerAccountID']), - 'id': member['customerAccountID'], - 'number': int(member['cardNo']), + 'corepos_card_number': card_number, + 'number': card_number, 'name': normalize_full_name(customer['firstName'], customer['lastName']), @@ -159,26 +159,37 @@ class PersonImporter(FromCOREPOSAPI, corepos_importing.model.PersonImporter): """ Importer for person data from CORE POS API. """ - key = 'corepos_customer_id' + key = ('customer_uuid', 'customer_person_ordinal') supported_fields = [ - 'corepos_customer_id', + 'customer_uuid', + 'customer_person_ordinal', 'first_name', 'last_name', 'display_name', - 'customer_uuid', - 'customer_person_ordinal', 'phone_number', 'phone_number_2', 'email_address', ] def setup(self): - super(PersonImporter, self).setup() - model = self.config.get_model() + super().setup() + model = self.model - self.customers = self.app.cache_model(self.session, - model.Customer, - key='id') + self.customers_by_card_number = self.app.cache_model( + self.session, + model.Customer, + key='corepos_card_number', + query_options=[orm.joinedload(model.Customer._corepos)]) + + self.ignore_new_members = self.should_ignore_new_members() + + def should_ignore_new_members(self): + if hasattr(self, 'ignore_new_members'): + return self.ignore_new_members + + return self.config.getbool('rattail_corepos', + 'importing_ignore_new_members', + default=False) def get_host_objects(self): @@ -216,46 +227,93 @@ class PersonImporter(FromCOREPOSAPI, corepos_importing.model.PersonImporter): if mixedup: raise NotImplementedError("TODO: should re-sort the customers list for member {}".format(member['cardNo'])) + ignore_new_members = self.should_ignore_new_members() + for i, customer in enumerate(customers, 1): person = dict(customer) - person['customer_person_ordinal'] = i - people.append(person) + + if (ignore_new_members + and not customer['firstName'] + and customer['lastName'] == 'NEW MEMBER'): + + log.debug("ignoring new member #%s: %s", + member['cardNo'], member) + + else: + person['member_card_number'] = member['cardNo'] + person['customer_person_ordinal'] = i + people.append(person) return people - def get_customer(self, id): - if hasattr(self, 'customers'): - return self.customers.get(id) + def get_customer_by_card_number(self, card_number): + if hasattr(self, 'customers_by_card_number'): + return self.customers_by_card_number.get(card_number) - model = self.config.get_model() + model = self.model try: return self.session.query(model.Customer)\ - .filter(model.Customer.id == id)\ + .join(model.CoreCustomer)\ + .filter(model.CoreCustomer.corepos_card_number == card_number)\ .one() except orm.exc.NoResultFound: pass def normalize_host_object(self, person): + card_number = person['member_card_number'] - customer = self.get_customer(person['customerAccountID']) + customer = self.get_customer_by_card_number(card_number) if not customer: - log.warning("Rattail customer not found for customerAccountID: %s", - person['customerAccountID']) + log.warning("Rattail customer not found for CardNo %s: %s", + card_number, person) return - return { - 'corepos_customer_id': int(person['customerID']), + data = { + 'customer_uuid': customer.uuid, + 'customer_person_ordinal': person['customer_person_ordinal'], + 'first_name': person['firstName'], 'last_name': person['lastName'], 'display_name': normalize_full_name(person['firstName'], person['lastName']), - 'customer_uuid': customer.uuid, - 'customer_person_ordinal': person['customer_person_ordinal'], + 'phone_number': person['phone'] or None, 'phone_number_2': person['altPhone'] or None, 'email_address': person['email'] or None, } + # truncate phone number data if needed + if data['phone_number'] and len(data['phone_number']) > self.maxlen_phone_number: + log.warning("phone_number is too long (%s chars), will truncate to %s chars: %s", + len(data['phone_number']), + self.maxlen_phone_number, + data['phone_number']) + data['phone_number'] = data['phone_number'][:self.maxlen_phone_number] + if data['phone_number_2'] and len(data['phone_number_2']) > self.maxlen_phone_number: + log.warning("phone_number_2 is too long (%s chars), will truncate to %s chars: %s", + len(data['phone_number_2']), + self.maxlen_phone_number, + data['phone_number_2']) + data['phone_number_2'] = data['phone_number_2'][:self.maxlen_phone_number] + + # swap 1st and 2nd phone numbers if only latter has value + self.prioritize_2(data, 'phone_number') + + return data + + def normalize_local_object(self, person): + data = super().normalize_local_object(person) + if data: + + # ignore local Person records with no customer_uuid; + # otherwise importer will try to delete them, and/or + # they just cause "duplicate keys" noise + if 'customer_uuid' in self.key: + if not data['customer_uuid']: + return + + return data + class CustomerPersonImporter(FromCOREPOSAPI, importing.model.CustomerPersonImporter): """ @@ -747,7 +805,10 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp product = self.get_product(item) if not product: - log.warning("product not found for CORE vendor item: %s", item) + # just debug logging since this is a common scenario; the + # CORE table is for items "available from vendor" but not + # necssarily items carried by store + log.debug("product not found for CORE vendor item: %s", item) return core_product = self.get_corepos_product(item) @@ -760,7 +821,12 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp preferred = True case_size = decimal.Decimal(item['units']) - unit_cost = decimal.Decimal(item['cost']) + unit_cost = item.get('cost') + if unit_cost is not None: + unit_cost = decimal.Decimal(unit_cost) + case_cost = None + if unit_cost is not None: + case_cost = unit_cost * case_size return { 'corepos_id': int(item['vendorItemID']), @@ -768,7 +834,7 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp 'vendor_uuid': vendor.uuid, 'code': (item['sku'] or '').strip() or None, 'case_size': case_size, - 'case_cost': case_size * unit_cost, + 'case_cost': case_cost, 'unit_cost': unit_cost, 'preferred': preferred, } @@ -778,11 +844,9 @@ class MemberImporter(FromCOREPOSAPI, corepos_importing.model.MemberImporter): """ Importer for member data from CORE POS API. """ - key = 'corepos_account_id' + key = 'number' supported_fields = [ - 'corepos_account_id', 'number', - 'id', 'customer_uuid', 'person_uuid', 'joined', @@ -802,20 +866,22 @@ class MemberImporter(FromCOREPOSAPI, corepos_importing.model.MemberImporter): ] def setup(self): - super(MemberImporter, self).setup() - model = self.config.get_model() - self.customers = self.app.cache_model(self.session, - model.Customer, - key='number') + super().setup() + model = self.model + + self.customers_by_number = self.app.cache_model( + self.session, + model.Customer, + key='number') def get_host_objects(self): return self.get_core_members() - def get_customer(self, number): - if hasattr(self, 'customers'): - return self.customers.get(number) + def get_customer_by_number(self, number): + if hasattr(self, 'customers_by_number'): + return self.customers_by_number.get(number) - model = self.config.get_model() + model = self.model try: return self.session.query(model.Customer)\ .filter(model.Customer.number == number)\ @@ -824,26 +890,27 @@ class MemberImporter(FromCOREPOSAPI, corepos_importing.model.MemberImporter): pass def normalize_host_object(self, member): - customer = self.get_customer(member['cardNo']) + card_number = member['cardNo'] + customer = self.get_customer_by_number(card_number) if not customer: log.warning("Rattail customer not found for cardNo %s: %s", - member['cardNo'], member) + card_number, member) return person = customer.first_person() if not person: log.warning("Rattail person not found for cardNo %s: %s", - member['cardNo'], member) + card_number, member) return if member['memberStatus'] in self.non_member_status_codes: log.debug("skipping non-member %s with status '%s': %s", - member['memberStatus'], member['cardNo'], member) + member['memberStatus'], card_number, member) return if member['memberStatus'] not in self.member_status_codes: # note that we will still import this one! we don't skip it log.warning("unexpected status '%s' for member %s: %s", - member['memberStatus'], member['cardNo'], member) + member['memberStatus'], card_number, member) joined = None if member['startDate'] and member['startDate'] != '0000-00-00 00:00:00': @@ -852,15 +919,15 @@ class MemberImporter(FromCOREPOSAPI, corepos_importing.model.MemberImporter): joined = joined.date() withdrew = None - if member['endDate'] and member['endDate'] != '0000-00-00 00:00:00': + if (member['endDate'] + and member['endDate'] != '0000-00-00 00:00:00' + and member['endDate'] != '1900-01-01 00:00:00'): withdrew = datetime.datetime.strptime(member['endDate'], '%Y-%m-%d %H:%M:%S') withdrew = withdrew.date() return { - 'corepos_account_id': int(member['customerAccountID']), - 'number': int(member['cardNo']), - 'id': str(member['customerAccountID']), + 'number': card_number, 'customer_uuid': customer.uuid, 'person_uuid': person.uuid, 'joined': joined, diff --git a/rattail_corepos/importing/model.py b/rattail_corepos/importing/model.py index 58b12ef..0529677 100644 --- a/rattail_corepos/importing/model.py +++ b/rattail_corepos/importing/model.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2022 Lance Edgar +# Copyright © 2010-2023 Lance Edgar # # This file is part of Rattail. # @@ -59,6 +59,7 @@ class CustomerImporter(importing.model.CustomerImporter): extensions = { '_corepos': [ 'corepos_account_id', + 'corepos_card_number', ], }