fix: improve ProductCost sorting for import from CORE API

this hopefully ensures a more consistent preference order, fewer diffs

rattail_corepos/importing/corepos/api.py

@@ -655,9 +655,7 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp
     """
     Importer for product cost data from CORE POS API.
     """
-    # TODO: should change key after live sites are updated
+    key = ('corepos_vendor_id', 'corepos_sku')
-    key = ('vendor_uuid', 'code')
-    # key = ('corepos_vendor_id', 'corepos_sku')
     supported_fields = [
         'corepos_vendor_id',
         'corepos_sku',
@@ -696,7 +694,63 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp
                                                         key='item_id')
 
     def get_host_objects(self):
-        return self.api.get_vendor_items()
+
+        # first we will cache API products by upc
+        products = OrderedDict()
+
+        def cache(product, i):
+            if product.get('upc'):
+                products[product['upc']] = product
+
+        self.progress_loop(cache, self.api.get_products(),
+                           message="Caching product data from CORE")
+
+        # next we cache API vendor items, also by upc
+        vendor_items = {}
+
+        def cache(item, i):
+            if not item['upc']:
+                log.warning("CORE vendor item has no upc: %s", item)
+                return
+            if item['vendorID'] == '0':
+                log.warning("CORE vendor item has no vendorID: %s", item)
+                return
+            vendor_items.setdefault(item['upc'], []).append(item)
+
+        self.progress_loop(cache, self.api.get_vendor_items(),
+                           message="Caching vendor item data from CORE")
+
+        # now we must "sort" the vendor items for each upc.  to do
+        # this we just ensure the item for default vendor is first
+
+        def organize(upc, i):
+            product = products.get(upc)
+            if not product:
+                return          # product not found
+
+            vendor_id = product['default_vendor_id']
+            if not vendor_id:
+                return          # product has no default vendor
+
+            items = vendor_items[upc]
+            for item in items:
+                if item['vendorID'] == vendor_id:
+                    # found the default vendor item
+                    j = items.index(item)
+                    if j != 0:
+                        # it was not first; make it so
+                        items.pop(j)
+                        items.insert(0, item)
+                    break
+
+        self.progress_loop(organize, list(vendor_items),
+                           message="Sorting items by default vendor")
+
+        # keep the vendor item cache for reference later
+        self.api_vendor_items = vendor_items
+
+        # host objects are the API products (in original sequence)
+        return list(products.values())
 
     def get_vendor(self, item):
         corepos_id = int(item['vendorID'])
@@ -733,6 +787,31 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp
         except orm.exc.NoResultFound:
             pass
 
+    def normalize_host_data(self, host_objects=None):
+
+        # TODO: this all seems a bit hacky but works for now..
+        # could even be we don't need this method?
+
+        if host_objects is None:
+            host_objects = self.get_host_objects()
+        normalized = []
+        self.sorted_vendor_items = {}
+
+        def normalize(product, i):
+            if not product.get('upc'):
+                log.warning("product has no upc: %s", product)
+                return
+            items = self.sort_vendor_items(product)
+            self.sorted_vendor_items[product['upc']] = items
+            for item in items:
+                data = self.normalize_host_object(item)
+                if data:
+                    normalized.append(data)
+
+        self.progress_loop(normalize, host_objects,
+                           message=f"Reading Product data from {self.host_system_title}")
+        return normalized
+
     def normalize_host_object(self, item):
         vendor = self.get_vendor(item)
         if not vendor:
@@ -752,10 +831,6 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp
         #     log.warning("CORE POS product not found for item: %s", item)
         #     return
 
-        preferred = False
-        if core_product and core_product['default_vendor_id'] == item['vendorID']:
-            preferred = True
-
         case_size = decimal.Decimal(item['units'])
         unit_cost = item.get('cost')
         if unit_cost is not None:
@@ -764,7 +839,7 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp
         if unit_cost is not None:
             case_cost = unit_cost * case_size
 
-        return {
+        data = {
             'corepos_vendor_id': int(item['vendorID']),
             'corepos_sku': item['sku'],
             'product_uuid': product.uuid,
@@ -773,9 +848,35 @@ class ProductCostImporter(FromCOREPOSAPI, corepos_importing.model.ProductCostImp
             'case_size': case_size,
             'case_cost': case_cost,
             'unit_cost': unit_cost,
-            'preferred': preferred,
         }
 
+        if self.fields_active(['preference', 'preferred']):
+            items = self.get_sorted_vendor_items(item)
+            i = items.index(item)
+            data['preference'] = i + 1
+            data['preferred'] = i == 0
+
+        return data
+
+    def get_sorted_vendor_items(self, item):
+        if hasattr(self, 'sorted_vendor_items'):
+            return self.sorted_vendor_items.get(item['upc'])
+
+        product = self.api.get_product(item['upc'])
+        return self.sort_vendor_items(product)
+
+    def sort_vendor_items(self, product):
+
+        # TODO: this all seems a bit hacky but works for now..
+
+        if not product.get('upc'):
+            return []
+
+        if hasattr(self, 'api_vendor_items'):
+            return self.api_vendor_items.get(product['upc'], [])
+
+        raise NotImplementedError("must add real-time datasync support")
+
 
 class MembershipTypeImporter(FromCOREPOSAPI, importing.model.MembershipTypeImporter):
     """