Oerhaul the Vendor import/export between Rattail and CORE

also, add new DB schema specific to this integration, to hold PKs etc.

rattail_corepos/config.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2018 Lance Edgar
+#  Copyright © 2010-2020 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -24,8 +24,6 @@
 Rattail-COREPOS Config Extension
 """
 
-from __future__ import unicode_literals, absolute_import
-
 from rattail.config import ConfigExtension
 from rattail.db.config import get_engines
 
@@ -37,8 +35,8 @@ class RattailCOREPOSExtension(ConfigExtension):
     key = 'rattail-corepos'
 
     def configure(self, config):
-        from corepos.db import Session as CoreSession
+        from corepos.db.office_op import Session as CoreSession
-        from corepos.trans.db import Session as CoreTransSession
+        from corepos.db.office_trans import Session as CoreTransSession
 
         engines = get_engines(config, section='corepos.db.office_op')
         config.corepos_engines = engines

rattail_corepos/corepos/importing/model.py

@@ -96,8 +96,7 @@ class VendorImporter(ToCoreAPI):
         'halfCases',
     ]
     # TODO: this importer is in a bit of an experimental state at the moment.
-    # we only allow "update" b/c it will use the API instead of direct DB
+    # we only allow create/update b/c it will use the API instead of direct DB
-    allow_create = False
     allow_delete = False
 
     def get_local_objects(self, host_data=None):
@@ -122,6 +121,10 @@ class VendorImporter(ToCoreAPI):
 
         return data
 
+    def create_object(self, key, data):
+        # we can get away with using the same logic for both here
+        return self.update_object(None, data)
+
     def update_object(self, vendor, data, local_data=None):
         """
         Push an update for the vendor, via the CORE API.
@@ -130,5 +133,5 @@ class VendorImporter(ToCoreAPI):
             return data
 
         vendorID = data.pop('vendorID')
-        self.api.set_vendor(vendorID, **data)
+        vendor = self.api.set_vendor(vendorID, **data)
-        return data
+        return vendor

rattail_corepos/corepos/importing/rattail.py

@@ -28,8 +28,10 @@ import logging
 
 from rattail import importing
 from rattail.db import model
+from rattail.db.util import short_session
 from rattail.util import OrderedDict
 from rattail_corepos.corepos import importing as corepos_importing
+from rattail_corepos.corepos.util import get_max_existing_vendor_id
 
 
 log = logging.getLogger(__name__)
@@ -71,17 +73,31 @@ class VendorImporter(FromRattail, corepos_importing.model.VendorImporter):
         'email',
     ]
 
+    def setup(self):
+        super(VendorImporter, self).setup()
+
+        # self.max_existing_vendor_id = self.get_max_existing_vendor_id()
+        self.max_existing_vendor_id = get_max_existing_vendor_id()
+        self.last_vendor_id = self.max_existing_vendor_id
+
+    def get_next_vendor_id(self):
+        if hasattr(self, 'last_vendor_id'):
+            self.last_vendor_id += 1
+            return self.last_vendor_id
+
+        last_vendor_id = get_max_existing_vendor_id()
+        return last_vendor_id + 1
+
     def normalize_host_object(self, vendor):
-        if not vendor.id or not vendor.id.isdigit():
+        vendor_id = vendor.corepos_id
-            log.warning("vendor %s has incompatible ID value: %s",
+        if not vendor_id:
-                        vendor.uuid, vendor.id)
+            vendor_id = self.get_next_vendor_id()
-            return
 
         data = {
-            'vendorID': vendor.id,
+            'vendorID': str(vendor_id),
             'vendorName': vendor.name,
-            'vendorAbbreviation': vendor.abbreviation,
+            'vendorAbbreviation': vendor.abbreviation or '',
-            'discountRate': float(vendor.special_discount),
+            'discountRate': float(vendor.special_discount or 0),
         }
 
         if 'phone' in self.fields:
@@ -98,4 +114,24 @@ class VendorImporter(FromRattail, corepos_importing.model.VendorImporter):
             email = vendor.email
             data['email'] = email.address if email else ''
 
+        # also embed original Rattail vendor object, if we'll be needing to
+        # update it later with a new CORE ID
+        if not vendor.corepos_id:
+            data['_rattail_vendor'] = vendor
+
         return data
+
+    def create_object(self, key, data):
+
+        # grab vendor object we (maybe) stashed when normalizing
+        rattail_vendor = data.pop('_rattail_vendor', None)
+
+        # do normal create logic
+        vendor = super(VendorImporter, self).create_object(key, data)
+        if vendor:
+
+            # maybe set the CORE ID for vendor in Rattail
+            if rattail_vendor:
+                rattail_vendor.corepos_id = int(vendor['vendorID'])
+
+            return vendor

rattail_corepos/corepos/util.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2020 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+CORE-POS misc. utilities
+"""
+
+import sqlalchemy as sa
+
+from corepos.db.office_op import Session as CoreSession, model as corepos
+
+from rattail.db.util import short_session
+
+
+def get_max_existing_vendor_id(session=None):
+    """
+    Returns the "last" (max) existing value for the ``vendors.vendorID``
+    column, for use when creating new records, since it is not auto-increment.
+    """
+    with short_session(Session=CoreSession, session=session) as s:
+        return s.query(sa.func.max(corepos.Vendor.id))\
+                .scalar() or 0

rattail_corepos/datasync/corepos.py

@@ -160,8 +160,10 @@ class FromRattailToCore(NewDataSyncImportConsumer):
         ]
         for change in [c for c in changes if c.payload_type in types]:
             if change.payload_type == 'Vendor' and change.deletion:
-                # just do default logic for this one
+                # # just do default logic for this one
-                self.invoke_importer(session, change)
+                # self.invoke_importer(session, change)
+                # TODO: we have no way to delete a CORE vendor via API, right?
+                pass
             else: # we consider this a "vendor add/update"
                 vendor = self.get_host_vendor(session, change)
                 if vendor:

rattail_corepos/db/alembic/versions/b43e93d32275_initial_core_pos_integration_tables.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""initial CORE-POS integration tables
+
+Revision ID: b43e93d32275
+Revises: dfc1ed686f3f
+Create Date: 2020-03-04 14:21:23.625568
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'b43e93d32275'
+down_revision = None
+branch_labels = ('rattail_corepos',)
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+import rattail.db.types
+
+
+
+def upgrade():
+
+    # corepos_vendor
+    op.create_table('corepos_vendor',
+                    sa.Column('uuid', sa.String(length=32), nullable=False),
+                    sa.Column('corepos_id', sa.Integer(), nullable=False),
+                    sa.ForeignKeyConstraint(['uuid'], ['vendor.uuid'], name='corepos_vendor_fk_vendor'),
+                    sa.PrimaryKeyConstraint('uuid')
+    )
+    op.create_table('corepos_vendor_version',
+                    sa.Column('uuid', sa.String(length=32), autoincrement=False, nullable=False),
+                    sa.Column('corepos_id', sa.Integer(), autoincrement=False, nullable=True),
+                    sa.Column('transaction_id', sa.BigInteger(), autoincrement=False, nullable=False),
+                    sa.Column('end_transaction_id', sa.BigInteger(), nullable=True),
+                    sa.Column('operation_type', sa.SmallInteger(), nullable=False),
+                    sa.PrimaryKeyConstraint('uuid', 'transaction_id')
+    )
+    op.create_index(op.f('ix_corepos_vendor_version_end_transaction_id'), 'corepos_vendor_version', ['end_transaction_id'], unique=False)
+    op.create_index(op.f('ix_corepos_vendor_version_operation_type'), 'corepos_vendor_version', ['operation_type'], unique=False)
+    op.create_index(op.f('ix_corepos_vendor_version_transaction_id'), 'corepos_vendor_version', ['transaction_id'], unique=False)
+
+
+def downgrade():
+
+    # corepos_vendor
+    op.drop_index(op.f('ix_corepos_vendor_version_transaction_id'), table_name='corepos_vendor_version')
+    op.drop_index(op.f('ix_corepos_vendor_version_operation_type'), table_name='corepos_vendor_version')
+    op.drop_index(op.f('ix_corepos_vendor_version_end_transaction_id'), table_name='corepos_vendor_version')
+    op.drop_table('corepos_vendor_version')
+    op.drop_table('corepos_vendor')

rattail_corepos/db/model.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2020 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Database schema extensions for CORE-POS integration
+"""
+
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from rattail.db import model
+
+
+__all__ = ['CoreVendor']
+
+
+class CoreVendor(model.Base):
+    """
+    CORE-specific extensions to :class:`rattail:rattail.db.model.Vendor`.
+    """
+    __tablename__ = 'corepos_vendor'
+    __table_args__ = (
+        sa.ForeignKeyConstraint(['uuid'], ['vendor.uuid'],
+                                name='corepos_vendor_fk_vendor'),
+    )
+    __versioned__ = {}
+
+    uuid = model.uuid_column(default=None)
+    vendor = orm.relationship(
+        model.Vendor,
+        doc="""
+        Reference to the actual vendor record, which this one extends.
+        """,
+        backref=orm.backref(
+            '_corepos',
+            uselist=False,
+            cascade='all, delete-orphan',
+            doc="""
+            Reference to the CORE-POS extension record for this vendor.
+            """))
+
+    corepos_id = sa.Column(sa.Integer(), nullable=False, doc="""
+    ``vendorID`` value for the vendor, within CORE-POS.
+    """)
+
+    def __str__(self):
+        return str(self.vendor)
+
+CoreVendor.make_proxy(model.Vendor, '_corepos', 'corepos_id')

rattail_corepos/importing/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2020 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Importing data into Rattail
+"""
+
+from . import model

rattail_corepos/importing/corepos.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2018 Lance Edgar
+#  Copyright © 2010-2020 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -24,18 +24,14 @@
 CORE POS -> Rattail data importing
 """
 
-from __future__ import unicode_literals, absolute_import
-
 import decimal
 
-import six
+from corepos.db.office_op import model as corepos, Session as CoreSession
-
-from corepos.db import model as corepos
-from corepos.db import Session as CoreSession
 
 from rattail import importing
 from rattail.gpc import GPC
 from rattail.util import OrderedDict
+from rattail_corepos import importing as corepos_importing
 
 
 class FromCOREPOSToRattail(importing.FromSQLAlchemyHandler, importing.ToRattailHandler):
@@ -66,14 +62,14 @@ class FromCOREPOS(importing.FromSQLAlchemy):
     """
 
 
-class VendorImporter(FromCOREPOS, importing.model.VendorImporter):
+class VendorImporter(FromCOREPOS, corepos_importing.model.VendorImporter):
     """
     Importer for vendor data from CORE POS.
     """
     host_model_class = corepos.Vendor
-    key = 'id'
+    key = 'corepos_id'
     supported_fields = [
-        'id',
+        'corepos_id',
         'name',
         'abbreviation',
         'special_discount',
@@ -82,6 +78,26 @@ class VendorImporter(FromCOREPOS, importing.model.VendorImporter):
         'email_address',
     ]
 
+    def cache_query(self):
+        """
+        Return the query to be used when caching "local" data.
+        """
+        # can't just use rattail.db.model b/c the CoreVendor would normally not
+        # be in there!  this still requires custom model to be configured though.
+        model = self.config.get_model()
+
+        # first get default query
+        query = super(VendorImporter, self).cache_query()
+
+        # maybe filter a bit, to ensure only "relevant" records are involved
+        if 'corepos_id' in self.key:
+            # note, the filter is probably redundant since we INNER JOIN on the
+            # extension table, and it doesn't allow null ID values.  but clarity.
+            query = query.join(model.CoreVendor)\
+                         .filter(model.CoreVendor.corepos_id != None)
+
+        return query
+
     def normalize_host_object(self, vendor):
 
         special_discount = None
@@ -89,9 +105,9 @@ class VendorImporter(FromCOREPOS, importing.model.VendorImporter):
             special_discount = decimal.Decimal('{:0.3f}'.format(vendor.discount_rate))
 
         return {
-            'id': six.text_type(vendor.id),
+            'corepos_id': vendor.id,
             'name': vendor.name,
-            'abbreviation': vendor.abbreviation,
+            'abbreviation': vendor.abbreviation or None,
             'special_discount': special_discount,
             'phone_number': vendor.phone or None,
             'fax_number': vendor.fax or None,

rattail_corepos/importing/model.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2020 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Rattail model importer extensions, for CORE-POS integration
+"""
+
+from rattail import importing
+
+
+##############################
+# core importer overrides
+##############################
+
+class VendorImporter(importing.model.VendorImporter):
+
+    extension_attr = '_corepos'
+    extension_fields = [
+        'corepos_id',
+    ]