Add cache table, importer for NationBuilder People

rattail_nationbuilder/commands.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+rattail-nationbuilder commands
+"""
+
+from rattail import commands
+
+
+class ImportNationBuilder(commands.ImportSubcommand):
+    """
+    Import data for NationBuilder => Rattail
+    """
+    name = 'import-nationbuilder'
+    description = __doc__.strip()
+    handler_key = 'to_rattail.from_nationbuilder.import'

rattail_nationbuilder/config.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Config Extension
+"""
+
+from rattail.config import ConfigExtension
+
+
+class RattailNationBuilderExtension(ConfigExtension):
+    """
+    Config extension for rattail-nationbuilder
+    """
+    key = 'rattail_nationbuilder'
+
+    def configure(self, config):
+
+        # rattail import-nationbuilder
+        config.setdefault('rattail.importing', 'to_rattail.from_nationbulder.import.default_handler',
+                          'rattail_nationbuilder.importing.nationbuilder:FromNationBuilderToRattail')
+        config.setdefault('rattail.importing', 'to_rattail.from_nationbuilder.import.default_cmd',
+                          'rattail import-nationbuilder')

rattail_nationbuilder/db/alembic/versions/1e17031c4b3e_first_cache_tables.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8; -*-
+"""first cache tables
+
+Revision ID: 1e17031c4b3e
+Revises: fa3aec1556bc
+Create Date: 2023-09-12 15:05:08.853989
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '1e17031c4b3e'
+down_revision = None
+branch_labels = ('rattail_nationbuilder',)
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+import rattail.db.types
+
+
+
+def upgrade():
+
+    # nationbuilder_cache_person
+    op.create_table('nationbuilder_cache_person',
+                    sa.Column('uuid', sa.String(length=32), nullable=False),
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('created_at', sa.DateTime(), nullable=True),
+                    sa.Column('email', sa.String(length=255), nullable=True),
+                    sa.Column('email_opt_in', sa.Boolean(), nullable=True),
+                    sa.Column('external_id', sa.String(length=50), nullable=True),
+                    sa.Column('first_name', sa.String(length=100), nullable=True),
+                    sa.Column('middle_name', sa.String(length=100), nullable=True),
+                    sa.Column('last_name', sa.String(length=100), nullable=True),
+                    sa.Column('mobile', sa.String(length=50), nullable=True),
+                    sa.Column('mobile_opt_in', sa.Boolean(), nullable=True),
+                    sa.Column('note', sa.Text(), nullable=True),
+                    sa.Column('phone', sa.String(length=50), nullable=True),
+                    sa.Column('primary_image_url_ssl', sa.String(length=255), nullable=True),
+                    sa.Column('signup_type', sa.Integer(), nullable=True),
+                    sa.Column('primary_address_address1', sa.String(length=100), nullable=True),
+                    sa.Column('primary_address_address2', sa.String(length=100), nullable=True),
+                    sa.Column('primary_address_city', sa.String(length=100), nullable=True),
+                    sa.Column('primary_address_state', sa.String(length=50), nullable=True),
+                    sa.Column('primary_address_zip', sa.String(length=10), nullable=True),
+                    sa.Column('tags', sa.Text(), nullable=True),
+                    sa.Column('updated_at', sa.DateTime(), nullable=True),
+                    sa.PrimaryKeyConstraint('uuid'),
+                    sa.UniqueConstraint('id', name='nationbuilder_cache_person_uq_id')
+                    )
+    op.create_table('nationbuilder_cache_person_version',
+                    sa.Column('uuid', sa.String(length=32), autoincrement=False, nullable=False),
+                    sa.Column('id', sa.Integer(), autoincrement=False, nullable=True),
+                    sa.Column('created_at', sa.DateTime(), autoincrement=False, nullable=True),
+                    sa.Column('email', sa.String(length=255), autoincrement=False, nullable=True),
+                    sa.Column('email_opt_in', sa.Boolean(), autoincrement=False, nullable=True),
+                    sa.Column('external_id', sa.String(length=50), autoincrement=False, nullable=True),
+                    sa.Column('first_name', sa.String(length=100), autoincrement=False, nullable=True),
+                    sa.Column('middle_name', sa.String(length=100), autoincrement=False, nullable=True),
+                    sa.Column('last_name', sa.String(length=100), autoincrement=False, nullable=True),
+                    sa.Column('mobile', sa.String(length=50), autoincrement=False, nullable=True),
+                    sa.Column('mobile_opt_in', sa.Boolean(), autoincrement=False, nullable=True),
+                    sa.Column('note', sa.Text(), autoincrement=False, nullable=True),
+                    sa.Column('phone', sa.String(length=50), autoincrement=False, nullable=True),
+                    sa.Column('primary_image_url_ssl', sa.String(length=255), autoincrement=False, nullable=True),
+                    sa.Column('signup_type', sa.Integer(), autoincrement=False, nullable=True),
+                    sa.Column('primary_address_address1', sa.String(length=100), autoincrement=False, nullable=True),
+                    sa.Column('primary_address_address2', sa.String(length=100), autoincrement=False, nullable=True),
+                    sa.Column('primary_address_city', sa.String(length=100), autoincrement=False, nullable=True),
+                    sa.Column('primary_address_state', sa.String(length=50), autoincrement=False, nullable=True),
+                    sa.Column('primary_address_zip', sa.String(length=10), autoincrement=False, nullable=True),
+                    sa.Column('tags', sa.Text(), autoincrement=False, nullable=True),
+                    sa.Column('updated_at', sa.DateTime(), autoincrement=False, nullable=True),
+                    sa.Column('transaction_id', sa.BigInteger(), autoincrement=False, nullable=False),
+                    sa.Column('end_transaction_id', sa.BigInteger(), nullable=True),
+                    sa.Column('operation_type', sa.SmallInteger(), nullable=False),
+                    sa.PrimaryKeyConstraint('uuid', 'transaction_id')
+                    )
+    op.create_index(op.f('ix_nationbuilder_cache_person_version_end_transaction_id'), 'nationbuilder_cache_person_version', ['end_transaction_id'], unique=False)
+    op.create_index(op.f('ix_nationbuilder_cache_person_version_operation_type'), 'nationbuilder_cache_person_version', ['operation_type'], unique=False)
+    op.create_index(op.f('ix_nationbuilder_cache_person_version_transaction_id'), 'nationbuilder_cache_person_version', ['transaction_id'], unique=False)
+
+
+def downgrade():
+
+    # nationbuilder_cache_person
+    op.drop_index(op.f('ix_nationbuilder_cache_person_version_transaction_id'), table_name='nationbuilder_cache_person_version')
+    op.drop_index(op.f('ix_nationbuilder_cache_person_version_operation_type'), table_name='nationbuilder_cache_person_version')
+    op.drop_index(op.f('ix_nationbuilder_cache_person_version_end_transaction_id'), table_name='nationbuilder_cache_person_version')
+    op.drop_table('nationbuilder_cache_person_version')
+    op.drop_table('nationbuilder_cache_person')

rattail_nationbuilder/db/model/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+DB schema for NationBuilder integration
+"""
+
+from .nationbuilder import NationBuilderCachePerson

rattail_nationbuilder/db/model/nationbuilder.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+NationBuilder cache tables
+"""
+
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from rattail.db import model
+from rattail.db.util import normalize_full_name
+from rattail.config import parse_list
+
+
+class NationBuilderCachePerson(model.Base):
+    """
+    Represents a Person record in NationBuilder.
+
+    https://apiexplorer.nationbuilder.com/nationbuilder#People
+    """
+    __tablename__ = 'nationbuilder_cache_person'
+    __table_args__ = (
+        sa.UniqueConstraint('id', name='nationbuilder_cache_person_uq_id'),
+    )
+    __versioned__ = {}
+    model_title = "NationBuilder Person"
+    model_title_plural = "NationBuilder People"
+
+    uuid = model.uuid_column()
+
+    id = sa.Column(sa.Integer(), nullable=False)
+    signup_type = sa.Column(sa.Integer(), nullable=True)
+    external_id = sa.Column(sa.String(length=50), nullable=True)
+    tags = sa.Column(sa.String(length=255), nullable=True)
+    first_name = sa.Column(sa.String(length=100), nullable=True)
+    middle_name = sa.Column(sa.String(length=100), nullable=True)
+    last_name = sa.Column(sa.String(length=100), nullable=True)
+    email = sa.Column(sa.String(length=255), nullable=True)
+    email_opt_in = sa.Column(sa.Boolean(), nullable=True)
+    mobile = sa.Column(sa.String(length=50), nullable=True)
+    mobile_opt_in = sa.Column(sa.Boolean(), nullable=True)
+    phone = sa.Column(sa.String(length=50), nullable=True)
+    primary_address_address1 = sa.Column(sa.String(length=100), nullable=True)
+    primary_address_address2 = sa.Column(sa.String(length=100), nullable=True)
+    primary_address_city = sa.Column(sa.String(length=100), nullable=True)
+    primary_address_state = sa.Column(sa.String(length=50), nullable=True)
+    primary_address_zip = sa.Column(sa.String(length=10), nullable=True)
+    primary_image_url_ssl = sa.Column(sa.String(length=255), nullable=True)
+    note = sa.Column(sa.Text(), nullable=True)
+    created_at = sa.Column(sa.DateTime(), nullable=True)
+    updated_at = sa.Column(sa.DateTime(), nullable=True)
+
+    def __str__(self):
+        return normalize_full_name(self.first_name, self.last_name)
+
+    def has_tag(self, tag):
+        if self.tags:
+            for value in parse_list(self.tags):
+                if value == tag:
+                    return True
+        return False

rattail_nationbuilder/importing/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+rattail-nationbuilder importing
+"""
+
+from . import model

rattail_nationbuilder/importing/model.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+rattail-nationbuilder model importers
+"""
+
+from rattail.importing.model import ToRattail
+from rattail_nationbuilder.db import model
+
+
+##############################
+# nationbuilder cache
+##############################
+
+class NationBuilderCachePersonImporter(ToRattail):
+    model_class = model.NationBuilderCachePerson

rattail_nationbuilder/importing/nationbuilder.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+NationBuilder -> Rattail importing
+"""
+
+import datetime
+from collections import OrderedDict
+
+from rattail import importing
+from rattail_nationbuilder import importing as nationbuilder_importing
+from rattail_nationbuilder.nationbuilder.webapi import NationBuilderWebAPI
+
+
+class FromNationBuilderToRattail(importing.ToRattailHandler):
+    """
+    Import handler for NationBuilder -> Rattail
+    """
+    host_key = 'nationbuilder'
+    host_title = "NationBuilder"
+    generic_host_title = "NationBuilder"
+
+    def get_importers(self):
+        importers = OrderedDict()
+        importers['NationBuilderCachePerson'] = NationBuilderCachePersonImporter
+        return importers
+
+
+class FromNationBuilder(importing.Importer):
+    """
+    Base class for all NationBuilder importers
+    """
+
+    def setup(self):
+        super().setup()
+        self.setup_api()
+
+    def setup_api(self):
+        self.nationbuilder = NationBuilderWebAPI(self.config)
+
+
+class NationBuilderCachePersonImporter(FromNationBuilder, nationbuilder_importing.model.NationBuilderCachePersonImporter):
+    """
+    Importer for NB Person cache
+    """
+    key = 'id'
+
+    primary_address_fields = [
+        'primary_address_address1',
+        'primary_address_address2',
+        'primary_address_city',
+        'primary_address_state',
+        'primary_address_zip',
+    ]
+
+    supported_fields = [
+        'id',
+        'created_at',
+        'email',
+        'email_opt_in',
+        'external_id',
+        'first_name',
+        'middle_name',
+        'last_name',
+        'mobile',
+        'mobile_opt_in',
+        'note',
+        'phone',
+        'primary_image_url_ssl',
+        'signup_type',
+        'tags',
+        'updated_at',
+    ] + primary_address_fields
+
+    def get_host_objects(self):
+        return self.nationbuilder.get_people(page_size=500)
+
+    def normalize_timestamp(self, value):
+        if not value:
+            return
+
+        dt = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S%z')
+        dt = self.app.localtime(dt)
+        return self.app.make_utc(dt)
+
+    def normalize_host_object(self, person):
+
+        # nb. some fields may not be present in person dict
+        data = dict([(field, person.get(field))
+                     for field in self.fields])
+        if data:
+
+            for field in ('created_at', 'updated_at'):
+                data[field] = self.normalize_timestamp(data[field])
+
+            if 'tags' in self.fields:
+                tags = data['tags']
+                if tags:
+                    data['tags'] = self.config.make_list_string(tags)
+
+            if self.fields_active(self.primary_address_fields):
+                address = person.get('primary_address')
+                if address:
+                    data.update({
+                        'primary_address_address1': address['address1'],
+                        'primary_address_address2': address['address2'],
+                        'primary_address_state': address['state'],
+                        'primary_address_city': address['city'],
+                        'primary_address_zip': address['zip'],
+                    })
+
+            return data

rattail_nationbuilder/importing/versions.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+rattail-nationbuilder version importing
+"""
+
+from rattail.importing import versions as base
+
+
+class NationBuilderVersionMixin(object):
+
+    def add_nationbuilder_importers(self, importers):
+        importers['NationBuilderCachePerson'] = NationBuilderCachePersonImporter
+        return importers
+
+
+class NationBuilderCachePersonImporter(base.VersionImporter):
+
+    @property
+    def host_model_class(self):
+        return self.model.NationBuilderCachePerson

rattail_nationbuilder/nationbuilder/util.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  Rattail -- Retail Software Framework
+#  Copyright © 2010-2023 Lance Edgar
+#
+#  This file is part of Rattail.
+#
+#  Rattail is free software: you can redistribute it and/or modify it under the
+#  terms of the GNU General Public License as published by the Free Software
+#  Foundation, either version 3 of the License, or (at your option) any later
+#  version.
+#
+#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+#  details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+NationBuilder utils
+"""
+
+
+def get_nationbuilder_url(config):
+    url = config.get('nationbuilder', 'url')
+    if url:
+        return url.rstrip('/')

rattail_nationbuilder/nationbuilder/webapi.py

@@ -89,7 +89,7 @@ class NationBuilderWebAPI(object):
         """
         return self._request('GET', api_method, params=params)
 
-    def get_people(self, page_size=10, progress=None, **kwargs):
+    def get_people(self, page_size=10, max_pages=None, progress=None, **kwargs):
         """
         Retrieve all Person records.
 
@@ -109,8 +109,12 @@ class NationBuilderWebAPI(object):
             data = response.json()
             people.extend(data['results'])
             url['next'] = data['next']
+            log.debug("have fetched %s pages", page + 1)
 
-        self.app.progress_loop(fetch, range(pages), progress,
+        pages = list(range(pages))
+        if max_pages:
+            pages = pages[:max_pages]
+        self.app.progress_loop(fetch, pages, progress,
                                message="Fetching Person data from NationBuilder")
         return people
 

setup.cfg

@@ -27,3 +27,15 @@ install_requires =
 
 packages = find:
 include_package_data = True
+
+
+[options.entry_points]
+
+rattail.commands =
+        import-nationbuilder = rattail_nationbuilder.commands:ImportNationBuilder
+
+rattail.config.extensions =
+        rattail_nationbuilder = rattail_nationbuilder.config:RattailNationBuilderExtension
+
+rattail.importing =
+        to_rattail.from_nationbuilder.import = rattail_nationbuilder.importing.nationbuilder:FromNationBuilderToRattail