Add cache table, importer for NationBuilder People

This commit is contained in:
Lance Edgar 2023-09-12 18:15:23 -05:00
parent 93af35eb4e
commit c3b93edd4d
13 changed files with 562 additions and 2 deletions

View file

@ -0,0 +1,36 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
rattail-nationbuilder commands
"""
from rattail import commands
class ImportNationBuilder(commands.ImportSubcommand):
"""
Import data for NationBuilder => Rattail
"""
name = 'import-nationbuilder'
description = __doc__.strip()
handler_key = 'to_rattail.from_nationbuilder.import'

View file

@ -0,0 +1,42 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Config Extension
"""
from rattail.config import ConfigExtension
class RattailNationBuilderExtension(ConfigExtension):
"""
Config extension for rattail-nationbuilder
"""
key = 'rattail_nationbuilder'
def configure(self, config):
# rattail import-nationbuilder
config.setdefault('rattail.importing', 'to_rattail.from_nationbulder.import.default_handler',
'rattail_nationbuilder.importing.nationbuilder:FromNationBuilderToRattail')
config.setdefault('rattail.importing', 'to_rattail.from_nationbuilder.import.default_cmd',
'rattail import-nationbuilder')

View file

View file

@ -0,0 +1,91 @@
# -*- coding: utf-8; -*-
"""first cache tables
Revision ID: 1e17031c4b3e
Revises: fa3aec1556bc
Create Date: 2023-09-12 15:05:08.853989
"""
# revision identifiers, used by Alembic.
revision = '1e17031c4b3e'
down_revision = None
branch_labels = ('rattail_nationbuilder',)
depends_on = None
from alembic import op
import sqlalchemy as sa
import rattail.db.types
def upgrade():
# nationbuilder_cache_person
op.create_table('nationbuilder_cache_person',
sa.Column('uuid', sa.String(length=32), nullable=False),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('email', sa.String(length=255), nullable=True),
sa.Column('email_opt_in', sa.Boolean(), nullable=True),
sa.Column('external_id', sa.String(length=50), nullable=True),
sa.Column('first_name', sa.String(length=100), nullable=True),
sa.Column('middle_name', sa.String(length=100), nullable=True),
sa.Column('last_name', sa.String(length=100), nullable=True),
sa.Column('mobile', sa.String(length=50), nullable=True),
sa.Column('mobile_opt_in', sa.Boolean(), nullable=True),
sa.Column('note', sa.Text(), nullable=True),
sa.Column('phone', sa.String(length=50), nullable=True),
sa.Column('primary_image_url_ssl', sa.String(length=255), nullable=True),
sa.Column('signup_type', sa.Integer(), nullable=True),
sa.Column('primary_address_address1', sa.String(length=100), nullable=True),
sa.Column('primary_address_address2', sa.String(length=100), nullable=True),
sa.Column('primary_address_city', sa.String(length=100), nullable=True),
sa.Column('primary_address_state', sa.String(length=50), nullable=True),
sa.Column('primary_address_zip', sa.String(length=10), nullable=True),
sa.Column('tags', sa.Text(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('uuid'),
sa.UniqueConstraint('id', name='nationbuilder_cache_person_uq_id')
)
op.create_table('nationbuilder_cache_person_version',
sa.Column('uuid', sa.String(length=32), autoincrement=False, nullable=False),
sa.Column('id', sa.Integer(), autoincrement=False, nullable=True),
sa.Column('created_at', sa.DateTime(), autoincrement=False, nullable=True),
sa.Column('email', sa.String(length=255), autoincrement=False, nullable=True),
sa.Column('email_opt_in', sa.Boolean(), autoincrement=False, nullable=True),
sa.Column('external_id', sa.String(length=50), autoincrement=False, nullable=True),
sa.Column('first_name', sa.String(length=100), autoincrement=False, nullable=True),
sa.Column('middle_name', sa.String(length=100), autoincrement=False, nullable=True),
sa.Column('last_name', sa.String(length=100), autoincrement=False, nullable=True),
sa.Column('mobile', sa.String(length=50), autoincrement=False, nullable=True),
sa.Column('mobile_opt_in', sa.Boolean(), autoincrement=False, nullable=True),
sa.Column('note', sa.Text(), autoincrement=False, nullable=True),
sa.Column('phone', sa.String(length=50), autoincrement=False, nullable=True),
sa.Column('primary_image_url_ssl', sa.String(length=255), autoincrement=False, nullable=True),
sa.Column('signup_type', sa.Integer(), autoincrement=False, nullable=True),
sa.Column('primary_address_address1', sa.String(length=100), autoincrement=False, nullable=True),
sa.Column('primary_address_address2', sa.String(length=100), autoincrement=False, nullable=True),
sa.Column('primary_address_city', sa.String(length=100), autoincrement=False, nullable=True),
sa.Column('primary_address_state', sa.String(length=50), autoincrement=False, nullable=True),
sa.Column('primary_address_zip', sa.String(length=10), autoincrement=False, nullable=True),
sa.Column('tags', sa.Text(), autoincrement=False, nullable=True),
sa.Column('updated_at', sa.DateTime(), autoincrement=False, nullable=True),
sa.Column('transaction_id', sa.BigInteger(), autoincrement=False, nullable=False),
sa.Column('end_transaction_id', sa.BigInteger(), nullable=True),
sa.Column('operation_type', sa.SmallInteger(), nullable=False),
sa.PrimaryKeyConstraint('uuid', 'transaction_id')
)
op.create_index(op.f('ix_nationbuilder_cache_person_version_end_transaction_id'), 'nationbuilder_cache_person_version', ['end_transaction_id'], unique=False)
op.create_index(op.f('ix_nationbuilder_cache_person_version_operation_type'), 'nationbuilder_cache_person_version', ['operation_type'], unique=False)
op.create_index(op.f('ix_nationbuilder_cache_person_version_transaction_id'), 'nationbuilder_cache_person_version', ['transaction_id'], unique=False)
def downgrade():
# nationbuilder_cache_person
op.drop_index(op.f('ix_nationbuilder_cache_person_version_transaction_id'), table_name='nationbuilder_cache_person_version')
op.drop_index(op.f('ix_nationbuilder_cache_person_version_operation_type'), table_name='nationbuilder_cache_person_version')
op.drop_index(op.f('ix_nationbuilder_cache_person_version_end_transaction_id'), table_name='nationbuilder_cache_person_version')
op.drop_table('nationbuilder_cache_person_version')
op.drop_table('nationbuilder_cache_person')

View file

@ -0,0 +1,27 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
DB schema for NationBuilder integration
"""
from .nationbuilder import NationBuilderCachePerson

View file

@ -0,0 +1,81 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
NationBuilder cache tables
"""
import sqlalchemy as sa
from sqlalchemy import orm
from rattail.db import model
from rattail.db.util import normalize_full_name
from rattail.config import parse_list
class NationBuilderCachePerson(model.Base):
"""
Represents a Person record in NationBuilder.
https://apiexplorer.nationbuilder.com/nationbuilder#People
"""
__tablename__ = 'nationbuilder_cache_person'
__table_args__ = (
sa.UniqueConstraint('id', name='nationbuilder_cache_person_uq_id'),
)
__versioned__ = {}
model_title = "NationBuilder Person"
model_title_plural = "NationBuilder People"
uuid = model.uuid_column()
id = sa.Column(sa.Integer(), nullable=False)
signup_type = sa.Column(sa.Integer(), nullable=True)
external_id = sa.Column(sa.String(length=50), nullable=True)
tags = sa.Column(sa.String(length=255), nullable=True)
first_name = sa.Column(sa.String(length=100), nullable=True)
middle_name = sa.Column(sa.String(length=100), nullable=True)
last_name = sa.Column(sa.String(length=100), nullable=True)
email = sa.Column(sa.String(length=255), nullable=True)
email_opt_in = sa.Column(sa.Boolean(), nullable=True)
mobile = sa.Column(sa.String(length=50), nullable=True)
mobile_opt_in = sa.Column(sa.Boolean(), nullable=True)
phone = sa.Column(sa.String(length=50), nullable=True)
primary_address_address1 = sa.Column(sa.String(length=100), nullable=True)
primary_address_address2 = sa.Column(sa.String(length=100), nullable=True)
primary_address_city = sa.Column(sa.String(length=100), nullable=True)
primary_address_state = sa.Column(sa.String(length=50), nullable=True)
primary_address_zip = sa.Column(sa.String(length=10), nullable=True)
primary_image_url_ssl = sa.Column(sa.String(length=255), nullable=True)
note = sa.Column(sa.Text(), nullable=True)
created_at = sa.Column(sa.DateTime(), nullable=True)
updated_at = sa.Column(sa.DateTime(), nullable=True)
def __str__(self):
return normalize_full_name(self.first_name, self.last_name)
def has_tag(self, tag):
if self.tags:
for value in parse_list(self.tags):
if value == tag:
return True
return False

View file

@ -0,0 +1,27 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
rattail-nationbuilder importing
"""
from . import model

View file

@ -0,0 +1,36 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
rattail-nationbuilder model importers
"""
from rattail.importing.model import ToRattail
from rattail_nationbuilder.db import model
##############################
# nationbuilder cache
##############################
class NationBuilderCachePersonImporter(ToRattail):
model_class = model.NationBuilderCachePerson

View file

@ -0,0 +1,132 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
NationBuilder -> Rattail importing
"""
import datetime
from collections import OrderedDict
from rattail import importing
from rattail_nationbuilder import importing as nationbuilder_importing
from rattail_nationbuilder.nationbuilder.webapi import NationBuilderWebAPI
class FromNationBuilderToRattail(importing.ToRattailHandler):
"""
Import handler for NationBuilder -> Rattail
"""
host_key = 'nationbuilder'
host_title = "NationBuilder"
generic_host_title = "NationBuilder"
def get_importers(self):
importers = OrderedDict()
importers['NationBuilderCachePerson'] = NationBuilderCachePersonImporter
return importers
class FromNationBuilder(importing.Importer):
"""
Base class for all NationBuilder importers
"""
def setup(self):
super().setup()
self.setup_api()
def setup_api(self):
self.nationbuilder = NationBuilderWebAPI(self.config)
class NationBuilderCachePersonImporter(FromNationBuilder, nationbuilder_importing.model.NationBuilderCachePersonImporter):
"""
Importer for NB Person cache
"""
key = 'id'
primary_address_fields = [
'primary_address_address1',
'primary_address_address2',
'primary_address_city',
'primary_address_state',
'primary_address_zip',
]
supported_fields = [
'id',
'created_at',
'email',
'email_opt_in',
'external_id',
'first_name',
'middle_name',
'last_name',
'mobile',
'mobile_opt_in',
'note',
'phone',
'primary_image_url_ssl',
'signup_type',
'tags',
'updated_at',
] + primary_address_fields
def get_host_objects(self):
return self.nationbuilder.get_people(page_size=500)
def normalize_timestamp(self, value):
if not value:
return
dt = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S%z')
dt = self.app.localtime(dt)
return self.app.make_utc(dt)
def normalize_host_object(self, person):
# nb. some fields may not be present in person dict
data = dict([(field, person.get(field))
for field in self.fields])
if data:
for field in ('created_at', 'updated_at'):
data[field] = self.normalize_timestamp(data[field])
if 'tags' in self.fields:
tags = data['tags']
if tags:
data['tags'] = self.config.make_list_string(tags)
if self.fields_active(self.primary_address_fields):
address = person.get('primary_address')
if address:
data.update({
'primary_address_address1': address['address1'],
'primary_address_address2': address['address2'],
'primary_address_state': address['state'],
'primary_address_city': address['city'],
'primary_address_zip': address['zip'],
})
return data

View file

@ -0,0 +1,41 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
rattail-nationbuilder version importing
"""
from rattail.importing import versions as base
class NationBuilderVersionMixin(object):
def add_nationbuilder_importers(self, importers):
importers['NationBuilderCachePerson'] = NationBuilderCachePersonImporter
return importers
class NationBuilderCachePersonImporter(base.VersionImporter):
@property
def host_model_class(self):
return self.model.NationBuilderCachePerson

View file

@ -0,0 +1,31 @@
# -*- coding: utf-8; -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2023 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
NationBuilder utils
"""
def get_nationbuilder_url(config):
url = config.get('nationbuilder', 'url')
if url:
return url.rstrip('/')

View file

@ -89,7 +89,7 @@ class NationBuilderWebAPI(object):
"""
return self._request('GET', api_method, params=params)
def get_people(self, page_size=10, progress=None, **kwargs):
def get_people(self, page_size=10, max_pages=None, progress=None, **kwargs):
"""
Retrieve all Person records.
@ -109,8 +109,12 @@ class NationBuilderWebAPI(object):
data = response.json()
people.extend(data['results'])
url['next'] = data['next']
log.debug("have fetched %s pages", page + 1)
self.app.progress_loop(fetch, range(pages), progress,
pages = list(range(pages))
if max_pages:
pages = pages[:max_pages]
self.app.progress_loop(fetch, pages, progress,
message="Fetching Person data from NationBuilder")
return people

View file

@ -27,3 +27,15 @@ install_requires =
packages = find:
include_package_data = True
[options.entry_points]
rattail.commands =
import-nationbuilder = rattail_nationbuilder.commands:ImportNationBuilder
rattail.config.extensions =
rattail_nationbuilder = rattail_nationbuilder.config:RattailNationBuilderExtension
rattail.importing =
to_rattail.from_nationbuilder.import = rattail_nationbuilder.importing.nationbuilder:FromNationBuilderToRattail