fix: implement deletion logic; add cli params for max changes

also add special UUID field handling for CSV -> SQLAlchemy ORM, to
normalize string from CSV to proper UUID so key matching works
This commit is contained in:
Lance Edgar 2024-12-06 15:18:23 -06:00
parent a73896b75d
commit 328f8d9952
6 changed files with 735 additions and 115 deletions

View file

@ -89,60 +89,237 @@ class TestImporter(DataTestCase):
def test_process_data(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting, caches_target=True)
imp = self.make_importer(model_class=model.Setting, caches_target=True,
delete=True)
# empty data set / just for coverage
with patch.object(imp, 'normalize_source_data') as normalize_source_data:
normalize_source_data.return_value = []
def make_cache():
setting1 = model.Setting(name='foo1', value='bar1')
setting2 = model.Setting(name='foo2', value='bar2')
setting3 = model.Setting(name='foo3', value='bar3')
cache = {
('foo1',): {
'object': setting1,
'data': {'name': 'foo1', 'value': 'bar1'},
},
('foo2',): {
'object': setting2,
'data': {'name': 'foo2', 'value': 'bar2'},
},
('foo3',): {
'object': setting3,
'data': {'name': 'foo3', 'value': 'bar3'},
},
}
return cache
with patch.object(imp, 'get_target_cache') as get_target_cache:
get_target_cache.return_value = {}
# nb. delete always succeeds
with patch.object(imp, 'delete_target_object', return_value=True):
result = imp.process_data()
self.assertEqual(result, ([], [], []))
# create + update + delete all as needed
with patch.object(imp, 'get_target_cache', return_value=make_cache()):
created, updated, deleted = imp.process_data([
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
{'name': 'foo5', 'value': 'BAR5'},
])
self.assertEqual(len(created), 2)
self.assertEqual(len(updated), 1)
self.assertEqual(len(deleted), 2)
# same but with --max-total so delete gets skipped
with patch.object(imp, 'get_target_cache', return_value=make_cache()):
with patch.object(imp, 'max_total', new=3):
created, updated, deleted = imp.process_data([
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
{'name': 'foo5', 'value': 'BAR5'},
])
self.assertEqual(len(created), 2)
self.assertEqual(len(updated), 1)
self.assertEqual(len(deleted), 0)
# delete all if source data empty
with patch.object(imp, 'get_target_cache', return_value=make_cache()):
created, updated, deleted = imp.process_data()
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 0)
self.assertEqual(len(deleted), 3)
def test_do_create_update(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting, caches_target=True)
def make_cache():
setting1 = model.Setting(name='foo1', value='bar1')
setting2 = model.Setting(name='foo2', value='bar2')
cache = {
('foo1',): {
'object': setting1,
'data': {'name': 'foo1', 'value': 'bar1'},
},
('foo2',): {
'object': setting2,
'data': {'name': 'foo2', 'value': 'bar2'},
},
}
return cache
# change nothing if data matches
with patch.multiple(imp, create=True, cached_target=make_cache()):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'bar1'},
{'name': 'foo2', 'value': 'bar2'},
])
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 0)
# update all as needed
with patch.multiple(imp, create=True, cached_target=make_cache()):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'BAR1'},
{'name': 'foo2', 'value': 'BAR2'},
])
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 2)
# update all, with --max-update
with patch.multiple(imp, create=True, cached_target=make_cache(), max_update=1):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'BAR1'},
{'name': 'foo2', 'value': 'BAR2'},
])
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 1)
# update all, with --max-total
with patch.multiple(imp, create=True, cached_target=make_cache(), max_total=1):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'BAR1'},
{'name': 'foo2', 'value': 'BAR2'},
])
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 1)
# create all as needed
with patch.multiple(imp, create=True, cached_target=make_cache()):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'bar1'},
{'name': 'foo2', 'value': 'bar2'},
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
])
self.assertEqual(len(created), 2)
self.assertEqual(len(updated), 0)
# what happens when create gets skipped
with patch.multiple(imp, create=True, cached_target=make_cache()):
with patch.object(imp, 'create_target_object', return_value=None):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'bar1'},
{'name': 'foo2', 'value': 'bar2'},
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
])
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 0)
# create all, with --max-create
with patch.multiple(imp, create=True, cached_target=make_cache(), max_create=1):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'bar1'},
{'name': 'foo2', 'value': 'bar2'},
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
])
self.assertEqual(len(created), 1)
self.assertEqual(len(updated), 0)
# create all, with --max-total
with patch.multiple(imp, create=True, cached_target=make_cache(), max_total=1):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'bar1'},
{'name': 'foo2', 'value': 'bar2'},
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
])
self.assertEqual(len(created), 1)
self.assertEqual(len(updated), 0)
# create + update all as needed
with patch.multiple(imp, create=True, cached_target=make_cache()):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'BAR1'},
{'name': 'foo2', 'value': 'BAR2'},
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
])
self.assertEqual(len(created), 2)
self.assertEqual(len(updated), 2)
# create + update all, with --max-total
with patch.multiple(imp, create=True, cached_target=make_cache(), max_total=1):
created, updated = imp.do_create_update([
{'name': 'foo1', 'value': 'BAR1'},
{'name': 'foo2', 'value': 'BAR2'},
{'name': 'foo3', 'value': 'BAR3'},
{'name': 'foo4', 'value': 'BAR4'},
])
# nb. foo1 is updated first
self.assertEqual(len(created), 0)
self.assertEqual(len(updated), 1)
def test_do_delete(self):
model = self.app.model
# this requires a mock target cache
setting1 = model.Setting(name='foo1', value='bar1')
setting2 = model.Setting(name='foo2', value='bar2')
imp = self.make_importer(model_class=model.Setting, caches_target=True)
setting = model.Setting(name='foo', value='bar')
imp.cached_target = {
('foo',): {
'object': setting,
'data': {'name': 'foo', 'value': 'bar'},
cache = {
('foo1',): {
'object': setting1,
'data': {'name': 'foo1', 'value': 'bar1'},
},
('foo2',): {
'object': setting2,
'data': {'name': 'foo2', 'value': 'bar2'},
},
}
# will update the one record
result = imp.do_create_update([{'name': 'foo', 'value': 'baz'}])
self.assertIs(result[1][0][0], setting)
self.assertEqual(result, ([], [(setting,
# nb. target
{'name': 'foo', 'value': 'bar'},
# nb. source
{'name': 'foo', 'value': 'baz'})]))
self.assertEqual(setting.value, 'baz')
with patch.object(imp, 'delete_target_object') as delete_target_object:
# will create a new record
result = imp.do_create_update([{'name': 'blah', 'value': 'zay'}])
self.assertIsNot(result[0][0][0], setting)
setting_new = result[0][0][0]
self.assertEqual(result, ([(setting_new,
# nb. source
{'name': 'blah', 'value': 'zay'})],
[]))
self.assertEqual(setting_new.name, 'blah')
self.assertEqual(setting_new.value, 'zay')
# delete nothing if source has same keys
with patch.multiple(imp, create=True, cached_target=dict(cache)):
source_keys = set(imp.cached_target)
result = imp.do_delete(source_keys)
self.assertFalse(delete_target_object.called)
self.assertEqual(result, [])
# but what if new record is *not* created
with patch.object(imp, 'create_target_object', return_value=None):
result = imp.do_create_update([{'name': 'another', 'value': 'one'}])
self.assertEqual(result, ([], []))
# delete both if source has no keys
delete_target_object.reset_mock()
with patch.multiple(imp, create=True, cached_target=dict(cache)):
source_keys = set()
result = imp.do_delete(source_keys)
self.assertEqual(delete_target_object.call_count, 2)
self.assertEqual(len(result), 2)
# def test_do_delete(self):
# model = self.app.model
# imp = self.make_importer(model_class=model.Setting)
# delete just one if --max-delete was set
delete_target_object.reset_mock()
with patch.multiple(imp, create=True, cached_target=dict(cache)):
source_keys = set()
with patch.object(imp, 'max_delete', new=1):
result = imp.do_delete(source_keys)
self.assertEqual(delete_target_object.call_count, 1)
self.assertEqual(len(result), 1)
# delete just one if --max-total was set
delete_target_object.reset_mock()
with patch.multiple(imp, create=True, cached_target=dict(cache)):
source_keys = set()
with patch.object(imp, 'max_total', new=1):
result = imp.do_delete(source_keys)
self.assertEqual(delete_target_object.call_count, 1)
self.assertEqual(len(result), 1)
def test_get_record_key(self):
model = self.app.model
@ -182,6 +359,22 @@ class TestImporter(DataTestCase):
# nb. default normalizer returns object as-is
self.assertIs(data[0], setting)
def test_get_unique_data(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting)
setting1 = model.Setting(name='foo', value='bar1')
setting2 = model.Setting(name='foo', value='bar2')
result = imp.get_unique_data([setting2, setting1])
self.assertIsInstance(result, tuple)
self.assertEqual(len(result), 2)
self.assertIsInstance(result[0], list)
self.assertEqual(len(result[0]), 1)
self.assertIs(result[0][0], setting2) # nb. not setting1
self.assertIsInstance(result[1], set)
self.assertEqual(result[1], {('foo',)})
def test_get_source_objects(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting)
@ -263,6 +456,34 @@ class TestImporter(DataTestCase):
data = imp.normalize_target_object(setting)
self.assertEqual(data, {'name': 'foo', 'value': 'bar'})
def test_get_deletable_keys(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting)
# empty set by default (nb. no target cache)
result = imp.get_deletable_keys()
self.assertIsInstance(result, set)
self.assertEqual(result, set())
setting = model.Setting(name='foo', value='bar')
cache = {
('foo',): {
'object': setting,
'data': {'name': 'foo', 'value': 'bar'},
},
}
with patch.multiple(imp, create=True, caches_target=True, cached_target=cache):
# all are deletable by default
result = imp.get_deletable_keys()
self.assertEqual(result, {('foo',)})
# but some maybe can't be deleted
with patch.object(imp, 'can_delete_object', return_value=False):
result = imp.get_deletable_keys()
self.assertEqual(result, set())
def test_create_target_object(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting)
@ -301,6 +522,19 @@ class TestImporter(DataTestCase):
self.assertIs(obj, setting)
self.assertEqual(setting.value, 'bar')
def test_can_delete_object(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting)
setting = model.Setting(name='foo')
self.assertTrue(imp.can_delete_object(setting))
def test_delete_target_object(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting)
setting = model.Setting(name='foo')
# nb. default implementation always returns false
self.assertFalse(imp.delete_target_object(setting))
class TestFromFile(DataTestCase):
@ -390,6 +624,20 @@ class TestToSqlalchemy(DataTestCase):
kwargs.setdefault('handler', self.handler)
return mod.ToSqlalchemy(self.config, **kwargs)
def test_get_target_objects(self):
model = self.app.model
imp = self.make_importer(model_class=model.Setting, target_session=self.session)
setting1 = model.Setting(name='foo', value='bar')
self.session.add(setting1)
setting2 = model.Setting(name='foo2', value='bar2')
self.session.add(setting2)
self.session.commit()
result = imp.get_target_objects()
self.assertEqual(len(result), 2)
self.assertEqual(set(result), {setting1, setting2})
def test_get_target_object(self):
model = self.app.model
setting = model.Setting(name='foo', value='bar')
@ -416,15 +664,19 @@ class TestToSqlalchemy(DataTestCase):
self.session.add(setting2)
self.session.commit()
# then we should be able to fetch that via query
imp.target_session = self.session
result = imp.get_target_object(('foo2',))
self.assertIsInstance(result, model.Setting)
self.assertIs(result, setting2)
# nb. disable target cache
with patch.multiple(imp, create=True,
target_session=self.session,
caches_target=False):
# but sometimes it will not be found
result = imp.get_target_object(('foo3',))
self.assertIsNone(result)
# now we should be able to fetch that via query
result = imp.get_target_object(('foo2',))
self.assertIsInstance(result, model.Setting)
self.assertIs(result, setting2)
# but sometimes it will not be found
result = imp.get_target_object(('foo3',))
self.assertIsNone(result)
def test_create_target_object(self):
model = self.app.model
@ -438,16 +690,13 @@ class TestToSqlalchemy(DataTestCase):
self.assertEqual(setting.value, 'bar')
self.assertIn(setting, self.session)
def test_get_target_objects(self):
def test_delete_target_object(self):
model = self.app.model
setting = model.Setting(name='foo', value='bar')
self.session.add(setting)
self.assertEqual(self.session.query(model.Setting).count(), 1)
imp = self.make_importer(model_class=model.Setting, target_session=self.session)
setting1 = model.Setting(name='foo', value='bar')
self.session.add(setting1)
setting2 = model.Setting(name='foo2', value='bar2')
self.session.add(setting2)
self.session.commit()
result = imp.get_target_objects()
self.assertEqual(len(result), 2)
self.assertEqual(set(result), {setting1, setting2})
imp.delete_target_object(setting)
self.assertEqual(self.session.query(model.Setting).count(), 0)

View file

@ -1,6 +1,7 @@
#-*- coding: utf-8; -*-
import csv
import uuid as _uuid
from unittest.mock import patch
from wuttjamaican.testing import DataTestCase
@ -87,23 +88,74 @@ foo2,bar2
self.assertEqual(objects[1], {'name': 'foo2', 'value': 'bar2'})
class MockMixinHandler(mod.FromCsvToSqlalchemyMixin, ToSqlalchemyHandler):
ToImporterBase = ToSqlalchemy
class MockMixinImporter(mod.FromCsvToSqlalchemyMixin, mod.FromCsv, ToSqlalchemy):
pass
class TestFromCsvToSqlalchemyMixin(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ImportHandler(self.config)
def make_importer(self, **kwargs):
kwargs.setdefault('handler', self.handler)
return MockMixinImporter(self.config, **kwargs)
def test_constructor(self):
model = self.app.model
# no uuid keys
imp = self.make_importer(model_class=model.Setting)
self.assertEqual(imp.uuid_keys, [])
# typical
# nb. as of now Upgrade is the only table using proper UUID
imp = self.make_importer(model_class=model.Upgrade)
self.assertEqual(imp.uuid_keys, ['uuid'])
def test_normalize_source_object(self):
model = self.app.model
# no uuid keys
imp = self.make_importer(model_class=model.Setting)
result = imp.normalize_source_object({'name': 'foo', 'value': 'bar'})
self.assertEqual(result, {'name': 'foo', 'value': 'bar'})
# source has proper UUID
# nb. as of now Upgrade is the only table using proper UUID
imp = self.make_importer(model_class=model.Upgrade, fields=['uuid', 'description'])
result = imp.normalize_source_object({'uuid': _uuid.UUID('06753693-d892-77f0-8000-ce71bf7ebbba'),
'description': 'testing'})
self.assertEqual(result, {'uuid': _uuid.UUID('06753693-d892-77f0-8000-ce71bf7ebbba'),
'description': 'testing'})
# source has string uuid
# nb. as of now Upgrade is the only table using proper UUID
imp = self.make_importer(model_class=model.Upgrade, fields=['uuid', 'description'])
result = imp.normalize_source_object({'uuid': '06753693d89277f08000ce71bf7ebbba',
'description': 'testing'})
self.assertEqual(result, {'uuid': _uuid.UUID('06753693-d892-77f0-8000-ce71bf7ebbba'),
'description': 'testing'})
class MockMixinHandler(mod.FromCsvToSqlalchemyHandlerMixin, ToSqlalchemyHandler):
ToImporterBase = ToSqlalchemy
class TestFromCsvToSqlalchemyHandlerMixin(DataTestCase):
def make_handler(self, **kwargs):
return MockMixinHandler(self.config, **kwargs)
def test_get_target_model(self):
with patch.object(mod.FromCsvToSqlalchemyMixin, 'define_importers', return_value={}):
with patch.object(mod.FromCsvToSqlalchemyHandlerMixin, 'define_importers', return_value={}):
handler = self.make_handler()
self.assertRaises(NotImplementedError, handler.get_target_model)
def test_define_importers(self):
model = self.app.model
with patch.object(mod.FromCsvToSqlalchemyMixin, 'get_target_model', return_value=model):
with patch.object(mod.FromCsvToSqlalchemyHandlerMixin, 'get_target_model', return_value=model):
handler = self.make_handler()
importers = handler.define_importers()
self.assertIn('Setting', importers)
@ -115,7 +167,7 @@ class TestFromCsvToSqlalchemyMixin(DataTestCase):
def test_make_importer_factory(self):
model = self.app.model
with patch.object(mod.FromCsvToSqlalchemyMixin, 'define_importers', return_value={}):
with patch.object(mod.FromCsvToSqlalchemyHandlerMixin, 'define_importers', return_value={}):
handler = self.make_handler()
factory = handler.make_importer_factory(model.Setting, 'Setting')
self.assertTrue(issubclass(factory, mod.FromCsv))