From aa1b53d96cf3ac99224b725501a0119425b5144d Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 4 Oct 2023 18:52:07 -0500 Subject: [PATCH] Improve the anonymization logic some more hopefully this scrubs all personal data from a `core_op` DB --- rattail_corepos/corepos/office/commands.py | 77 ++++++++++++++++--- .../corepos/office/importing/db/local.py | 14 +--- rattail_corepos/corepos/office/util.py | 18 +++++ 3 files changed, 87 insertions(+), 22 deletions(-) diff --git a/rattail_corepos/corepos/office/commands.py b/rattail_corepos/corepos/office/commands.py index 179baf2..a5f45b8 100644 --- a/rattail_corepos/corepos/office/commands.py +++ b/rattail_corepos/corepos/office/commands.py @@ -34,7 +34,7 @@ from requests.auth import HTTPDigestAuth from rattail import commands from rattail_corepos import __version__ from rattail.util import load_object -from rattail_corepos.corepos.office.util import get_fannie_config_value +from rattail_corepos.corepos.office.util import get_fannie_config_value, get_blueline_template, make_blueline from rattail_corepos.corepos.util import get_core_members from rattail_corepos.config import core_office_url @@ -110,47 +110,97 @@ class Anonymize(commands.Subcommand): self.anonymize_all(args) def anonymize_all(self, args): + import names + import us + core_handler = self.app.get_corepos_handler() op_session = core_handler.make_session_office_op(dbkey=args.dbkey) op_model = core_handler.get_model_office_op() states = [state.abbr for state in us.states.STATES] - members = op_session.query(op_model.MemberInfo)\ - .all() + # meminfo + members = op_session.query(op_model.MemberInfo).all() + members_by_card_number = {} def anon_meminfo(member, i): member.first_name = names.get_first_name() member.last_name = names.get_last_name() member.other_first_name = names.get_first_name() member.other_last_name = names.get_last_name() - # member.city = self.random_city() + member.street = '123 Main St.' + member.city = 'Anytown' member.state = random.choice(states) - # member.zipcode = self.random_zipcode() - member.zipcode = '12345' # TODO + member.zipcode = self.random_zipcode() member.phone = self.random_phone() member.email = self.random_email() member.notes.clear() + members_by_card_number[member.card_number] = member self.progress_loop(anon_meminfo, members, message="Anonymizing meminfo") - customers = op_session.query(op_model.CustomerClassic)\ - .all() + # custdata + customers = op_session.query(op_model.CustomerClassic).all() + blueline_template = get_blueline_template(self.config) def anon_custdata(customer, i): - customer.first_name = names.get_first_name() - customer.last_name = names.get_last_name() + member = members_by_card_number.get(customer.card_number) + if member: + customer.first_name = member.first_name + customer.last_name = member.last_name + else: + customer.first_name = names.get_first_name() + customer.last_name = names.get_last_name() + customer.blue_line = make_blueline(self.config, customer, + template=blueline_template) self.progress_loop(anon_custdata, customers, message="Anonymizing custdata") + # Customers + customers = op_session.query(op_model.Customer).all() + + def del_customer(customer, i): + op_session.delete(customer) + + self.progress_loop(del_customer, customers, + message="Deleting from Customers") + + # CustomerAccounts + accounts = op_session.query(op_model.CustomerAccount).all() + + def del_account(account, i): + op_session.delete(account) + + self.progress_loop(del_account, accounts, + message="Deleting from CustomerAccounts") + + # employees + employees = op_session.query(op_model.Employee).all() + + def anon_employee(employee, i): + employee.first_name = names.get_first_name() + employee.last_name = names.get_last_name() + + self.progress_loop(anon_employee, employees, + message="Anonymizing employees") + + # Users + users = op_session.query(op_model.User).all() + + def anon_user(user, i): + user.real_name = names.get_full_name() + + self.progress_loop(anon_user, users, + message="Anonymizing users") + self.finalize_session(op_session, dry_run=args.dry_run) def random_phone(self): digits = [random.choice('0123456789') for i in range(10)] - return self.app.normalize_phone_number(''.join(digits)) + return self.app.format_phone_number(''.join(digits)) def random_email(self): import names @@ -158,6 +208,11 @@ class Anonymize(commands.Subcommand): name = name.replace(' ', '_') return f'{name}@mailinator.com' + def random_zipcode(self): + digits = [random.choice('0123456789') + for i in range(5)] + return ''.join(digits) + class CoreDBImportSubcommand(commands.ImportSubcommand): """ diff --git a/rattail_corepos/corepos/office/importing/db/local.py b/rattail_corepos/corepos/office/importing/db/local.py index ac0c6bb..79f7ad7 100644 --- a/rattail_corepos/corepos/office/importing/db/local.py +++ b/rattail_corepos/corepos/office/importing/db/local.py @@ -32,7 +32,7 @@ from corepos.db.office_op import model as corepos from rattail import importing from rattail_corepos.corepos.office.importing import db as corepos_importing from rattail_corepos.corepos.office.importing.db.corepos import FromCoreHandler -from rattail_corepos.corepos.office.util import get_fannie_config_value +from rattail_corepos.corepos.office.util import get_blueline_template, make_blueline log = logging.getLogger(__name__) @@ -85,8 +85,7 @@ class CustomerClassicImporter(FromCoreOffice, corepos_importing.model.CustomerCl def setup(self): super().setup() - self.blueline_template = get_fannie_config_value(self.config, - 'BLUELINE_TEMPLATE') + self.blueline_template = get_blueline_template(self.config) def normalize_host_object(self, customer): @@ -95,14 +94,7 @@ class CustomerClassicImporter(FromCoreOffice, corepos_importing.model.CustomerCl customer.card_number, customer) return - blueline = self.blueline_template - blueline = blueline.replace('{{ACCOUNTNO}}', str(customer.card_number)) - blueline = blueline.replace('{{ACCOUNTTYPE}}', customer.member_type.description) - blueline = blueline.replace('{{FIRSTNAME}}', customer.first_name or '') - blueline = blueline.replace('{{FIRSTINITIAL}}', customer.first_name[0] if customer.first_name else '') - blueline = blueline.replace('{{LASTNAME}}', customer.last_name or '') - blueline = blueline.replace('{{LASTINITIAL}}', customer.last_name[0] if customer.last_name else '') - + blueline = make_blueline(self.config, customer, template=self.blueline_template) return { 'id': customer.id, 'blue_line': blueline, diff --git a/rattail_corepos/corepos/office/util.py b/rattail_corepos/corepos/office/util.py index 44aed12..c2c7426 100644 --- a/rattail_corepos/corepos/office/util.py +++ b/rattail_corepos/corepos/office/util.py @@ -60,3 +60,21 @@ def get_fannie_config_value(config, name): raise ValueError(f"failed to read value: {error.output.decode('utf_8')}") return json.loads(output.decode('utf_8')) + + +def get_blueline_template(config): + return get_fannie_config_value(config, 'BLUELINE_TEMPLATE') + + +def make_blueline(config, customer, template=None): + if not template: + template = get_blueline_template(config) + + blueline = template + blueline = blueline.replace('{{ACCOUNTNO}}', str(customer.card_number)) + blueline = blueline.replace('{{ACCOUNTTYPE}}', customer.member_type.description if customer.member_type else '??') + blueline = blueline.replace('{{FIRSTNAME}}', customer.first_name or '') + blueline = blueline.replace('{{FIRSTINITIAL}}', customer.first_name[0] if customer.first_name else '') + blueline = blueline.replace('{{LASTNAME}}', customer.last_name or '') + blueline = blueline.replace('{{LASTINITIAL}}', customer.last_name[0] if customer.last_name else '') + return blueline