Allow import of harvest time entries w/ missing project

rattail_harvest/importing/harvest.py

@@ -183,14 +183,6 @@ class HarvestTimeEntryImporter(FromHarvest, rattail_harvest_importing.model.Harv
     Import time entry data from Harvest
     """
 
-    def setup(self):
-        super(HarvestTimeEntryImporter, self).setup()
-        model = self.model
-
-        self.harvest_projects_by_id = self.app.cache_model(self.session,
-                                                           model.HarvestProject,
-                                                           key='id')
-
     def get_host_objects(self):
         return self.webapi.get_time_entries(**{'from': self.start_date,
                                                'to': self.end_date})
@@ -202,16 +194,20 @@ class HarvestTimeEntryImporter(FromHarvest, rattail_harvest_importing.model.Harv
 
         data['user_id'] = entry['user']['id']
         data['client_id'] = entry['client']['id']
-
-        data['project_id'] = entry['project']['id']
-        if data['project_id'] not in self.harvest_projects_by_id:
-            log.warning("time entry references non-existent project id %s: %s",
-                        data['project_id'], entry)
-            data['project_id'] = None
-
         data['task_id'] = entry['task']['id']
         data['invoice_id'] = entry['invoice']['id'] if entry['invoice'] else None
 
+        # project_id
+        if 'project_id' in self.fields:
+            data['project_id'] = entry['project']['id']
+            project = self.get_harvest_project(data['project_id'])
+            if not project:
+                logger = log.warning if self.warn_for_unknown_project else log.debug
+                logger("time entry references non-existent project id %s: %s",
+                       data['project_id'], entry)
+                if not self.auto_create_unknown_project:
+                    data['project_id'] = None
+
         # spent_date
         spent_date = data['spent_date']
         if spent_date:

rattail_harvest/importing/model.py

@@ -24,10 +24,15 @@
 rattail-harvest model importers
 """
 
+import logging
+
 from rattail.importing.model import ToRattail
 from rattail_harvest.db import model
 
 
+log = logging.getLogger(__name__)
+
+
 ##############################
 # harvest cache models
 ##############################
@@ -47,7 +52,53 @@ class HarvestTaskImporter(ToRattail):
 class HarvestTimeEntryImporter(ToRattail):
     model_class = model.HarvestTimeEntry
 
+    # flags to auto-create records for "unknown" references
+    auto_create_unknown_project = True
+
+    # flags to log warning vs. debug for "unknown" references
+    warn_for_unknown_project = True
+
+    def setup(self):
+        super(HarvestTimeEntryImporter, self).setup()
+        model = self.model
+
+        if 'project_id' in self.fields:
+            self.harvest_projects_by_id = self.app.cache_model(
+                self.session, model.HarvestProject, key='id')
+
     def cache_query(self):
         query = super(HarvestTimeEntryImporter, self).cache_query()
         return query.filter(self.model_class.spent_date >= self.start_date)\
                     .filter(self.model_class.spent_date <= self.end_date)
+
+    def get_harvest_project(self, project_id):
+        if hasattr(self, 'harvest_projects_by_id'):
+            return self.harvest_projects_by_id.get(project_id)
+
+        model = self.model
+        return self.session.query(model.HarvestProject)\
+                           .filter(model.HarvestProject.id == project_id)\
+                           .first()
+
+    def update_object(self, entry, data, local_data=None):
+        entry = super(HarvestTimeEntryImporter, self).update_object(entry, data, local_data)
+        model = self.model
+
+        if 'project_id' in self.fields:
+            project_id = data['project_id']
+            project = self.get_harvest_project(project_id)
+            if not project:
+                logger = log.warning if self.warn_for_unknown_project else log.debug
+                logger("unknown project id %s for time entry id %s: %s",
+                       project_id, entry.id, entry)
+                if self.auto_create_unknown_project:
+                    project = model.HarvestProject()
+                    project.id = project_id
+                    project.name = "(unknown)"
+                    self.session.add(project)
+                    if hasattr(self, 'harvest_projects_by_id'):
+                        self.harvest_projects_by_id[project_id] = project
+                elif entry.project_id:
+                    entry.project_id = None
+
+        return entry