Fix how keys are stored for luigi overnight/backfill tasks

rattail/luigi/handler.py

@@ -30,6 +30,7 @@ import os
 import logging
 import subprocess
 import sys
+import warnings
 
 import sqlalchemy as sa
 import six
@@ -90,28 +91,58 @@ class LuigiHandler(GenericHandler):
 
     def get_all_overnight_tasks(self, **kwargs):
         tasks = []
-        keys = self.config.getlist('rattail.luigi', 'overnight_tasks',
+
+        keys = self.config.getlist('rattail.luigi', 'overnight.tasks',
                                    default=[])
+        if not keys:
+            keys = self.config.getlist('rattail.luigi', 'overnight_tasks',
+                                       default=[])
+            if keys:
+                warnings.warn("setting is deprecated: [rattail.luigi] overnight_tasks; "
+                              "please use [rattail.luigi] overnight.tasks instead",
+                              DeprecationWarning)
+
         for key in keys:
-            lastrun = self.config.get(
+            if key.startswith('overnight-'):
-                'rattail.luigi', 'overnight.{}.lastrun'.format(key))
+                key = key[len('overnight-'):]
+                warnings.warn("overnight task keys use deprecated 'overnight-' prefix",
+                              DeprecationWarning)
+
+            lastrun = self.get_overnight_task_setting(key, 'lastrun')
             lastrun = self.app.parse_utctime(lastrun, local=True)
             tasks.append({
                 'key': key,
-                'description': self.config.get(
+                'description': self.get_overnight_task_setting(key, 'description'),
-                    'rattail.luigi', 'overnight.{}.description'.format(key)),
+                'script': self.get_overnight_task_setting(key, 'script'),
-                'script': self.config.get(
+                'notes': self.get_overnight_task_setting(key, 'notes'),
-                    'rattail.luigi', 'overnight.{}.script'.format(key)),
-                'notes': self.config.get(
-                    'rattail.luigi', 'overnight.{}.notes'.format(key)),
                 'lastrun': lastrun,
-                'last_date': self.config.getdate(
+                'last_date': self.get_overnight_task_setting(key, 'last_date',
-                    'rattail.luigi', 'overnight.{}.last_date'.format(key)),
+                                                             typ='date'),
             })
         tasks.sort(key=lambda t: t['description'])
         return tasks
 
+    def get_overnight_task_setting(self, key, name, typ=None, **kwargs):
+        getter = self.config.get
+        if typ == 'date':
+            getter = self.config.getdate
+        value = getter('rattail.luigi',
+                       'overnight.task.{}.{}'.format(key, name))
+        if value is None:
+            value = getter('rattail.luigi',
+                           'overnight.overnight-{}.{}'.format(key, name))
+            if value is not None:
+                warnings.warn("[rattail.luigi] overnight.overnight-* settings are deprecated; "
+                              "please use [rattail.luigi] overnight.task.* instead",
+                              DeprecationWarning)
+        return value
+
     def get_overnight_task(self, key, **kwargs):
+        if key.startswith('overnight-'):
+            key = key[len('overnight-'):]
+            warnings.warn("overnight task keys use deprecated 'overnight-' prefix",
+                          DeprecationWarning, stacklevel=2)
+
         for task in self.get_all_overnight_tasks():
             if task['key'] == key:
                 return task
@@ -149,39 +180,70 @@ class LuigiHandler(GenericHandler):
             raise
 
     def record_overnight_last_date(self, task, date, session=None, **kwargs):
-        name = 'rattail.luigi.overnight.{}.last_date'.format(task['key'])
+        name = 'rattail.luigi.overnight.task.{}.last_date'.format(task['key'])
         with self.app.short_session(session=session, commit=True) as s:
             self.app.save_setting(s, name, six.text_type(date))
 
     def get_all_backfill_tasks(self, **kwargs):
         tasks = []
-        keys = self.config.getlist('rattail.luigi', 'backfill_tasks',
+
+        keys = self.config.getlist('rattail.luigi', 'backfill.tasks',
                                    default=[])
+        if not keys:
+            keys = self.config.getlist('rattail.luigi', 'backfill_tasks',
+                                       default=[])
+            if keys:
+                warnings.warn("setting is deprecated: [rattail.luigi] backfill_tasks; "
+                              "please use [rattail.luigi] backfill.tasks instead",
+                              DeprecationWarning)
+
         for key in keys:
-            lastrun = self.config.get(
+            if key.startswith('backfill-'):
-                'rattail.luigi', 'backfill.{}.lastrun'.format(key))
+                key = key[len('backfill-'):]
+                warnings.warn("backfill task keys use deprecated 'backfill-' prefix",
+                              DeprecationWarning)
+
+            lastrun = self.get_backfill_task_setting(key, 'lastrun')
             lastrun = self.app.parse_utctime(lastrun, local=True)
             tasks.append({
                 'key': key,
-                'description': self.config.get(
+                'description': self.get_backfill_task_setting(key, 'description'),
-                    'rattail.luigi', 'backfill.{}.description'.format(key)),
+                'script': self.get_backfill_task_setting(key, 'script'),
-                'script': self.config.get(
+                'forward': self.get_backfill_task_setting(key, 'forward',
-                    'rattail.luigi', 'backfill.{}.script'.format(key)),
+                                                          typ='bool') or False,
-                'forward': self.config.getbool(
+                'notes': self.get_backfill_task_setting(key, 'notes'),
-                    'rattail.luigi', 'backfill.{}.forward'.format(key),
-                    default=False),
-                'notes': self.config.get(
-                    'rattail.luigi', 'backfill.{}.notes'.format(key)),
                 'lastrun': lastrun,
-                'last_date': self.config.getdate(
+                'last_date': self.get_backfill_task_setting(key, 'last_date',
-                    'rattail.luigi', 'backfill.{}.last_date'.format(key)),
+                                                            typ='date'),
-                'target_date': self.config.getdate(
+                'target_date': self.get_backfill_task_setting(key, 'target_date',
-                    'rattail.luigi', 'backfill.{}.target_date'.format(key)),
+                                                              typ='date'),
             })
         tasks.sort(key=lambda t: t['description'])
         return tasks
 
+    def get_backfill_task_setting(self, key, name, typ=None, **kwargs):
+        getter = self.config.get
+        if typ == 'bool':
+            getter = self.config.getbool
+        elif typ == 'date':
+            getter = self.config.getdate
+        value = getter('rattail.luigi',
+                       'backfill.task.{}.{}'.format(key, name))
+        if value is None:
+            value = getter('rattail.luigi',
+                           'backfill.backfill-{}.{}'.format(key, name))
+            if value is not None:
+                warnings.warn("[rattail.luigi] backfill.backfill-* settings are deprecated; "
+                              "please use [rattail.luigi] backfill.task.* instead",
+                              DeprecationWarning)
+        return value
+
     def get_backfill_task(self, key, **kwargs):
+        if key.startswith('backfill-'):
+            key = key[len('backfill-'):]
+            warnings.warn("backfill task keys use deprecated 'backfill-' prefix",
+                          DeprecationWarning, stacklevel=2)
+
         for task in self.get_all_backfill_tasks():
             if task['key'] == key:
                 return task
@@ -239,6 +301,6 @@ class LuigiHandler(GenericHandler):
             raise
 
     def record_backfill_last_date(self, task, date, session=None, **kwargs):
-        name = 'rattail.luigi.backfill.{}.last_date'.format(task['key'])
+        name = 'rattail.luigi.backfill.task.{}.last_date'.format(task['key'])
         with self.app.short_session(session=session, commit=True) as s:
             self.app.save_setting(s, name, six.text_type(date))