Fix how keys are stored for luigi overnight/backfill tasks
This commit is contained in:
Lance Edgar
parent
c8597434a0
commit
bea5278289
|
|
@ -30,6 +30,7 @@ import os
|
|||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
import sqlalchemy as sa
|
||||
import six
|
||||
|
|
@ -90,28 +91,58 @@ class LuigiHandler(GenericHandler):
|
|||
|
||||
def get_all_overnight_tasks(self, **kwargs):
|
||||
tasks = []
|
||||
keys = self.config.getlist('rattail.luigi', 'overnight_tasks',
|
||||
|
||||
keys = self.config.getlist('rattail.luigi', 'overnight.tasks',
|
||||
default=[])
|
||||
if not keys:
|
||||
keys = self.config.getlist('rattail.luigi', 'overnight_tasks',
|
||||
default=[])
|
||||
if keys:
|
||||
warnings.warn("setting is deprecated: [rattail.luigi] overnight_tasks; "
|
||||
"please use [rattail.luigi] overnight.tasks instead",
|
||||
DeprecationWarning)
|
||||
|
||||
for key in keys:
|
||||
lastrun = self.config.get(
|
||||
'rattail.luigi', 'overnight.{}.lastrun'.format(key))
|
||||
if key.startswith('overnight-'):
|
||||
key = key[len('overnight-'):]
|
||||
warnings.warn("overnight task keys use deprecated 'overnight-' prefix",
|
||||
DeprecationWarning)
|
||||
|
||||
lastrun = self.get_overnight_task_setting(key, 'lastrun')
|
||||
lastrun = self.app.parse_utctime(lastrun, local=True)
|
||||
tasks.append({
|
||||
'key': key,
|
||||
'description': self.config.get(
|
||||
'rattail.luigi', 'overnight.{}.description'.format(key)),
|
||||
'script': self.config.get(
|
||||
'rattail.luigi', 'overnight.{}.script'.format(key)),
|
||||
'notes': self.config.get(
|
||||
'rattail.luigi', 'overnight.{}.notes'.format(key)),
|
||||
'description': self.get_overnight_task_setting(key, 'description'),
|
||||
'script': self.get_overnight_task_setting(key, 'script'),
|
||||
'notes': self.get_overnight_task_setting(key, 'notes'),
|
||||
'lastrun': lastrun,
|
||||
'last_date': self.config.getdate(
|
||||
'rattail.luigi', 'overnight.{}.last_date'.format(key)),
|
||||
'last_date': self.get_overnight_task_setting(key, 'last_date',
|
||||
typ='date'),
|
||||
})
|
||||
tasks.sort(key=lambda t: t['description'])
|
||||
return tasks
|
||||
|
||||
def get_overnight_task_setting(self, key, name, typ=None, **kwargs):
|
||||
getter = self.config.get
|
||||
if typ == 'date':
|
||||
getter = self.config.getdate
|
||||
value = getter('rattail.luigi',
|
||||
'overnight.task.{}.{}'.format(key, name))
|
||||
if value is None:
|
||||
value = getter('rattail.luigi',
|
||||
'overnight.overnight-{}.{}'.format(key, name))
|
||||
if value is not None:
|
||||
warnings.warn("[rattail.luigi] overnight.overnight-* settings are deprecated; "
|
||||
"please use [rattail.luigi] overnight.task.* instead",
|
||||
DeprecationWarning)
|
||||
return value
|
||||
|
||||
def get_overnight_task(self, key, **kwargs):
|
||||
if key.startswith('overnight-'):
|
||||
key = key[len('overnight-'):]
|
||||
warnings.warn("overnight task keys use deprecated 'overnight-' prefix",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
|
||||
for task in self.get_all_overnight_tasks():
|
||||
if task['key'] == key:
|
||||
return task
|
||||
|
|
@ -149,39 +180,70 @@ class LuigiHandler(GenericHandler):
|
|||
raise
|
||||
|
||||
def record_overnight_last_date(self, task, date, session=None, **kwargs):
|
||||
name = 'rattail.luigi.overnight.{}.last_date'.format(task['key'])
|
||||
name = 'rattail.luigi.overnight.task.{}.last_date'.format(task['key'])
|
||||
with self.app.short_session(session=session, commit=True) as s:
|
||||
self.app.save_setting(s, name, six.text_type(date))
|
||||
|
||||
def get_all_backfill_tasks(self, **kwargs):
|
||||
tasks = []
|
||||
keys = self.config.getlist('rattail.luigi', 'backfill_tasks',
|
||||
|
||||
keys = self.config.getlist('rattail.luigi', 'backfill.tasks',
|
||||
default=[])
|
||||
if not keys:
|
||||
keys = self.config.getlist('rattail.luigi', 'backfill_tasks',
|
||||
default=[])
|
||||
if keys:
|
||||
warnings.warn("setting is deprecated: [rattail.luigi] backfill_tasks; "
|
||||
"please use [rattail.luigi] backfill.tasks instead",
|
||||
DeprecationWarning)
|
||||
|
||||
for key in keys:
|
||||
lastrun = self.config.get(
|
||||
'rattail.luigi', 'backfill.{}.lastrun'.format(key))
|
||||
if key.startswith('backfill-'):
|
||||
key = key[len('backfill-'):]
|
||||
warnings.warn("backfill task keys use deprecated 'backfill-' prefix",
|
||||
DeprecationWarning)
|
||||
|
||||
lastrun = self.get_backfill_task_setting(key, 'lastrun')
|
||||
lastrun = self.app.parse_utctime(lastrun, local=True)
|
||||
tasks.append({
|
||||
'key': key,
|
||||
'description': self.config.get(
|
||||
'rattail.luigi', 'backfill.{}.description'.format(key)),
|
||||
'script': self.config.get(
|
||||
'rattail.luigi', 'backfill.{}.script'.format(key)),
|
||||
'forward': self.config.getbool(
|
||||
'rattail.luigi', 'backfill.{}.forward'.format(key),
|
||||
default=False),
|
||||
'notes': self.config.get(
|
||||
'rattail.luigi', 'backfill.{}.notes'.format(key)),
|
||||
'description': self.get_backfill_task_setting(key, 'description'),
|
||||
'script': self.get_backfill_task_setting(key, 'script'),
|
||||
'forward': self.get_backfill_task_setting(key, 'forward',
|
||||
typ='bool') or False,
|
||||
'notes': self.get_backfill_task_setting(key, 'notes'),
|
||||
'lastrun': lastrun,
|
||||
'last_date': self.config.getdate(
|
||||
'rattail.luigi', 'backfill.{}.last_date'.format(key)),
|
||||
'target_date': self.config.getdate(
|
||||
'rattail.luigi', 'backfill.{}.target_date'.format(key)),
|
||||
'last_date': self.get_backfill_task_setting(key, 'last_date',
|
||||
typ='date'),
|
||||
'target_date': self.get_backfill_task_setting(key, 'target_date',
|
||||
typ='date'),
|
||||
})
|
||||
tasks.sort(key=lambda t: t['description'])
|
||||
return tasks
|
||||
|
||||
def get_backfill_task_setting(self, key, name, typ=None, **kwargs):
|
||||
getter = self.config.get
|
||||
if typ == 'bool':
|
||||
getter = self.config.getbool
|
||||
elif typ == 'date':
|
||||
getter = self.config.getdate
|
||||
value = getter('rattail.luigi',
|
||||
'backfill.task.{}.{}'.format(key, name))
|
||||
if value is None:
|
||||
value = getter('rattail.luigi',
|
||||
'backfill.backfill-{}.{}'.format(key, name))
|
||||
if value is not None:
|
||||
warnings.warn("[rattail.luigi] backfill.backfill-* settings are deprecated; "
|
||||
"please use [rattail.luigi] backfill.task.* instead",
|
||||
DeprecationWarning)
|
||||
return value
|
||||
|
||||
def get_backfill_task(self, key, **kwargs):
|
||||
if key.startswith('backfill-'):
|
||||
key = key[len('backfill-'):]
|
||||
warnings.warn("backfill task keys use deprecated 'backfill-' prefix",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
|
||||
for task in self.get_all_backfill_tasks():
|
||||
if task['key'] == key:
|
||||
return task
|
||||
|
|
@ -239,6 +301,6 @@ class LuigiHandler(GenericHandler):
|
|||
raise
|
||||
|
||||
def record_backfill_last_date(self, task, date, session=None, **kwargs):
|
||||
name = 'rattail.luigi.backfill.{}.last_date'.format(task['key'])
|
||||
name = 'rattail.luigi.backfill.task.{}.last_date'.format(task['key'])
|
||||
with self.app.short_session(session=session, commit=True) as s:
|
||||
self.app.save_setting(s, name, six.text_type(date))
|
||||
|
|
|
|||
Loading…
Reference in a new issue