rattail/tailbone
2
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

Fix how keys are stored for luigi overnight/backfill tasks

Browse source
This commit is contained in:
Lance Edgar 2022-11-20 13:58:39 -06:00
parent d4801f58e3
commit 7f0305fb7a
2 changed files with 54 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

47
tailbone/views/luigi.py
View file

@ -166,56 +166,39 @@ class LuigiTaskView(MasterView):
# overnight tasks
keys = []
for task in json.loads(data['overnight_tasks']):
key = task['key']
if key.startswith('_new_'):
key = app.make_uuid()
key = task['key']
if key.startswith('_new_'):
cmd = shlex.split(task['script'])
script = os.path.basename(cmd[0])
root, ext = os.path.splitext(script)
key = re.sub(r'\s+', '-', root)
keys.append(key)
settings.extend([
{'name': 'rattail.luigi.overnight.{}.description'.format(key),
{'name': 'rattail.luigi.overnight.task.{}.description'.format(key),
'value': task['description']},
{'name': 'rattail.luigi.overnight.{}.script'.format(key),
{'name': 'rattail.luigi.overnight.task.{}.script'.format(key),
'value': task['script']},
{'name': 'rattail.luigi.overnight.{}.notes'.format(key),
{'name': 'rattail.luigi.overnight.task.{}.notes'.format(key),
'value': task['notes']},
])
if keys:
settings.append({'name': 'rattail.luigi.overnight_tasks',
settings.append({'name': 'rattail.luigi.overnight.tasks',
'value': ', '.join(keys)})
# backfill tasks
keys = []
for task in json.loads(data['backfill_tasks']):
key = task['key']
if key.startswith('_new_'):
script = os.path.basename(task['script'])
root, ext = os.path.splitext(script)
key = re.sub(r'\s+', '-', root)
keys.append(key)
settings.extend([
{'name': 'rattail.luigi.backfill.{}.description'.format(key),
{'name': 'rattail.luigi.backfill.task.{}.description'.format(key),
'value': task['description']},
{'name': 'rattail.luigi.backfill.{}.script'.format(key),
{'name': 'rattail.luigi.backfill.task.{}.script'.format(key),
'value': task['script']},
{'name': 'rattail.luigi.backfill.{}.forward'.format(key),
{'name': 'rattail.luigi.backfill.task.{}.forward'.format(key),
'value': 'true' if task['forward'] else 'false'},
{'name': 'rattail.luigi.backfill.{}.notes'.format(key),
{'name': 'rattail.luigi.backfill.task.{}.notes'.format(key),
'value': task['notes']},
{'name': 'rattail.luigi.backfill.{}.target_date'.format(key),
{'name': 'rattail.luigi.backfill.task.{}.target_date'.format(key),
'value': six.text_type(task['target_date'])},
])
if keys:
settings.append({'name': 'rattail.luigi.backfill_tasks',
settings.append({'name': 'rattail.luigi.backfill.tasks',
'value': ', '.join(keys)})
return settings
@ -228,15 +211,25 @@ class LuigiTaskView(MasterView):
to_delete = session.query(model.Setting)\
.filter(sa.or_(
model.Setting.name == 'rattail.luigi.backfill.tasks',
model.Setting.name == 'rattail.luigi.backfill_tasks',
model.Setting.name.like('rattail.luigi.backfill.task.%.description'),
model.Setting.name.like('rattail.luigi.backfill.%.description'),
model.Setting.name.like('rattail.luigi.backfill.task.%.forward'),
model.Setting.name.like('rattail.luigi.backfill.%.forward'),
model.Setting.name.like('rattail.luigi.backfill.task.%.notes'),
model.Setting.name.like('rattail.luigi.backfill.%.notes'),
model.Setting.name.like('rattail.luigi.backfill.task.%.script'),
model.Setting.name.like('rattail.luigi.backfill.%.script'),
model.Setting.name.like('rattail.luigi.backfill.task.%.target_date'),
model.Setting.name.like('rattail.luigi.backfill.%.target_date'),
model.Setting.name == 'rattail.luigi.overnight.tasks',
model.Setting.name == 'rattail.luigi.overnight_tasks',
model.Setting.name.like('rattail.luigi.overnight.task.%.description'),
model.Setting.name.like('rattail.luigi.overnight.%.description'),
model.Setting.name.like('rattail.luigi.overnight.task.%.notes'),
model.Setting.name.like('rattail.luigi.overnight.%.notes'),
model.Setting.name.like('rattail.luigi.overnight.task.%.script'),
model.Setting.name.like('rattail.luigi.overnight.%.script')))\
.all()