Add basic views for Luigi / overnight tasks
This commit is contained in:
Lance Edgar
parent
9de35a6e8b
commit
d23e5d169a
129
tailbone/templates/luigi/configure.mako
Normal file
129
tailbone/templates/luigi/configure.mako
Normal file
|
|
@ -0,0 +1,129 @@
|
||||||
|
## -*- coding: utf-8; -*-
|
||||||
|
<%inherit file="/configure.mako" />
|
||||||
|
|
||||||
|
<%def name="form_content()">
|
||||||
|
${h.hidden('overnight_tasks', **{':value': 'JSON.stringify(overnightTasks)'})}
|
||||||
|
|
||||||
|
<h3 class="is-size-3">Overnight Tasks</h3>
|
||||||
|
<div class="block" style="padding-left: 2rem; display: flex;">
|
||||||
|
|
||||||
|
<b-table :data="overnightTasks">
|
||||||
|
<template slot-scope="props">
|
||||||
|
<b-table-column field="key"
|
||||||
|
label="Key"
|
||||||
|
sortable>
|
||||||
|
{{ props.row.key }}
|
||||||
|
</b-table-column>
|
||||||
|
</template>
|
||||||
|
</b-table>
|
||||||
|
|
||||||
|
<div style="margin-left: 1rem;">
|
||||||
|
<b-button type="is-primary"
|
||||||
|
icon-pack="fas"
|
||||||
|
icon-left="plus"
|
||||||
|
@click="overnightTaskCreate()">
|
||||||
|
New Task
|
||||||
|
</b-button>
|
||||||
|
|
||||||
|
<b-modal has-modal-card
|
||||||
|
:active.sync="overnightTaskShowDialog">
|
||||||
|
<div class="modal-card">
|
||||||
|
|
||||||
|
<header class="modal-card-head">
|
||||||
|
<p class="modal-card-title">Overnight Task</p>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section class="modal-card-body">
|
||||||
|
<b-field label="Key">
|
||||||
|
<b-input v-model.trim="overnightTaskKey"
|
||||||
|
ref="overnightTaskKey">
|
||||||
|
</b-input>
|
||||||
|
</b-field>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<footer class="modal-card-foot">
|
||||||
|
<b-button type="is-primary"
|
||||||
|
icon-pack="fas"
|
||||||
|
icon-left="save"
|
||||||
|
@click="overnightTaskSave()"
|
||||||
|
:disabled="!overnightTaskKey">
|
||||||
|
Save
|
||||||
|
</b-button>
|
||||||
|
<b-button @click="overnightTaskShowDialog = false">
|
||||||
|
Cancel
|
||||||
|
</b-button>
|
||||||
|
</footer>
|
||||||
|
</div>
|
||||||
|
</b-modal>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h3 class="is-size-3">Luigi Proper</h3>
|
||||||
|
<div class="block" style="padding-left: 2rem;">
|
||||||
|
|
||||||
|
<b-field label="Luigi URL"
|
||||||
|
message="This should be the URL to Luigi Task Visualiser web user interface."
|
||||||
|
expanded>
|
||||||
|
<b-input name="luigi.url"
|
||||||
|
v-model="simpleSettings['luigi.url']"
|
||||||
|
@input="settingsNeedSaved = true">
|
||||||
|
</b-input>
|
||||||
|
</b-field>
|
||||||
|
|
||||||
|
<b-field label="Supervisor Process Name"
|
||||||
|
message="This should be the complete name, including group - e.g. luigi:luigid"
|
||||||
|
expanded>
|
||||||
|
<b-input name="luigi.scheduler.supervisor_process_name"
|
||||||
|
v-model="simpleSettings['luigi.scheduler.supervisor_process_name']"
|
||||||
|
@input="settingsNeedSaved = true">
|
||||||
|
</b-input>
|
||||||
|
</b-field>
|
||||||
|
|
||||||
|
<b-field label="Restart Command"
|
||||||
|
message="This will run as '${system_user}' system user - please configure sudoers as needed. Typical command is like: sudo supervisorctl restart luigi:luigid"
|
||||||
|
expanded>
|
||||||
|
<b-input name="luigi.scheduler.restart_command"
|
||||||
|
v-model="simpleSettings['luigi.scheduler.restart_command']"
|
||||||
|
@input="settingsNeedSaved = true">
|
||||||
|
</b-input>
|
||||||
|
</b-field>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
<%def name="modify_this_page_vars()">
|
||||||
|
${parent.modify_this_page_vars()}
|
||||||
|
<script type="text/javascript">
|
||||||
|
|
||||||
|
ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n}
|
||||||
|
ThisPageData.overnightTaskShowDialog = false
|
||||||
|
ThisPageData.overnightTask = null
|
||||||
|
ThisPageData.overnightTaskKey = null
|
||||||
|
|
||||||
|
ThisPage.methods.overnightTaskCreate = function() {
|
||||||
|
this.overnightTask = null
|
||||||
|
this.overnightTaskKey = null
|
||||||
|
this.overnightTaskShowDialog = true
|
||||||
|
this.$nextTick(() => {
|
||||||
|
this.$refs.overnightTaskKey.focus()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
ThisPage.methods.overnightTaskSave = function() {
|
||||||
|
if (this.overnightTask) {
|
||||||
|
this.overnightTask.key = this.overnightTaskKey
|
||||||
|
} else {
|
||||||
|
let task = {key: this.overnightTaskKey}
|
||||||
|
this.overnightTasks.push(task)
|
||||||
|
}
|
||||||
|
this.overnightTaskShowDialog = false
|
||||||
|
this.settingsNeedSaved = true
|
||||||
|
}
|
||||||
|
|
||||||
|
</script>
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
|
||||||
|
${parent.body()}
|
||||||
126
tailbone/templates/luigi/index.mako
Normal file
126
tailbone/templates/luigi/index.mako
Normal file
|
|
@ -0,0 +1,126 @@
|
||||||
|
## -*- coding: utf-8; -*-
|
||||||
|
<%inherit file="/page.mako" />
|
||||||
|
|
||||||
|
<%def name="title()">Luigi Jobs</%def>
|
||||||
|
|
||||||
|
<%def name="page_content()">
|
||||||
|
<br />
|
||||||
|
<div class="form">
|
||||||
|
|
||||||
|
<div class="buttons">
|
||||||
|
|
||||||
|
<b-button tag="a"
|
||||||
|
% if luigi_url:
|
||||||
|
href="${luigi_url}"
|
||||||
|
% else:
|
||||||
|
href="#" disabled
|
||||||
|
title="Luigi URL is not configured"
|
||||||
|
% endif
|
||||||
|
icon-pack="fas"
|
||||||
|
icon-left="external-link-alt"
|
||||||
|
target="_blank">
|
||||||
|
Luigi Task Visualiser
|
||||||
|
</b-button>
|
||||||
|
|
||||||
|
<b-button tag="a"
|
||||||
|
% if luigi_history_url:
|
||||||
|
href="${luigi_history_url}"
|
||||||
|
% else:
|
||||||
|
href="#" disabled
|
||||||
|
title="Luigi URL is not configured"
|
||||||
|
% endif
|
||||||
|
icon-pack="fas"
|
||||||
|
icon-left="external-link-alt"
|
||||||
|
target="_blank">
|
||||||
|
Luigi Task History
|
||||||
|
</b-button>
|
||||||
|
|
||||||
|
% if master.has_perm('restart_scheduler'):
|
||||||
|
${h.form(url('{}.restart_scheduler'.format(route_prefix)), **{'@submit': 'submitRestartSchedulerForm'})}
|
||||||
|
${h.csrf_token(request)}
|
||||||
|
<b-button type="is-primary"
|
||||||
|
native-type="submit"
|
||||||
|
icon-pack="fas"
|
||||||
|
icon-left="redo"
|
||||||
|
:disabled="restartSchedulerFormSubmitting">
|
||||||
|
{{ restartSchedulerFormSubmitting ? "Working, please wait..." : "Restart Luigi Scheduler" }}
|
||||||
|
</b-button>
|
||||||
|
${h.end_form()}
|
||||||
|
% endif
|
||||||
|
</div>
|
||||||
|
|
||||||
|
% if master.has_perm('launch'):
|
||||||
|
<h3 class="block is-size-3">Overnight Tasks</h3>
|
||||||
|
% for task in overnight_tasks:
|
||||||
|
<launch-job job-name="${task['key']}"
|
||||||
|
button-text="Restart Overnight ${task['key'].capitalize()}">
|
||||||
|
</launch-job>
|
||||||
|
% endfor
|
||||||
|
% endif
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
<%def name="modify_this_page_vars()">
|
||||||
|
${parent.modify_this_page_vars()}
|
||||||
|
% if master.has_perm('restart_scheduler'):
|
||||||
|
<script type="text/javascript">
|
||||||
|
|
||||||
|
ThisPageData.restartSchedulerFormSubmitting = false
|
||||||
|
|
||||||
|
ThisPage.methods.submitRestartSchedulerForm = function() {
|
||||||
|
this.restartSchedulerFormSubmitting = true
|
||||||
|
}
|
||||||
|
|
||||||
|
</script>
|
||||||
|
% endif
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
<%def name="finalize_this_page_vars()">
|
||||||
|
${parent.finalize_this_page_vars()}
|
||||||
|
% if master.has_perm('launch'):
|
||||||
|
<script type="text/javascript">
|
||||||
|
|
||||||
|
const LaunchJob = {
|
||||||
|
template: '#launch-job-template',
|
||||||
|
props: {
|
||||||
|
jobName: String,
|
||||||
|
buttonText: String,
|
||||||
|
},
|
||||||
|
data() {
|
||||||
|
return {
|
||||||
|
formSubmitting: false,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
methods: {
|
||||||
|
submitForm() {
|
||||||
|
this.formSubmitting = true
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
Vue.component('launch-job', LaunchJob)
|
||||||
|
|
||||||
|
</script>
|
||||||
|
% endif
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
<%def name="render_this_page_template()">
|
||||||
|
${parent.render_this_page_template()}
|
||||||
|
% if master.has_perm('launch'):
|
||||||
|
<script type="text/x-template" id="launch-job-template">
|
||||||
|
${h.form(url('{}.launch'.format(route_prefix)), method='post', **{'@submit': 'submitForm'})}
|
||||||
|
${h.csrf_token(request)}
|
||||||
|
<input type="hidden" name="job" v-model="jobName" />
|
||||||
|
<b-button type="is-primary"
|
||||||
|
native-type="submit"
|
||||||
|
:disabled="formSubmitting">
|
||||||
|
{{ formSubmitting ? "Working, please wait..." : buttonText }}
|
||||||
|
</b-button>
|
||||||
|
${h.end_form()}
|
||||||
|
</script>
|
||||||
|
% endif
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
|
||||||
|
${parent.body()}
|
||||||
|
|
@ -26,7 +26,6 @@ DataSync Views
|
||||||
|
|
||||||
from __future__ import unicode_literals, absolute_import
|
from __future__ import unicode_literals, absolute_import
|
||||||
|
|
||||||
import getpass
|
|
||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
import logging
|
import logging
|
||||||
|
|
@ -234,7 +233,6 @@ class DataSyncThreadView(MasterView):
|
||||||
'rattail.datasync', 'supervisor_process_name'),
|
'rattail.datasync', 'supervisor_process_name'),
|
||||||
'restart_command': self.rattail_config.get(
|
'restart_command': self.rattail_config.get(
|
||||||
'tailbone', 'datasync.restart'),
|
'tailbone', 'datasync.restart'),
|
||||||
'system_user': getpass.getuser(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def configure_gather_settings(self, data):
|
def configure_gather_settings(self, data):
|
||||||
|
|
|
||||||
164
tailbone/views/luigi.py
Normal file
164
tailbone/views/luigi.py
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
# -*- coding: utf-8; -*-
|
||||||
|
################################################################################
|
||||||
|
#
|
||||||
|
# Rattail -- Retail Software Framework
|
||||||
|
# Copyright © 2010-2022 Lance Edgar
|
||||||
|
#
|
||||||
|
# This file is part of Rattail.
|
||||||
|
#
|
||||||
|
# Rattail is free software: you can redistribute it and/or modify it under the
|
||||||
|
# terms of the GNU General Public License as published by the Free Software
|
||||||
|
# Foundation, either version 3 of the License, or (at your option) any later
|
||||||
|
# version.
|
||||||
|
#
|
||||||
|
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||||
|
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||||
|
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||||
|
# details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License along with
|
||||||
|
# Rattail. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
################################################################################
|
||||||
|
"""
|
||||||
|
Views for Luigi
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import unicode_literals, absolute_import
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from rattail.util import simple_error
|
||||||
|
|
||||||
|
from tailbone.views import MasterView
|
||||||
|
|
||||||
|
|
||||||
|
class LuigiJobView(MasterView):
|
||||||
|
"""
|
||||||
|
Simple views for Luigi jobs.
|
||||||
|
"""
|
||||||
|
normalized_model_name = 'luigijobs'
|
||||||
|
model_key = 'jobname'
|
||||||
|
model_title = "Luigi Job"
|
||||||
|
route_prefix = 'luigi'
|
||||||
|
url_prefix = '/luigi'
|
||||||
|
|
||||||
|
viewable = False
|
||||||
|
creatable = False
|
||||||
|
editable = False
|
||||||
|
deletable = False
|
||||||
|
configurable = True
|
||||||
|
|
||||||
|
def __init__(self, request, context=None):
|
||||||
|
super(LuigiJobView, self).__init__(request, context=context)
|
||||||
|
app = self.get_rattail_app()
|
||||||
|
self.luigi_handler = app.get_luigi_handler()
|
||||||
|
|
||||||
|
def index(self):
|
||||||
|
luigi_url = self.rattail_config.get('luigi', 'url')
|
||||||
|
history_url = '{}/history'.format(luigi_url.rstrip('/')) if luigi_url else None
|
||||||
|
return self.render_to_response('index', {
|
||||||
|
'use_buefy': self.get_use_buefy(),
|
||||||
|
'index_url': None,
|
||||||
|
'luigi_url': luigi_url,
|
||||||
|
'luigi_history_url': history_url,
|
||||||
|
'overnight_tasks': self.luigi_handler.get_all_overnight_tasks(),
|
||||||
|
})
|
||||||
|
|
||||||
|
def launch(self):
|
||||||
|
key = self.request.POST['job']
|
||||||
|
assert key
|
||||||
|
self.luigi_handler.restart_overnight_task(key)
|
||||||
|
self.request.session.flash("Scheduled overnight task for immediate launch: {}".format(key))
|
||||||
|
return self.redirect(self.get_index_url())
|
||||||
|
|
||||||
|
def restart_scheduler(self):
|
||||||
|
try:
|
||||||
|
self.luigi_handler.restart_supervisor_process()
|
||||||
|
self.request.session.flash("Luigi scheduler has been restarted.")
|
||||||
|
|
||||||
|
except Exception as error:
|
||||||
|
self.request.session.flash(simple_error(error), 'error')
|
||||||
|
|
||||||
|
return self.redirect(self.request.get_referrer(
|
||||||
|
default=self.get_index_url()))
|
||||||
|
|
||||||
|
def configure_get_simple_settings(self):
|
||||||
|
return [
|
||||||
|
|
||||||
|
# luigi proper
|
||||||
|
{'section': 'luigi',
|
||||||
|
'option': 'url'},
|
||||||
|
{'section': 'luigi',
|
||||||
|
'option': 'scheduler.supervisor_process_name'},
|
||||||
|
{'section': 'luigi',
|
||||||
|
'option': 'scheduler.restart_command'},
|
||||||
|
|
||||||
|
]
|
||||||
|
|
||||||
|
def configure_get_context(self, **kwargs):
|
||||||
|
context = super(LuigiJobView, self).configure_get_context(**kwargs)
|
||||||
|
context['overnight_tasks'] = self.luigi_handler.get_all_overnight_tasks()
|
||||||
|
return context
|
||||||
|
|
||||||
|
def configure_gather_settings(self, data):
|
||||||
|
settings = super(LuigiJobView, self).configure_gather_settings(data)
|
||||||
|
|
||||||
|
keys = []
|
||||||
|
for task in json.loads(data['overnight_tasks']):
|
||||||
|
keys.append(task['key'])
|
||||||
|
|
||||||
|
if keys:
|
||||||
|
settings.append({'name': 'luigi.overnight_tasks',
|
||||||
|
'value': ', '.join(keys)})
|
||||||
|
|
||||||
|
return settings
|
||||||
|
|
||||||
|
def configure_remove_settings(self):
|
||||||
|
super(LuigiJobView, self).configure_remove_settings()
|
||||||
|
self.luigi_handler.purge_luigi_settings(self.Session())
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def defaults(cls, config):
|
||||||
|
cls._defaults(config)
|
||||||
|
cls._luigi_defaults(config)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _luigi_defaults(cls, config):
|
||||||
|
route_prefix = cls.get_route_prefix()
|
||||||
|
permission_prefix = cls.get_permission_prefix()
|
||||||
|
url_prefix = cls.get_url_prefix()
|
||||||
|
model_title_plural = cls.get_model_title_plural()
|
||||||
|
|
||||||
|
# launch job
|
||||||
|
config.add_tailbone_permission(permission_prefix,
|
||||||
|
'{}.launch'.format(permission_prefix),
|
||||||
|
label="Launch any Luigi job")
|
||||||
|
config.add_route('{}.launch'.format(route_prefix),
|
||||||
|
'{}/launch'.format(url_prefix),
|
||||||
|
request_method='POST')
|
||||||
|
config.add_view(cls, attr='launch',
|
||||||
|
route_name='{}.launch'.format(route_prefix),
|
||||||
|
permission='{}.launch'.format(permission_prefix))
|
||||||
|
|
||||||
|
# restart luigid scheduler
|
||||||
|
config.add_tailbone_permission(permission_prefix,
|
||||||
|
'{}.restart_scheduler'.format(permission_prefix),
|
||||||
|
label="Restart the Luigi Scheduler daemon")
|
||||||
|
config.add_route('{}.restart_scheduler'.format(route_prefix),
|
||||||
|
'{}/restart-scheduler'.format(url_prefix),
|
||||||
|
request_method='POST')
|
||||||
|
config.add_view(cls, attr='restart_scheduler',
|
||||||
|
route_name='{}.restart_scheduler'.format(route_prefix),
|
||||||
|
permission='{}.restart_scheduler'.format(permission_prefix))
|
||||||
|
|
||||||
|
|
||||||
|
def defaults(config, **kwargs):
|
||||||
|
base = globals()
|
||||||
|
|
||||||
|
LuigiJobView = kwargs.get('LuigiJobView', base['LuigiJobView'])
|
||||||
|
LuigiJobView.defaults(config)
|
||||||
|
|
||||||
|
|
||||||
|
def includeme(config):
|
||||||
|
defaults(config)
|
||||||
|
|
@ -29,6 +29,7 @@ from __future__ import unicode_literals, absolute_import
|
||||||
import os
|
import os
|
||||||
import csv
|
import csv
|
||||||
import datetime
|
import datetime
|
||||||
|
import getpass
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
import logging
|
import logging
|
||||||
|
|
@ -4324,6 +4325,10 @@ class MasterView(View):
|
||||||
context = self.configure_get_context()
|
context = self.configure_get_context()
|
||||||
return self.render_to_response('configure', context)
|
return self.render_to_response('configure', context)
|
||||||
|
|
||||||
|
def template_kwargs_configure(self, **kwargs):
|
||||||
|
kwargs['system_user'] = getpass.getuser()
|
||||||
|
return kwargs
|
||||||
|
|
||||||
def configure_flash_settings_saved(self):
|
def configure_flash_settings_saved(self):
|
||||||
self.request.session.flash("Settings have been saved.")
|
self.request.session.flash("Settings have been saved.")
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue