Add basic views for Luigi / overnight tasks
This commit is contained in:
Lance Edgar
parent
9de35a6e8b
commit
d23e5d169a
129
tailbone/templates/luigi/configure.mako
Normal file
129
tailbone/templates/luigi/configure.mako
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
## -*- coding: utf-8; -*-
|
||||
<%inherit file="/configure.mako" />
|
||||
|
||||
<%def name="form_content()">
|
||||
${h.hidden('overnight_tasks', **{':value': 'JSON.stringify(overnightTasks)'})}
|
||||
|
||||
<h3 class="is-size-3">Overnight Tasks</h3>
|
||||
<div class="block" style="padding-left: 2rem; display: flex;">
|
||||
|
||||
<b-table :data="overnightTasks">
|
||||
<template slot-scope="props">
|
||||
<b-table-column field="key"
|
||||
label="Key"
|
||||
sortable>
|
||||
{{ props.row.key }}
|
||||
</b-table-column>
|
||||
</template>
|
||||
</b-table>
|
||||
|
||||
<div style="margin-left: 1rem;">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="plus"
|
||||
@click="overnightTaskCreate()">
|
||||
New Task
|
||||
</b-button>
|
||||
|
||||
<b-modal has-modal-card
|
||||
:active.sync="overnightTaskShowDialog">
|
||||
<div class="modal-card">
|
||||
|
||||
<header class="modal-card-head">
|
||||
<p class="modal-card-title">Overnight Task</p>
|
||||
</header>
|
||||
|
||||
<section class="modal-card-body">
|
||||
<b-field label="Key">
|
||||
<b-input v-model.trim="overnightTaskKey"
|
||||
ref="overnightTaskKey">
|
||||
</b-input>
|
||||
</b-field>
|
||||
</section>
|
||||
|
||||
<footer class="modal-card-foot">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="save"
|
||||
@click="overnightTaskSave()"
|
||||
:disabled="!overnightTaskKey">
|
||||
Save
|
||||
</b-button>
|
||||
<b-button @click="overnightTaskShowDialog = false">
|
||||
Cancel
|
||||
</b-button>
|
||||
</footer>
|
||||
</div>
|
||||
</b-modal>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3 class="is-size-3">Luigi Proper</h3>
|
||||
<div class="block" style="padding-left: 2rem;">
|
||||
|
||||
<b-field label="Luigi URL"
|
||||
message="This should be the URL to Luigi Task Visualiser web user interface."
|
||||
expanded>
|
||||
<b-input name="luigi.url"
|
||||
v-model="simpleSettings['luigi.url']"
|
||||
@input="settingsNeedSaved = true">
|
||||
</b-input>
|
||||
</b-field>
|
||||
|
||||
<b-field label="Supervisor Process Name"
|
||||
message="This should be the complete name, including group - e.g. luigi:luigid"
|
||||
expanded>
|
||||
<b-input name="luigi.scheduler.supervisor_process_name"
|
||||
v-model="simpleSettings['luigi.scheduler.supervisor_process_name']"
|
||||
@input="settingsNeedSaved = true">
|
||||
</b-input>
|
||||
</b-field>
|
||||
|
||||
<b-field label="Restart Command"
|
||||
message="This will run as '${system_user}' system user - please configure sudoers as needed. Typical command is like: sudo supervisorctl restart luigi:luigid"
|
||||
expanded>
|
||||
<b-input name="luigi.scheduler.restart_command"
|
||||
v-model="simpleSettings['luigi.scheduler.restart_command']"
|
||||
@input="settingsNeedSaved = true">
|
||||
</b-input>
|
||||
</b-field>
|
||||
|
||||
</div>
|
||||
|
||||
</%def>
|
||||
|
||||
<%def name="modify_this_page_vars()">
|
||||
${parent.modify_this_page_vars()}
|
||||
<script type="text/javascript">
|
||||
|
||||
ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n}
|
||||
ThisPageData.overnightTaskShowDialog = false
|
||||
ThisPageData.overnightTask = null
|
||||
ThisPageData.overnightTaskKey = null
|
||||
|
||||
ThisPage.methods.overnightTaskCreate = function() {
|
||||
this.overnightTask = null
|
||||
this.overnightTaskKey = null
|
||||
this.overnightTaskShowDialog = true
|
||||
this.$nextTick(() => {
|
||||
this.$refs.overnightTaskKey.focus()
|
||||
})
|
||||
}
|
||||
|
||||
ThisPage.methods.overnightTaskSave = function() {
|
||||
if (this.overnightTask) {
|
||||
this.overnightTask.key = this.overnightTaskKey
|
||||
} else {
|
||||
let task = {key: this.overnightTaskKey}
|
||||
this.overnightTasks.push(task)
|
||||
}
|
||||
this.overnightTaskShowDialog = false
|
||||
this.settingsNeedSaved = true
|
||||
}
|
||||
|
||||
</script>
|
||||
</%def>
|
||||
|
||||
|
||||
${parent.body()}
|
||||
126
tailbone/templates/luigi/index.mako
Normal file
126
tailbone/templates/luigi/index.mako
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
## -*- coding: utf-8; -*-
|
||||
<%inherit file="/page.mako" />
|
||||
|
||||
<%def name="title()">Luigi Jobs</%def>
|
||||
|
||||
<%def name="page_content()">
|
||||
<br />
|
||||
<div class="form">
|
||||
|
||||
<div class="buttons">
|
||||
|
||||
<b-button tag="a"
|
||||
% if luigi_url:
|
||||
href="${luigi_url}"
|
||||
% else:
|
||||
href="#" disabled
|
||||
title="Luigi URL is not configured"
|
||||
% endif
|
||||
icon-pack="fas"
|
||||
icon-left="external-link-alt"
|
||||
target="_blank">
|
||||
Luigi Task Visualiser
|
||||
</b-button>
|
||||
|
||||
<b-button tag="a"
|
||||
% if luigi_history_url:
|
||||
href="${luigi_history_url}"
|
||||
% else:
|
||||
href="#" disabled
|
||||
title="Luigi URL is not configured"
|
||||
% endif
|
||||
icon-pack="fas"
|
||||
icon-left="external-link-alt"
|
||||
target="_blank">
|
||||
Luigi Task History
|
||||
</b-button>
|
||||
|
||||
% if master.has_perm('restart_scheduler'):
|
||||
${h.form(url('{}.restart_scheduler'.format(route_prefix)), **{'@submit': 'submitRestartSchedulerForm'})}
|
||||
${h.csrf_token(request)}
|
||||
<b-button type="is-primary"
|
||||
native-type="submit"
|
||||
icon-pack="fas"
|
||||
icon-left="redo"
|
||||
:disabled="restartSchedulerFormSubmitting">
|
||||
{{ restartSchedulerFormSubmitting ? "Working, please wait..." : "Restart Luigi Scheduler" }}
|
||||
</b-button>
|
||||
${h.end_form()}
|
||||
% endif
|
||||
</div>
|
||||
|
||||
% if master.has_perm('launch'):
|
||||
<h3 class="block is-size-3">Overnight Tasks</h3>
|
||||
% for task in overnight_tasks:
|
||||
<launch-job job-name="${task['key']}"
|
||||
button-text="Restart Overnight ${task['key'].capitalize()}">
|
||||
</launch-job>
|
||||
% endfor
|
||||
% endif
|
||||
|
||||
</div>
|
||||
</%def>
|
||||
|
||||
<%def name="modify_this_page_vars()">
|
||||
${parent.modify_this_page_vars()}
|
||||
% if master.has_perm('restart_scheduler'):
|
||||
<script type="text/javascript">
|
||||
|
||||
ThisPageData.restartSchedulerFormSubmitting = false
|
||||
|
||||
ThisPage.methods.submitRestartSchedulerForm = function() {
|
||||
this.restartSchedulerFormSubmitting = true
|
||||
}
|
||||
|
||||
</script>
|
||||
% endif
|
||||
</%def>
|
||||
|
||||
<%def name="finalize_this_page_vars()">
|
||||
${parent.finalize_this_page_vars()}
|
||||
% if master.has_perm('launch'):
|
||||
<script type="text/javascript">
|
||||
|
||||
const LaunchJob = {
|
||||
template: '#launch-job-template',
|
||||
props: {
|
||||
jobName: String,
|
||||
buttonText: String,
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
formSubmitting: false,
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
submitForm() {
|
||||
this.formSubmitting = true
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
Vue.component('launch-job', LaunchJob)
|
||||
|
||||
</script>
|
||||
% endif
|
||||
</%def>
|
||||
|
||||
<%def name="render_this_page_template()">
|
||||
${parent.render_this_page_template()}
|
||||
% if master.has_perm('launch'):
|
||||
<script type="text/x-template" id="launch-job-template">
|
||||
${h.form(url('{}.launch'.format(route_prefix)), method='post', **{'@submit': 'submitForm'})}
|
||||
${h.csrf_token(request)}
|
||||
<input type="hidden" name="job" v-model="jobName" />
|
||||
<b-button type="is-primary"
|
||||
native-type="submit"
|
||||
:disabled="formSubmitting">
|
||||
{{ formSubmitting ? "Working, please wait..." : buttonText }}
|
||||
</b-button>
|
||||
${h.end_form()}
|
||||
</script>
|
||||
% endif
|
||||
</%def>
|
||||
|
||||
|
||||
${parent.body()}
|
||||
|
|
@ -26,7 +26,6 @@ DataSync Views
|
|||
|
||||
from __future__ import unicode_literals, absolute_import
|
||||
|
||||
import getpass
|
||||
import json
|
||||
import subprocess
|
||||
import logging
|
||||
|
|
@ -234,7 +233,6 @@ class DataSyncThreadView(MasterView):
|
|||
'rattail.datasync', 'supervisor_process_name'),
|
||||
'restart_command': self.rattail_config.get(
|
||||
'tailbone', 'datasync.restart'),
|
||||
'system_user': getpass.getuser(),
|
||||
}
|
||||
|
||||
def configure_gather_settings(self, data):
|
||||
|
|
|
|||
164
tailbone/views/luigi.py
Normal file
164
tailbone/views/luigi.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# Rattail -- Retail Software Framework
|
||||
# Copyright © 2010-2022 Lance Edgar
|
||||
#
|
||||
# This file is part of Rattail.
|
||||
#
|
||||
# Rattail is free software: you can redistribute it and/or modify it under the
|
||||
# terms of the GNU General Public License as published by the Free Software
|
||||
# Foundation, either version 3 of the License, or (at your option) any later
|
||||
# version.
|
||||
#
|
||||
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Rattail. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Views for Luigi
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals, absolute_import
|
||||
|
||||
import json
|
||||
|
||||
from rattail.util import simple_error
|
||||
|
||||
from tailbone.views import MasterView
|
||||
|
||||
|
||||
class LuigiJobView(MasterView):
|
||||
"""
|
||||
Simple views for Luigi jobs.
|
||||
"""
|
||||
normalized_model_name = 'luigijobs'
|
||||
model_key = 'jobname'
|
||||
model_title = "Luigi Job"
|
||||
route_prefix = 'luigi'
|
||||
url_prefix = '/luigi'
|
||||
|
||||
viewable = False
|
||||
creatable = False
|
||||
editable = False
|
||||
deletable = False
|
||||
configurable = True
|
||||
|
||||
def __init__(self, request, context=None):
|
||||
super(LuigiJobView, self).__init__(request, context=context)
|
||||
app = self.get_rattail_app()
|
||||
self.luigi_handler = app.get_luigi_handler()
|
||||
|
||||
def index(self):
|
||||
luigi_url = self.rattail_config.get('luigi', 'url')
|
||||
history_url = '{}/history'.format(luigi_url.rstrip('/')) if luigi_url else None
|
||||
return self.render_to_response('index', {
|
||||
'use_buefy': self.get_use_buefy(),
|
||||
'index_url': None,
|
||||
'luigi_url': luigi_url,
|
||||
'luigi_history_url': history_url,
|
||||
'overnight_tasks': self.luigi_handler.get_all_overnight_tasks(),
|
||||
})
|
||||
|
||||
def launch(self):
|
||||
key = self.request.POST['job']
|
||||
assert key
|
||||
self.luigi_handler.restart_overnight_task(key)
|
||||
self.request.session.flash("Scheduled overnight task for immediate launch: {}".format(key))
|
||||
return self.redirect(self.get_index_url())
|
||||
|
||||
def restart_scheduler(self):
|
||||
try:
|
||||
self.luigi_handler.restart_supervisor_process()
|
||||
self.request.session.flash("Luigi scheduler has been restarted.")
|
||||
|
||||
except Exception as error:
|
||||
self.request.session.flash(simple_error(error), 'error')
|
||||
|
||||
return self.redirect(self.request.get_referrer(
|
||||
default=self.get_index_url()))
|
||||
|
||||
def configure_get_simple_settings(self):
|
||||
return [
|
||||
|
||||
# luigi proper
|
||||
{'section': 'luigi',
|
||||
'option': 'url'},
|
||||
{'section': 'luigi',
|
||||
'option': 'scheduler.supervisor_process_name'},
|
||||
{'section': 'luigi',
|
||||
'option': 'scheduler.restart_command'},
|
||||
|
||||
]
|
||||
|
||||
def configure_get_context(self, **kwargs):
|
||||
context = super(LuigiJobView, self).configure_get_context(**kwargs)
|
||||
context['overnight_tasks'] = self.luigi_handler.get_all_overnight_tasks()
|
||||
return context
|
||||
|
||||
def configure_gather_settings(self, data):
|
||||
settings = super(LuigiJobView, self).configure_gather_settings(data)
|
||||
|
||||
keys = []
|
||||
for task in json.loads(data['overnight_tasks']):
|
||||
keys.append(task['key'])
|
||||
|
||||
if keys:
|
||||
settings.append({'name': 'luigi.overnight_tasks',
|
||||
'value': ', '.join(keys)})
|
||||
|
||||
return settings
|
||||
|
||||
def configure_remove_settings(self):
|
||||
super(LuigiJobView, self).configure_remove_settings()
|
||||
self.luigi_handler.purge_luigi_settings(self.Session())
|
||||
|
||||
@classmethod
|
||||
def defaults(cls, config):
|
||||
cls._defaults(config)
|
||||
cls._luigi_defaults(config)
|
||||
|
||||
@classmethod
|
||||
def _luigi_defaults(cls, config):
|
||||
route_prefix = cls.get_route_prefix()
|
||||
permission_prefix = cls.get_permission_prefix()
|
||||
url_prefix = cls.get_url_prefix()
|
||||
model_title_plural = cls.get_model_title_plural()
|
||||
|
||||
# launch job
|
||||
config.add_tailbone_permission(permission_prefix,
|
||||
'{}.launch'.format(permission_prefix),
|
||||
label="Launch any Luigi job")
|
||||
config.add_route('{}.launch'.format(route_prefix),
|
||||
'{}/launch'.format(url_prefix),
|
||||
request_method='POST')
|
||||
config.add_view(cls, attr='launch',
|
||||
route_name='{}.launch'.format(route_prefix),
|
||||
permission='{}.launch'.format(permission_prefix))
|
||||
|
||||
# restart luigid scheduler
|
||||
config.add_tailbone_permission(permission_prefix,
|
||||
'{}.restart_scheduler'.format(permission_prefix),
|
||||
label="Restart the Luigi Scheduler daemon")
|
||||
config.add_route('{}.restart_scheduler'.format(route_prefix),
|
||||
'{}/restart-scheduler'.format(url_prefix),
|
||||
request_method='POST')
|
||||
config.add_view(cls, attr='restart_scheduler',
|
||||
route_name='{}.restart_scheduler'.format(route_prefix),
|
||||
permission='{}.restart_scheduler'.format(permission_prefix))
|
||||
|
||||
|
||||
def defaults(config, **kwargs):
|
||||
base = globals()
|
||||
|
||||
LuigiJobView = kwargs.get('LuigiJobView', base['LuigiJobView'])
|
||||
LuigiJobView.defaults(config)
|
||||
|
||||
|
||||
def includeme(config):
|
||||
defaults(config)
|
||||
|
|
@ -29,6 +29,7 @@ from __future__ import unicode_literals, absolute_import
|
|||
import os
|
||||
import csv
|
||||
import datetime
|
||||
import getpass
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
|
|
@ -4324,6 +4325,10 @@ class MasterView(View):
|
|||
context = self.configure_get_context()
|
||||
return self.render_to_response('configure', context)
|
||||
|
||||
def template_kwargs_configure(self, **kwargs):
|
||||
kwargs['system_user'] = getpass.getuser()
|
||||
return kwargs
|
||||
|
||||
def configure_flash_settings_saved(self):
|
||||
self.request.session.flash("Settings have been saved.")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue