Add basic support for backfill Luigi tasks
idea being, sometimes you must import many days worth of data into Trainwreck or what-not, and it must be split up b/c e.g. it would take too long to import all at once (i.e. might interfere with overnight tasks)
This commit is contained in:
Lance Edgar
parent
488696cb39
commit
78500770d9
|
|
@ -3,27 +3,56 @@
|
|||
|
||||
<%def name="form_content()">
|
||||
${h.hidden('overnight_tasks', **{':value': 'JSON.stringify(overnightTasks)'})}
|
||||
${h.hidden('backfill_tasks', **{':value': 'JSON.stringify(backfillTasks)'})}
|
||||
|
||||
<div class="level">
|
||||
<div class="level-left">
|
||||
<div class="level-item">
|
||||
<h3 class="is-size-3">Overnight Tasks</h3>
|
||||
<div class="block" style="padding-left: 2rem; display: flex;">
|
||||
|
||||
<b-table :data="overnightTasks">
|
||||
<template slot-scope="props">
|
||||
<b-table-column field="key"
|
||||
label="Key"
|
||||
sortable>
|
||||
{{ props.row.key }}
|
||||
</b-table-column>
|
||||
</template>
|
||||
</b-table>
|
||||
|
||||
<div style="margin-left: 1rem;">
|
||||
</div>
|
||||
<div class="level-item">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="plus"
|
||||
@click="overnightTaskCreate()">
|
||||
New Task
|
||||
</b-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="block" style="padding-left: 2rem; display: flex;">
|
||||
|
||||
<b-table :data="overnightTasks">
|
||||
<template slot-scope="props">
|
||||
<!-- <b-table-column field="key" -->
|
||||
<!-- label="Key" -->
|
||||
<!-- sortable> -->
|
||||
<!-- {{ props.row.key }} -->
|
||||
<!-- </b-table-column> -->
|
||||
<b-table-column field="description"
|
||||
label="Description">
|
||||
{{ props.row.description }}
|
||||
</b-table-column>
|
||||
<b-table-column field="script"
|
||||
label="Script">
|
||||
{{ props.row.script }}
|
||||
</b-table-column>
|
||||
<b-table-column label="Actions">
|
||||
<a href="#"
|
||||
@click.prevent="overnightTaskEdit(props.row)">
|
||||
<i class="fas fa-edit"></i>
|
||||
Edit
|
||||
</a>
|
||||
|
||||
<a href="#"
|
||||
class="has-text-danger"
|
||||
@click.prevent="overnightTaskDelete(props.row)">
|
||||
<i class="fas fa-trash"></i>
|
||||
Delete
|
||||
</a>
|
||||
</b-table-column>
|
||||
</template>
|
||||
</b-table>
|
||||
|
||||
<b-modal has-modal-card
|
||||
:active.sync="overnightTaskShowDialog">
|
||||
|
|
@ -34,9 +63,25 @@
|
|||
</header>
|
||||
|
||||
<section class="modal-card-body">
|
||||
<b-field label="Key">
|
||||
<b-input v-model.trim="overnightTaskKey"
|
||||
ref="overnightTaskKey">
|
||||
<!-- <b-field label="Key"> -->
|
||||
<!-- <b-input v-model.trim="overnightTaskKey" -->
|
||||
<!-- ref="overnightTaskKey"> -->
|
||||
<!-- </b-input> -->
|
||||
<!-- </b-field> -->
|
||||
<b-field label="Description"
|
||||
:type="overnightTaskDescription ? null : 'is-danger'">
|
||||
<b-input v-model.trim="overnightTaskDescription"
|
||||
ref="overnightTaskDescription">
|
||||
</b-input>
|
||||
</b-field>
|
||||
<b-field label="Script"
|
||||
:type="overnightTaskScript ? null : 'is-danger'">
|
||||
<b-input v-model.trim="overnightTaskScript">
|
||||
</b-input>
|
||||
</b-field>
|
||||
<b-field label="Notes">
|
||||
<b-input v-model.trim="overnightTaskNotes"
|
||||
type="textarea">
|
||||
</b-input>
|
||||
</b-field>
|
||||
</section>
|
||||
|
|
@ -46,7 +91,7 @@
|
|||
icon-pack="fas"
|
||||
icon-left="save"
|
||||
@click="overnightTaskSave()"
|
||||
:disabled="!overnightTaskKey">
|
||||
:disabled="!overnightTaskDescription || !overnightTaskScript">
|
||||
Save
|
||||
</b-button>
|
||||
<b-button @click="overnightTaskShowDialog = false">
|
||||
|
|
@ -57,6 +102,113 @@
|
|||
</b-modal>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="level">
|
||||
<div class="level-left">
|
||||
<div class="level-item">
|
||||
<h3 class="is-size-3">Backfill Tasks</h3>
|
||||
</div>
|
||||
<div class="level-item">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="plus"
|
||||
@click="backfillTaskCreate()">
|
||||
New Task
|
||||
</b-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="block" style="padding-left: 2rem; display: flex;">
|
||||
|
||||
<b-table :data="backfillTasks">
|
||||
<template slot-scope="props">
|
||||
<b-table-column field="description"
|
||||
label="Description">
|
||||
{{ props.row.description }}
|
||||
</b-table-column>
|
||||
<b-table-column field="script"
|
||||
label="Script">
|
||||
{{ props.row.script }}
|
||||
</b-table-column>
|
||||
<b-table-column field="forward"
|
||||
label="Orientation">
|
||||
{{ props.row.forward ? "Forward" : "Backward" }}
|
||||
</b-table-column>
|
||||
<b-table-column field="target_date"
|
||||
label="Target Date">
|
||||
{{ props.row.target_date }}
|
||||
</b-table-column>
|
||||
<b-table-column label="Actions">
|
||||
<a href="#"
|
||||
@click.prevent="backfillTaskEdit(props.row)">
|
||||
<i class="fas fa-edit"></i>
|
||||
Edit
|
||||
</a>
|
||||
|
||||
<a href="#"
|
||||
class="has-text-danger"
|
||||
@click.prevent="backfillTaskDelete(props.row)">
|
||||
<i class="fas fa-trash"></i>
|
||||
Delete
|
||||
</a>
|
||||
</b-table-column>
|
||||
</template>
|
||||
</b-table>
|
||||
|
||||
<b-modal has-modal-card
|
||||
:active.sync="backfillTaskShowDialog">
|
||||
<div class="modal-card">
|
||||
|
||||
<header class="modal-card-head">
|
||||
<p class="modal-card-title">Backfill Task</p>
|
||||
</header>
|
||||
|
||||
<section class="modal-card-body">
|
||||
<b-field label="Description"
|
||||
:type="backfillTaskDescription ? null : 'is-danger'">
|
||||
<b-input v-model.trim="backfillTaskDescription"
|
||||
ref="backfillTaskDescription">
|
||||
</b-input>
|
||||
</b-field>
|
||||
<b-field label="Script"
|
||||
:type="backfillTaskScript ? null : 'is-danger'">
|
||||
<b-input v-model.trim="backfillTaskScript">
|
||||
</b-input>
|
||||
</b-field>
|
||||
<b-field grouped>
|
||||
<b-field label="Orientation">
|
||||
<b-select v-model="backfillTaskForward">
|
||||
<option :value="false">Backward</option>
|
||||
<option :value="true">Forward</option>
|
||||
</b-select>
|
||||
</b-field>
|
||||
<b-field label="Target Date">
|
||||
<tailbone-datepicker v-model="backfillTaskTargetDate">
|
||||
</tailbone-datepicker>
|
||||
</b-field>
|
||||
</b-field>
|
||||
<b-field label="Notes">
|
||||
<b-input v-model.trim="backfillTaskNotes"
|
||||
type="textarea">
|
||||
</b-input>
|
||||
</b-field>
|
||||
</section>
|
||||
|
||||
<footer class="modal-card-foot">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="save"
|
||||
@click="backfillTaskSave()"
|
||||
:disabled="!backfillTaskDescription || !backfillTaskScript">
|
||||
Save
|
||||
</b-button>
|
||||
<b-button @click="backfillTaskShowDialog = false">
|
||||
Cancel
|
||||
</b-button>
|
||||
</footer>
|
||||
</div>
|
||||
</b-modal>
|
||||
|
||||
</div>
|
||||
|
||||
<h3 class="is-size-3">Luigi Proper</h3>
|
||||
|
|
@ -65,8 +217,8 @@
|
|||
<b-field label="Luigi URL"
|
||||
message="This should be the URL to Luigi Task Visualiser web user interface."
|
||||
expanded>
|
||||
<b-input name="luigi.url"
|
||||
v-model="simpleSettings['luigi.url']"
|
||||
<b-input name="rattail.luigi.url"
|
||||
v-model="simpleSettings['rattail.luigi.url']"
|
||||
@input="settingsNeedSaved = true">
|
||||
</b-input>
|
||||
</b-field>
|
||||
|
|
@ -74,8 +226,8 @@
|
|||
<b-field label="Supervisor Process Name"
|
||||
message="This should be the complete name, including group - e.g. luigi:luigid"
|
||||
expanded>
|
||||
<b-input name="luigi.scheduler.supervisor_process_name"
|
||||
v-model="simpleSettings['luigi.scheduler.supervisor_process_name']"
|
||||
<b-input name="rattail.luigi.scheduler.supervisor_process_name"
|
||||
v-model="simpleSettings['rattail.luigi.scheduler.supervisor_process_name']"
|
||||
@input="settingsNeedSaved = true">
|
||||
</b-input>
|
||||
</b-field>
|
||||
|
|
@ -83,8 +235,8 @@
|
|||
<b-field label="Restart Command"
|
||||
message="This will run as '${system_user}' system user - please configure sudoers as needed. Typical command is like: sudo supervisorctl restart luigi:luigid"
|
||||
expanded>
|
||||
<b-input name="luigi.scheduler.restart_command"
|
||||
v-model="simpleSettings['luigi.scheduler.restart_command']"
|
||||
<b-input name="rattail.luigi.scheduler.restart_command"
|
||||
v-model="simpleSettings['rattail.luigi.scheduler.restart_command']"
|
||||
@input="settingsNeedSaved = true">
|
||||
</b-input>
|
||||
</b-field>
|
||||
|
|
@ -100,28 +252,113 @@
|
|||
ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n}
|
||||
ThisPageData.overnightTaskShowDialog = false
|
||||
ThisPageData.overnightTask = null
|
||||
ThisPageData.overnightTaskCounter = 0
|
||||
ThisPageData.overnightTaskKey = null
|
||||
ThisPageData.overnightTaskDescription = null
|
||||
ThisPageData.overnightTaskScript = null
|
||||
ThisPageData.overnightTaskNotes = null
|
||||
|
||||
ThisPage.methods.overnightTaskCreate = function() {
|
||||
this.overnightTask = null
|
||||
this.overnightTask = {key: null}
|
||||
this.overnightTaskKey = null
|
||||
this.overnightTaskDescription = null
|
||||
this.overnightTaskScript = null
|
||||
this.overnightTaskNotes = null
|
||||
this.overnightTaskShowDialog = true
|
||||
this.$nextTick(() => {
|
||||
this.$refs.overnightTaskKey.focus()
|
||||
this.$refs.overnightTaskDescription.focus()
|
||||
})
|
||||
}
|
||||
|
||||
ThisPage.methods.overnightTaskSave = function() {
|
||||
if (this.overnightTask) {
|
||||
this.overnightTask.key = this.overnightTaskKey
|
||||
} else {
|
||||
let task = {key: this.overnightTaskKey}
|
||||
this.overnightTasks.push(task)
|
||||
ThisPage.methods.overnightTaskEdit = function(task) {
|
||||
this.overnightTask = task
|
||||
this.overnightTaskKey = task.key
|
||||
this.overnightTaskDescription = task.description
|
||||
this.overnightTaskScript = task.script
|
||||
this.overnightTaskNotes = task.notes
|
||||
this.overnightTaskShowDialog = true
|
||||
}
|
||||
|
||||
ThisPage.methods.overnightTaskSave = function() {
|
||||
this.overnightTask.description = this.overnightTaskDescription
|
||||
this.overnightTask.script = this.overnightTaskScript
|
||||
this.overnightTask.notes = this.overnightTaskNotes
|
||||
|
||||
if (!this.overnightTask.key) {
|
||||
this.overnightTask.key = `_new_${'$'}{++this.overnightTaskCounter}`
|
||||
this.overnightTasks.push(this.overnightTask)
|
||||
}
|
||||
|
||||
this.overnightTaskShowDialog = false
|
||||
this.settingsNeedSaved = true
|
||||
}
|
||||
|
||||
ThisPage.methods.overnightTaskDelete = function(task) {
|
||||
if (confirm("Really delete this task?")) {
|
||||
let i = this.overnightTasks.indexOf(task)
|
||||
this.overnightTasks.splice(i, 1)
|
||||
this.settingsNeedSaved = true
|
||||
}
|
||||
}
|
||||
|
||||
ThisPageData.backfillTasks = ${json.dumps(backfill_tasks)|n}
|
||||
ThisPageData.backfillTaskShowDialog = false
|
||||
ThisPageData.backfillTask = null
|
||||
ThisPageData.backfillTaskCounter = 0
|
||||
ThisPageData.backfillTaskKey = null
|
||||
ThisPageData.backfillTaskDescription = null
|
||||
ThisPageData.backfillTaskScript = null
|
||||
ThisPageData.backfillTaskForward = false
|
||||
ThisPageData.backfillTaskTargetDate = null
|
||||
ThisPageData.backfillTaskNotes = null
|
||||
|
||||
ThisPage.methods.backfillTaskCreate = function() {
|
||||
this.backfillTask = {key: null}
|
||||
this.backfillTaskDescription = null
|
||||
this.backfillTaskScript = null
|
||||
this.backfillTaskForward = false
|
||||
this.backfillTaskTargetDate = null
|
||||
this.backfillTaskNotes = null
|
||||
this.backfillTaskShowDialog = true
|
||||
this.$nextTick(() => {
|
||||
this.$refs.backfillTaskDescription.focus()
|
||||
})
|
||||
}
|
||||
|
||||
ThisPage.methods.backfillTaskEdit = function(task) {
|
||||
this.backfillTask = task
|
||||
this.backfillTaskDescription = task.description
|
||||
this.backfillTaskScript = task.script
|
||||
this.backfillTaskForward = task.forward
|
||||
this.backfillTaskTargetDate = task.target_date
|
||||
this.backfillTaskNotes = task.notes
|
||||
this.backfillTaskShowDialog = true
|
||||
}
|
||||
|
||||
ThisPage.methods.backfillTaskDelete = function(task) {
|
||||
if (confirm("Really delete this task?")) {
|
||||
let i = this.backfillTasks.indexOf(task)
|
||||
this.backfillTasks.splice(i, 1)
|
||||
this.settingsNeedSaved = true
|
||||
}
|
||||
}
|
||||
|
||||
ThisPage.methods.backfillTaskSave = function() {
|
||||
this.backfillTask.description = this.backfillTaskDescription
|
||||
this.backfillTask.script = this.backfillTaskScript
|
||||
this.backfillTask.forward = this.backfillTaskForward
|
||||
this.backfillTask.target_date = this.backfillTaskTargetDate
|
||||
this.backfillTask.notes = this.backfillTaskNotes
|
||||
|
||||
if (!this.backfillTask.key) {
|
||||
this.backfillTask.key = `_new_${'$'}{++this.backfillTaskCounter}`
|
||||
this.backfillTasks.push(this.backfillTask)
|
||||
}
|
||||
|
||||
this.backfillTaskShowDialog = false
|
||||
this.settingsNeedSaved = true
|
||||
}
|
||||
|
||||
</script>
|
||||
</%def>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
## -*- coding: utf-8; -*-
|
||||
<%inherit file="/page.mako" />
|
||||
|
||||
<%def name="title()">Luigi Jobs</%def>
|
||||
<%def name="title()">View / Launch Tasks</%def>
|
||||
|
||||
<%def name="page_content()">
|
||||
<br />
|
||||
|
|
@ -49,13 +49,141 @@
|
|||
% endif
|
||||
</div>
|
||||
|
||||
% if master.has_perm('launch'):
|
||||
% if master.has_perm('launch_overnight'):
|
||||
|
||||
<h3 class="block is-size-3">Overnight Tasks</h3>
|
||||
% for task in overnight_tasks:
|
||||
<launch-job job-name="${task['key']}"
|
||||
button-text="Restart Overnight ${task['key'].capitalize()}">
|
||||
</launch-job>
|
||||
% endfor
|
||||
|
||||
<b-table :data="overnightTasks" hoverable>
|
||||
<template slot-scope="props">
|
||||
<b-table-column field="description"
|
||||
label="Description">
|
||||
{{ props.row.description }}
|
||||
</b-table-column>
|
||||
<b-table-column field="script"
|
||||
label="Script">
|
||||
{{ props.row.script }}
|
||||
</b-table-column>
|
||||
<b-table-column field="last_date"
|
||||
label="Last Date"
|
||||
:class="overnightTextClass(props.row)">
|
||||
{{ props.row.last_date || "never!" }}
|
||||
</b-table-column>
|
||||
<b-table-column label="Actions">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="arrow-circle-right"
|
||||
:disabled="overnightTaskLaunching == props.row.key"
|
||||
@click="overnightTaskLaunch(props.row)">
|
||||
{{ overnightTaskLaunching == props.row.key ? "Working, please wait..." : "Launch" }}
|
||||
</b-button>
|
||||
</b-table-column>
|
||||
</template>
|
||||
<template #empty>
|
||||
<p class="block">No tasks defined.</p>
|
||||
</template>
|
||||
</b-table>
|
||||
|
||||
% endif
|
||||
|
||||
% if master.has_perm('launch_backfill'):
|
||||
|
||||
<h3 class="block is-size-3">Backfill Tasks</h3>
|
||||
|
||||
<b-table :data="backfillTasks" hoverable>
|
||||
<template slot-scope="props">
|
||||
<b-table-column field="description"
|
||||
label="Description">
|
||||
{{ props.row.description }}
|
||||
</b-table-column>
|
||||
<b-table-column field="script"
|
||||
label="Script">
|
||||
{{ props.row.script }}
|
||||
</b-table-column>
|
||||
<b-table-column field="forward"
|
||||
label="Orientation">
|
||||
{{ props.row.forward ? "Forward" : "Backward" }}
|
||||
</b-table-column>
|
||||
<b-table-column field="last_date"
|
||||
label="Last Date"
|
||||
:class="backfillTextClass(props.row)">
|
||||
{{ props.row.last_date }}
|
||||
</b-table-column>
|
||||
<b-table-column field="target_date"
|
||||
label="Target Date">
|
||||
{{ props.row.target_date }}
|
||||
</b-table-column>
|
||||
<b-table-column label="Actions">
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="arrow-circle-right"
|
||||
@click="backfillTaskLaunch(props.row)">
|
||||
Launch
|
||||
</b-button>
|
||||
</b-table-column>
|
||||
</template>
|
||||
<template #empty>
|
||||
<p class="block">No tasks defined.</p>
|
||||
</template>
|
||||
</b-table>
|
||||
|
||||
<b-modal has-modal-card
|
||||
:active.sync="backfillTaskShowLaunchDialog">
|
||||
<div class="modal-card">
|
||||
|
||||
<header class="modal-card-head">
|
||||
<p class="modal-card-title">Launch Backfill Task</p>
|
||||
</header>
|
||||
|
||||
<section class="modal-card-body"
|
||||
v-if="backfillTask">
|
||||
|
||||
<p class="block has-text-weight-bold">
|
||||
{{ backfillTask.description }}
|
||||
(goes {{ backfillTask.forward ? "FORWARD" : "BACKWARD" }})
|
||||
</p>
|
||||
|
||||
<b-field grouped>
|
||||
<b-field label="Last Date">
|
||||
{{ backfillTask.last_date || "n/a" }}
|
||||
</b-field>
|
||||
<b-field label="Target Date">
|
||||
{{ backfillTask.target_date || "n/a" }}
|
||||
</b-field>
|
||||
</b-field>
|
||||
|
||||
<b-field grouped>
|
||||
|
||||
<b-field label="Start Date"
|
||||
:type="backfillTaskStartDate ? null : 'is-danger'">
|
||||
<tailbone-datepicker v-model="backfillTaskStartDate">
|
||||
</tailbone-datepicker>
|
||||
</b-field>
|
||||
|
||||
<b-field label="End Date"
|
||||
:type="backfillTaskEndDate ? null : 'is-danger'">
|
||||
<tailbone-datepicker v-model="backfillTaskEndDate">
|
||||
</tailbone-datepicker>
|
||||
</b-field>
|
||||
|
||||
</b-field>
|
||||
|
||||
</section>
|
||||
|
||||
<footer class="modal-card-foot">
|
||||
<b-button @click="backfillTaskShowLaunchDialog = false">
|
||||
Cancel
|
||||
</b-button>
|
||||
<b-button type="is-primary"
|
||||
icon-pack="fas"
|
||||
icon-left="arrow-circle-right"
|
||||
@click="backfillTaskLaunchSubmit()"
|
||||
:disabled="backfillTaskLaunching || !backfillTaskStartDate || !backfillTaskEndDate">
|
||||
{{ backfillTaskLaunching ? "Working, please wait..." : "Launch" }}
|
||||
</b-button>
|
||||
</footer>
|
||||
</div>
|
||||
</b-modal>
|
||||
|
||||
% endif
|
||||
|
||||
</div>
|
||||
|
|
@ -63,63 +191,114 @@
|
|||
|
||||
<%def name="modify_this_page_vars()">
|
||||
${parent.modify_this_page_vars()}
|
||||
% if master.has_perm('restart_scheduler'):
|
||||
<script type="text/javascript">
|
||||
|
||||
% if master.has_perm('restart_scheduler'):
|
||||
|
||||
ThisPageData.restartSchedulerFormSubmitting = false
|
||||
|
||||
ThisPage.methods.submitRestartSchedulerForm = function() {
|
||||
this.restartSchedulerFormSubmitting = true
|
||||
}
|
||||
|
||||
</script>
|
||||
% endif
|
||||
</%def>
|
||||
|
||||
<%def name="finalize_this_page_vars()">
|
||||
${parent.finalize_this_page_vars()}
|
||||
% if master.has_perm('launch'):
|
||||
<script type="text/javascript">
|
||||
% if master.has_perm('launch_overnight'):
|
||||
|
||||
const LaunchJob = {
|
||||
template: '#launch-job-template',
|
||||
props: {
|
||||
jobName: String,
|
||||
buttonText: String,
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
formSubmitting: false,
|
||||
ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n}
|
||||
ThisPageData.overnightTaskLaunching = false
|
||||
|
||||
ThisPage.methods.overnightTextClass = function(task) {
|
||||
let yesterday = '${rattail_app.today() - datetime.timedelta(days=1)}'
|
||||
if (task.last_date) {
|
||||
if (task.last_date == yesterday) {
|
||||
return 'has-text-success'
|
||||
} else {
|
||||
return 'has-text-warning'
|
||||
}
|
||||
} else {
|
||||
return 'has-text-warning'
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
submitForm() {
|
||||
this.formSubmitting = true
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
Vue.component('launch-job', LaunchJob)
|
||||
ThisPage.methods.overnightTaskLaunch = function(task) {
|
||||
this.overnightTaskLaunching = task.key
|
||||
|
||||
let url = '${url('{}.launch_overnight'.format(route_prefix))}'
|
||||
let params = {key: task.key}
|
||||
|
||||
this.submitForm(url, params, response => {
|
||||
this.$buefy.toast.open({
|
||||
message: "Task has been scheduled for immediate launch!",
|
||||
type: 'is-success',
|
||||
duration: 5000, // 5 seconds
|
||||
})
|
||||
this.overnightTaskLaunching = false
|
||||
})
|
||||
}
|
||||
|
||||
% endif
|
||||
|
||||
% if master.has_perm('launch_backfill'):
|
||||
|
||||
ThisPageData.backfillTasks = ${json.dumps(backfill_tasks)|n}
|
||||
ThisPageData.backfillTask = null
|
||||
ThisPageData.backfillTaskStartDate = null
|
||||
ThisPageData.backfillTaskEndDate = null
|
||||
ThisPageData.backfillTaskShowLaunchDialog = false
|
||||
ThisPageData.backfillTaskLaunching = false
|
||||
|
||||
ThisPage.methods.backfillTextClass = function(task) {
|
||||
if (task.target_date) {
|
||||
if (task.last_date) {
|
||||
if (task.forward) {
|
||||
if (task.last_date >= task.target_date) {
|
||||
return 'has-text-success'
|
||||
} else {
|
||||
return 'has-text-warning'
|
||||
}
|
||||
} else {
|
||||
if (task.last_date <= task.target_date) {
|
||||
return 'has-text-success'
|
||||
} else {
|
||||
return 'has-text-warning'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ThisPage.methods.backfillTaskLaunch = function(task) {
|
||||
this.backfillTask = task
|
||||
this.backfillTaskStartDate = null
|
||||
this.backfillTaskEndDate = null
|
||||
this.backfillTaskShowLaunchDialog = true
|
||||
}
|
||||
|
||||
ThisPage.methods.backfillTaskLaunchSubmit = function() {
|
||||
this.backfillTaskLaunching = true
|
||||
|
||||
let url = '${url('{}.launch_backfill'.format(route_prefix))}'
|
||||
let params = {
|
||||
key: this.backfillTask.key,
|
||||
start_date: this.backfillTaskStartDate,
|
||||
end_date: this.backfillTaskEndDate,
|
||||
}
|
||||
|
||||
this.submitForm(url, params, response => {
|
||||
this.$buefy.toast.open({
|
||||
message: "Task has been scheduled for immediate launch!",
|
||||
type: 'is-success',
|
||||
duration: 5000, // 5 seconds
|
||||
})
|
||||
this.backfillTaskLaunching = false
|
||||
this.backfillTaskShowLaunchDialog = false
|
||||
})
|
||||
}
|
||||
|
||||
% endif
|
||||
|
||||
</script>
|
||||
% endif
|
||||
</%def>
|
||||
|
||||
<%def name="render_this_page_template()">
|
||||
${parent.render_this_page_template()}
|
||||
% if master.has_perm('launch'):
|
||||
<script type="text/x-template" id="launch-job-template">
|
||||
${h.form(url('{}.launch'.format(route_prefix)), method='post', **{'@submit': 'submitForm'})}
|
||||
${h.csrf_token(request)}
|
||||
<input type="hidden" name="job" v-model="jobName" />
|
||||
<b-button type="is-primary"
|
||||
native-type="submit"
|
||||
:disabled="formSubmitting">
|
||||
{{ formSubmitting ? "Working, please wait..." : buttonText }}
|
||||
</b-button>
|
||||
${h.end_form()}
|
||||
</script>
|
||||
% endif
|
||||
</%def>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,19 +27,29 @@ Views for Luigi
|
|||
from __future__ import unicode_literals, absolute_import
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
|
||||
import six
|
||||
import sqlalchemy as sa
|
||||
|
||||
from rattail.util import simple_error
|
||||
|
||||
from tailbone.views import MasterView
|
||||
|
||||
|
||||
class LuigiJobView(MasterView):
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LuigiTaskView(MasterView):
|
||||
"""
|
||||
Simple views for Luigi jobs.
|
||||
Simple views for Luigi tasks.
|
||||
"""
|
||||
normalized_model_name = 'luigijobs'
|
||||
model_key = 'jobname'
|
||||
model_title = "Luigi Job"
|
||||
normalized_model_name = 'luigitasks'
|
||||
model_key = 'key'
|
||||
model_title = "Luigi Task"
|
||||
route_prefix = 'luigi'
|
||||
url_prefix = '/luigi'
|
||||
|
||||
|
|
@ -50,27 +60,57 @@ class LuigiJobView(MasterView):
|
|||
configurable = True
|
||||
|
||||
def __init__(self, request, context=None):
|
||||
super(LuigiJobView, self).__init__(request, context=context)
|
||||
super(LuigiTaskView, self).__init__(request, context=context)
|
||||
app = self.get_rattail_app()
|
||||
self.luigi_handler = app.get_luigi_handler()
|
||||
|
||||
def index(self):
|
||||
luigi_url = self.rattail_config.get('luigi', 'url')
|
||||
luigi_url = self.rattail_config.get('rattail.luigi', 'url')
|
||||
history_url = '{}/history'.format(luigi_url.rstrip('/')) if luigi_url else None
|
||||
return self.render_to_response('index', {
|
||||
'use_buefy': self.get_use_buefy(),
|
||||
'index_url': None,
|
||||
'luigi_url': luigi_url,
|
||||
'luigi_history_url': history_url,
|
||||
'overnight_tasks': self.luigi_handler.get_all_overnight_tasks(),
|
||||
'overnight_tasks': self.get_overnight_tasks(),
|
||||
'backfill_tasks': self.get_backfill_tasks(),
|
||||
})
|
||||
|
||||
def launch(self):
|
||||
key = self.request.POST['job']
|
||||
assert key
|
||||
self.luigi_handler.restart_overnight_task(key)
|
||||
self.request.session.flash("Scheduled overnight task for immediate launch: {}".format(key))
|
||||
return self.redirect(self.get_index_url())
|
||||
def launch_overnight(self):
|
||||
app = self.get_rattail_app()
|
||||
data = self.request.json_body
|
||||
|
||||
key = data.get('key')
|
||||
task = self.luigi_handler.get_overnight_task(key) if key else None
|
||||
if not task:
|
||||
return self.json_response({'error': "Task not found"})
|
||||
|
||||
try:
|
||||
self.luigi_handler.launch_overnight_task(task, app.yesterday())
|
||||
except Exception as error:
|
||||
log.warning("failed to launch overnight task: %s", task,
|
||||
exc_info=True)
|
||||
return self.json_response({'error': simple_error(error)})
|
||||
return self.json_response({'ok': True})
|
||||
|
||||
def launch_backfill(self):
|
||||
app = self.get_rattail_app()
|
||||
data = self.request.json_body
|
||||
|
||||
key = data.get('key')
|
||||
task = self.luigi_handler.get_backfill_task(key) if key else None
|
||||
if not task:
|
||||
return self.json_response({'error': "Task not found"})
|
||||
|
||||
start_date = app.parse_date(data['start_date'])
|
||||
end_date = app.parse_date(data['end_date'])
|
||||
try:
|
||||
self.luigi_handler.launch_backfill_task(task, start_date, end_date)
|
||||
except Exception as error:
|
||||
log.warning("failed to launch backfill task: %s", task,
|
||||
exc_info=True)
|
||||
return self.json_response({'error': simple_error(error)})
|
||||
return self.json_response({'ok': True})
|
||||
|
||||
def restart_scheduler(self):
|
||||
try:
|
||||
|
|
@ -87,36 +127,120 @@ class LuigiJobView(MasterView):
|
|||
return [
|
||||
|
||||
# luigi proper
|
||||
{'section': 'luigi',
|
||||
{'section': 'rattail.luigi',
|
||||
'option': 'url'},
|
||||
{'section': 'luigi',
|
||||
{'section': 'rattail.luigi',
|
||||
'option': 'scheduler.supervisor_process_name'},
|
||||
{'section': 'luigi',
|
||||
{'section': 'rattail.luigi',
|
||||
'option': 'scheduler.restart_command'},
|
||||
|
||||
]
|
||||
|
||||
def configure_get_context(self, **kwargs):
|
||||
context = super(LuigiJobView, self).configure_get_context(**kwargs)
|
||||
context['overnight_tasks'] = self.luigi_handler.get_all_overnight_tasks()
|
||||
context = super(LuigiTaskView, self).configure_get_context(**kwargs)
|
||||
context['overnight_tasks'] = self.get_overnight_tasks()
|
||||
context['backfill_tasks'] = self.get_backfill_tasks()
|
||||
return context
|
||||
|
||||
def configure_gather_settings(self, data):
|
||||
settings = super(LuigiJobView, self).configure_gather_settings(data)
|
||||
def get_overnight_tasks(self):
|
||||
tasks = self.luigi_handler.get_all_overnight_tasks()
|
||||
for task in tasks:
|
||||
if task['last_date']:
|
||||
task['last_date'] = six.text_type(task['last_date'])
|
||||
return tasks
|
||||
|
||||
def get_backfill_tasks(self):
|
||||
tasks = self.luigi_handler.get_all_backfill_tasks()
|
||||
for task in tasks:
|
||||
if task['last_date']:
|
||||
task['last_date'] = six.text_type(task['last_date'])
|
||||
if task['target_date']:
|
||||
task['target_date'] = six.text_type(task['target_date'])
|
||||
return tasks
|
||||
|
||||
def configure_gather_settings(self, data):
|
||||
settings = super(LuigiTaskView, self).configure_gather_settings(data)
|
||||
app = self.get_rattail_app()
|
||||
|
||||
# overnight tasks
|
||||
keys = []
|
||||
for task in json.loads(data['overnight_tasks']):
|
||||
keys.append(task['key'])
|
||||
|
||||
key = task['key']
|
||||
if key.startswith('_new_'):
|
||||
key = app.make_uuid()
|
||||
|
||||
key = task['key']
|
||||
if key.startswith('_new_'):
|
||||
cmd = shlex.split(task['script'])
|
||||
script = os.path.basename(cmd[0])
|
||||
root, ext = os.path.splitext(script)
|
||||
key = re.sub(r'\s+', '-', root)
|
||||
|
||||
keys.append(key)
|
||||
settings.extend([
|
||||
{'name': 'rattail.luigi.overnight.{}.description'.format(key),
|
||||
'value': task['description']},
|
||||
{'name': 'rattail.luigi.overnight.{}.script'.format(key),
|
||||
'value': task['script']},
|
||||
{'name': 'rattail.luigi.overnight.{}.notes'.format(key),
|
||||
'value': task['notes']},
|
||||
])
|
||||
if keys:
|
||||
settings.append({'name': 'luigi.overnight_tasks',
|
||||
settings.append({'name': 'rattail.luigi.overnight_tasks',
|
||||
'value': ', '.join(keys)})
|
||||
|
||||
# backfill tasks
|
||||
keys = []
|
||||
for task in json.loads(data['backfill_tasks']):
|
||||
|
||||
key = task['key']
|
||||
if key.startswith('_new_'):
|
||||
script = os.path.basename(task['script'])
|
||||
root, ext = os.path.splitext(script)
|
||||
key = re.sub(r'\s+', '-', root)
|
||||
|
||||
keys.append(key)
|
||||
settings.extend([
|
||||
{'name': 'rattail.luigi.backfill.{}.description'.format(key),
|
||||
'value': task['description']},
|
||||
{'name': 'rattail.luigi.backfill.{}.script'.format(key),
|
||||
'value': task['script']},
|
||||
{'name': 'rattail.luigi.backfill.{}.forward'.format(key),
|
||||
'value': 'true' if task['forward'] else 'false'},
|
||||
{'name': 'rattail.luigi.backfill.{}.notes'.format(key),
|
||||
'value': task['notes']},
|
||||
{'name': 'rattail.luigi.backfill.{}.target_date'.format(key),
|
||||
'value': six.text_type(task['target_date'])},
|
||||
])
|
||||
if keys:
|
||||
settings.append({'name': 'rattail.luigi.backfill_tasks',
|
||||
'value': ', '.join(keys)})
|
||||
|
||||
return settings
|
||||
|
||||
def configure_remove_settings(self):
|
||||
super(LuigiJobView, self).configure_remove_settings()
|
||||
self.luigi_handler.purge_luigi_settings(self.Session())
|
||||
super(LuigiTaskView, self).configure_remove_settings()
|
||||
app = self.get_rattail_app()
|
||||
model = self.model
|
||||
session = self.Session()
|
||||
|
||||
to_delete = session.query(model.Setting)\
|
||||
.filter(sa.or_(
|
||||
model.Setting.name == 'rattail.luigi.backfill_tasks',
|
||||
model.Setting.name.like('rattail.luigi.backfill.%.description'),
|
||||
model.Setting.name.like('rattail.luigi.backfill.%.forward'),
|
||||
model.Setting.name.like('rattail.luigi.backfill.%.notes'),
|
||||
model.Setting.name.like('rattail.luigi.backfill.%.script'),
|
||||
model.Setting.name.like('rattail.luigi.backfill.%.target_date'),
|
||||
model.Setting.name == 'rattail.luigi.overnight_tasks',
|
||||
model.Setting.name.like('rattail.luigi.overnight.%.description'),
|
||||
model.Setting.name.like('rattail.luigi.overnight.%.notes'),
|
||||
model.Setting.name.like('rattail.luigi.overnight.%.script')))\
|
||||
.all()
|
||||
|
||||
for setting in to_delete:
|
||||
app.delete_setting(session, setting.name)
|
||||
|
||||
@classmethod
|
||||
def defaults(cls, config):
|
||||
|
|
@ -130,16 +254,27 @@ class LuigiJobView(MasterView):
|
|||
url_prefix = cls.get_url_prefix()
|
||||
model_title_plural = cls.get_model_title_plural()
|
||||
|
||||
# launch job
|
||||
# launch overnight
|
||||
config.add_tailbone_permission(permission_prefix,
|
||||
'{}.launch'.format(permission_prefix),
|
||||
label="Launch any Luigi job")
|
||||
config.add_route('{}.launch'.format(route_prefix),
|
||||
'{}/launch'.format(url_prefix),
|
||||
'{}.launch_overnight'.format(permission_prefix),
|
||||
label="Launch any Overnight Task")
|
||||
config.add_route('{}.launch_overnight'.format(route_prefix),
|
||||
'{}/launch-overnight'.format(url_prefix),
|
||||
request_method='POST')
|
||||
config.add_view(cls, attr='launch',
|
||||
route_name='{}.launch'.format(route_prefix),
|
||||
permission='{}.launch'.format(permission_prefix))
|
||||
config.add_view(cls, attr='launch_overnight',
|
||||
route_name='{}.launch_overnight'.format(route_prefix),
|
||||
permission='{}.launch_overnight'.format(permission_prefix))
|
||||
|
||||
# launch backfill
|
||||
config.add_tailbone_permission(permission_prefix,
|
||||
'{}.launch_backfill'.format(permission_prefix),
|
||||
label="Launch any Backfill Task")
|
||||
config.add_route('{}.launch_backfill'.format(route_prefix),
|
||||
'{}/launch-backfill'.format(url_prefix),
|
||||
request_method='POST')
|
||||
config.add_view(cls, attr='launch_backfill',
|
||||
route_name='{}.launch_backfill'.format(route_prefix),
|
||||
permission='{}.launch_backfill'.format(permission_prefix))
|
||||
|
||||
# restart luigid scheduler
|
||||
config.add_tailbone_permission(permission_prefix,
|
||||
|
|
@ -156,8 +291,8 @@ class LuigiJobView(MasterView):
|
|||
def defaults(config, **kwargs):
|
||||
base = globals()
|
||||
|
||||
LuigiJobView = kwargs.get('LuigiJobView', base['LuigiJobView'])
|
||||
LuigiJobView.defaults(config)
|
||||
LuigiTaskView = kwargs.get('LuigiTaskView', base['LuigiTaskView'])
|
||||
LuigiTaskView.defaults(config)
|
||||
|
||||
|
||||
def includeme(config):
|
||||
|
|
|
|||
Loading…
Reference in a new issue