Add basic support for backfill Luigi tasks

idea being, sometimes you must import many days worth of data into
Trainwreck or what-not, and it must be split up b/c e.g. it would take
too long to import all at once (i.e. might interfere with overnight
tasks)

tailbone/templates/luigi/configure.mako

@@ -3,27 +3,56 @@
 
 <%def name="form_content()">
   ${h.hidden('overnight_tasks', **{':value': 'JSON.stringify(overnightTasks)'})}
+  ${h.hidden('backfill_tasks', **{':value': 'JSON.stringify(backfillTasks)'})}
 
+  <div class="level">
+    <div class="level-left">
+      <div class="level-item">
         <h3 class="is-size-3">Overnight Tasks</h3>
-  <div class="block" style="padding-left: 2rem; display: flex;">
+      </div>
-
+      <div class="level-item">
-    <b-table :data="overnightTasks">
-      <template slot-scope="props">
-        <b-table-column field="key"
-                        label="Key"
-                        sortable>
-          {{ props.row.key }}
-        </b-table-column>
-      </template>
-    </b-table>
-
-    <div style="margin-left: 1rem;">
         <b-button type="is-primary"
                   icon-pack="fas"
                   icon-left="plus"
                   @click="overnightTaskCreate()">
           New Task
         </b-button>
+      </div>
+    </div>
+  </div>
+  <div class="block" style="padding-left: 2rem; display: flex;">
+
+    <b-table :data="overnightTasks">
+      <template slot-scope="props">
+        <!-- <b-table-column field="key" -->
+        <!--                 label="Key" -->
+        <!--                 sortable> -->
+        <!--   {{ props.row.key }} -->
+        <!-- </b-table-column> -->
+        <b-table-column field="description"
+                        label="Description">
+          {{ props.row.description }}
+        </b-table-column>
+        <b-table-column field="script"
+                        label="Script">
+          {{ props.row.script }}
+        </b-table-column>
+        <b-table-column label="Actions">
+          <a href="#"
+             @click.prevent="overnightTaskEdit(props.row)">
+            <i class="fas fa-edit"></i>
+            Edit
+          </a>
+          &nbsp;
+          <a href="#"
+             class="has-text-danger"
+             @click.prevent="overnightTaskDelete(props.row)">
+            <i class="fas fa-trash"></i>
+            Delete
+          </a>
+        </b-table-column>
+      </template>
+    </b-table>
 
     <b-modal has-modal-card
              :active.sync="overnightTaskShowDialog">
@@ -34,9 +63,25 @@
         </header>
 
         <section class="modal-card-body">
-            <b-field label="Key">
+          <!-- <b-field label="Key"> -->
-              <b-input v-model.trim="overnightTaskKey"
+          <!--   <b-input v-model.trim="overnightTaskKey" -->
-                       ref="overnightTaskKey">
+          <!--            ref="overnightTaskKey"> -->
+          <!--   </b-input> -->
+          <!-- </b-field> -->
+          <b-field label="Description"
+                   :type="overnightTaskDescription ? null : 'is-danger'">
+            <b-input v-model.trim="overnightTaskDescription"
+                     ref="overnightTaskDescription">
+            </b-input>
+          </b-field>
+          <b-field label="Script"
+                   :type="overnightTaskScript ? null : 'is-danger'">
+            <b-input v-model.trim="overnightTaskScript">
+            </b-input>
+          </b-field>
+          <b-field label="Notes">
+            <b-input v-model.trim="overnightTaskNotes"
+                     type="textarea">
             </b-input>
           </b-field>
         </section>
@@ -46,7 +91,7 @@
                     icon-pack="fas"
                     icon-left="save"
                     @click="overnightTaskSave()"
-                      :disabled="!overnightTaskKey">
+                    :disabled="!overnightTaskDescription || !overnightTaskScript">
             Save
           </b-button>
           <b-button @click="overnightTaskShowDialog = false">
@@ -57,6 +102,113 @@
     </b-modal>
 
   </div>
+
+  <div class="level">
+    <div class="level-left">
+      <div class="level-item">
+        <h3 class="is-size-3">Backfill Tasks</h3>
+      </div>
+      <div class="level-item">
+        <b-button type="is-primary"
+                  icon-pack="fas"
+                  icon-left="plus"
+                  @click="backfillTaskCreate()">
+          New Task
+        </b-button>
+      </div>
+    </div>
+  </div>
+  <div class="block" style="padding-left: 2rem; display: flex;">
+
+    <b-table :data="backfillTasks">
+      <template slot-scope="props">
+        <b-table-column field="description"
+                        label="Description">
+          {{ props.row.description }}
+        </b-table-column>
+        <b-table-column field="script"
+                        label="Script">
+          {{ props.row.script }}
+        </b-table-column>
+        <b-table-column field="forward"
+                        label="Orientation">
+          {{ props.row.forward ? "Forward" : "Backward" }}
+        </b-table-column>
+        <b-table-column field="target_date"
+                        label="Target Date">
+          {{ props.row.target_date }}
+        </b-table-column>
+        <b-table-column label="Actions">
+          <a href="#"
+             @click.prevent="backfillTaskEdit(props.row)">
+            <i class="fas fa-edit"></i>
+            Edit
+          </a>
+          &nbsp;
+          <a href="#"
+             class="has-text-danger"
+             @click.prevent="backfillTaskDelete(props.row)">
+            <i class="fas fa-trash"></i>
+            Delete
+          </a>
+        </b-table-column>
+      </template>
+    </b-table>
+
+    <b-modal has-modal-card
+             :active.sync="backfillTaskShowDialog">
+      <div class="modal-card">
+
+        <header class="modal-card-head">
+          <p class="modal-card-title">Backfill Task</p>
+        </header>
+
+        <section class="modal-card-body">
+          <b-field label="Description"
+                   :type="backfillTaskDescription ? null : 'is-danger'">
+            <b-input v-model.trim="backfillTaskDescription"
+                     ref="backfillTaskDescription">
+            </b-input>
+          </b-field>
+          <b-field label="Script"
+                   :type="backfillTaskScript ? null : 'is-danger'">
+            <b-input v-model.trim="backfillTaskScript">
+            </b-input>
+          </b-field>
+          <b-field grouped>
+            <b-field label="Orientation">
+              <b-select v-model="backfillTaskForward">
+                <option :value="false">Backward</option>
+                <option :value="true">Forward</option>
+              </b-select>
+            </b-field>
+            <b-field label="Target Date">
+              <tailbone-datepicker v-model="backfillTaskTargetDate">
+              </tailbone-datepicker>
+            </b-field>
+          </b-field>
+          <b-field label="Notes">
+            <b-input v-model.trim="backfillTaskNotes"
+                     type="textarea">
+            </b-input>
+          </b-field>
+        </section>
+
+        <footer class="modal-card-foot">
+          <b-button type="is-primary"
+                    icon-pack="fas"
+                    icon-left="save"
+                    @click="backfillTaskSave()"
+                    :disabled="!backfillTaskDescription || !backfillTaskScript">
+            Save
+          </b-button>
+          <b-button @click="backfillTaskShowDialog = false">
+            Cancel
+          </b-button>
+        </footer>
+      </div>
+    </b-modal>
+
   </div>
 
   <h3 class="is-size-3">Luigi Proper</h3>
@@ -65,8 +217,8 @@
     <b-field label="Luigi URL"
              message="This should be the URL to Luigi Task Visualiser web user interface."
              expanded>
-      <b-input name="luigi.url"
+      <b-input name="rattail.luigi.url"
-               v-model="simpleSettings['luigi.url']"
+               v-model="simpleSettings['rattail.luigi.url']"
                @input="settingsNeedSaved = true">
       </b-input>
     </b-field>
@@ -74,8 +226,8 @@
     <b-field label="Supervisor Process Name"
              message="This should be the complete name, including group - e.g. luigi:luigid"
              expanded>
-      <b-input name="luigi.scheduler.supervisor_process_name"
+      <b-input name="rattail.luigi.scheduler.supervisor_process_name"
-               v-model="simpleSettings['luigi.scheduler.supervisor_process_name']"
+               v-model="simpleSettings['rattail.luigi.scheduler.supervisor_process_name']"
                @input="settingsNeedSaved = true">
       </b-input>
     </b-field>
@@ -83,8 +235,8 @@
     <b-field label="Restart Command"
              message="This will run as '${system_user}' system user - please configure sudoers as needed.  Typical command is like:  sudo supervisorctl restart luigi:luigid"
              expanded>
-      <b-input name="luigi.scheduler.restart_command"
+      <b-input name="rattail.luigi.scheduler.restart_command"
-               v-model="simpleSettings['luigi.scheduler.restart_command']"
+               v-model="simpleSettings['rattail.luigi.scheduler.restart_command']"
                @input="settingsNeedSaved = true">
       </b-input>
     </b-field>
@@ -100,28 +252,113 @@
     ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n}
     ThisPageData.overnightTaskShowDialog = false
     ThisPageData.overnightTask = null
+    ThisPageData.overnightTaskCounter = 0
     ThisPageData.overnightTaskKey = null
+    ThisPageData.overnightTaskDescription = null
+    ThisPageData.overnightTaskScript = null
+    ThisPageData.overnightTaskNotes = null
 
     ThisPage.methods.overnightTaskCreate = function() {
-        this.overnightTask = null
+        this.overnightTask = {key: null}
         this.overnightTaskKey = null
+        this.overnightTaskDescription = null
+        this.overnightTaskScript = null
+        this.overnightTaskNotes = null
         this.overnightTaskShowDialog = true
         this.$nextTick(() => {
-            this.$refs.overnightTaskKey.focus()
+            this.$refs.overnightTaskDescription.focus()
         })
     }
 
-    ThisPage.methods.overnightTaskSave = function() {
+    ThisPage.methods.overnightTaskEdit = function(task) {
-        if (this.overnightTask) {
+        this.overnightTask = task
-            this.overnightTask.key = this.overnightTaskKey
+        this.overnightTaskKey = task.key
-        } else {
+        this.overnightTaskDescription = task.description
-            let task = {key: this.overnightTaskKey}
+        this.overnightTaskScript = task.script
-            this.overnightTasks.push(task)
+        this.overnightTaskNotes = task.notes
+        this.overnightTaskShowDialog = true
     }
+
+    ThisPage.methods.overnightTaskSave = function() {
+        this.overnightTask.description = this.overnightTaskDescription
+        this.overnightTask.script = this.overnightTaskScript
+        this.overnightTask.notes = this.overnightTaskNotes
+
+        if (!this.overnightTask.key) {
+            this.overnightTask.key = `_new_${'$'}{++this.overnightTaskCounter}`
+            this.overnightTasks.push(this.overnightTask)
+        }
+
         this.overnightTaskShowDialog = false
         this.settingsNeedSaved = true
     }
 
+    ThisPage.methods.overnightTaskDelete = function(task) {
+        if (confirm("Really delete this task?")) {
+            let i = this.overnightTasks.indexOf(task)
+            this.overnightTasks.splice(i, 1)
+            this.settingsNeedSaved = true
+        }
+    }
+
+    ThisPageData.backfillTasks = ${json.dumps(backfill_tasks)|n}
+    ThisPageData.backfillTaskShowDialog = false
+    ThisPageData.backfillTask = null
+    ThisPageData.backfillTaskCounter = 0
+    ThisPageData.backfillTaskKey = null
+    ThisPageData.backfillTaskDescription = null
+    ThisPageData.backfillTaskScript = null
+    ThisPageData.backfillTaskForward = false
+    ThisPageData.backfillTaskTargetDate = null
+    ThisPageData.backfillTaskNotes = null
+
+    ThisPage.methods.backfillTaskCreate = function() {
+        this.backfillTask = {key: null}
+        this.backfillTaskDescription = null
+        this.backfillTaskScript = null
+        this.backfillTaskForward = false
+        this.backfillTaskTargetDate = null
+        this.backfillTaskNotes = null
+        this.backfillTaskShowDialog = true
+        this.$nextTick(() => {
+            this.$refs.backfillTaskDescription.focus()
+        })
+    }
+
+    ThisPage.methods.backfillTaskEdit = function(task) {
+        this.backfillTask = task
+        this.backfillTaskDescription = task.description
+        this.backfillTaskScript = task.script
+        this.backfillTaskForward = task.forward
+        this.backfillTaskTargetDate = task.target_date
+        this.backfillTaskNotes = task.notes
+        this.backfillTaskShowDialog = true
+    }
+
+    ThisPage.methods.backfillTaskDelete = function(task) {
+        if (confirm("Really delete this task?")) {
+            let i = this.backfillTasks.indexOf(task)
+            this.backfillTasks.splice(i, 1)
+            this.settingsNeedSaved = true
+        }
+    }
+
+    ThisPage.methods.backfillTaskSave = function() {
+        this.backfillTask.description = this.backfillTaskDescription
+        this.backfillTask.script = this.backfillTaskScript
+        this.backfillTask.forward = this.backfillTaskForward
+        this.backfillTask.target_date = this.backfillTaskTargetDate
+        this.backfillTask.notes = this.backfillTaskNotes
+
+        if (!this.backfillTask.key) {
+            this.backfillTask.key = `_new_${'$'}{++this.backfillTaskCounter}`
+            this.backfillTasks.push(this.backfillTask)
+        }
+
+        this.backfillTaskShowDialog = false
+        this.settingsNeedSaved = true
+    }
+
   </script>
 </%def>
 

tailbone/templates/luigi/index.mako

@@ -1,7 +1,7 @@
 ## -*- coding: utf-8; -*-
 <%inherit file="/page.mako" />
 
-<%def name="title()">Luigi Jobs</%def>
+<%def name="title()">View / Launch Tasks</%def>
 
 <%def name="page_content()">
   <br />
@@ -49,13 +49,141 @@
       % endif
     </div>
 
-    % if master.has_perm('launch'):
+    % if  master.has_perm('launch_overnight'):
+
         <h3 class="block is-size-3">Overnight Tasks</h3>
-        % for task in overnight_tasks:
+
-            <launch-job job-name="${task['key']}"
+        <b-table :data="overnightTasks" hoverable>
-                        button-text="Restart Overnight ${task['key'].capitalize()}">
+          <template slot-scope="props">
-            </launch-job>
+            <b-table-column field="description"
-        % endfor
+                            label="Description">
+              {{ props.row.description }}
+            </b-table-column>
+            <b-table-column field="script"
+                            label="Script">
+              {{ props.row.script }}
+            </b-table-column>
+            <b-table-column field="last_date"
+                            label="Last Date"
+                            :class="overnightTextClass(props.row)">
+              {{ props.row.last_date || "never!" }}
+            </b-table-column>
+            <b-table-column label="Actions">
+              <b-button type="is-primary"
+                        icon-pack="fas"
+                        icon-left="arrow-circle-right"
+                        :disabled="overnightTaskLaunching == props.row.key"
+                        @click="overnightTaskLaunch(props.row)">
+                {{ overnightTaskLaunching == props.row.key ? "Working, please wait..." : "Launch" }}
+              </b-button>
+            </b-table-column>
+          </template>
+          <template #empty>
+            <p class="block">No tasks defined.</p>
+          </template>
+        </b-table>
+
+    % endif
+
+    % if master.has_perm('launch_backfill'):
+
+        <h3 class="block is-size-3">Backfill Tasks</h3>
+
+        <b-table :data="backfillTasks" hoverable>
+          <template slot-scope="props">
+            <b-table-column field="description"
+                            label="Description">
+              {{ props.row.description }}
+            </b-table-column>
+            <b-table-column field="script"
+                            label="Script">
+              {{ props.row.script }}
+            </b-table-column>
+            <b-table-column field="forward"
+                            label="Orientation">
+              {{ props.row.forward ? "Forward" : "Backward" }}
+            </b-table-column>
+            <b-table-column field="last_date"
+                            label="Last Date"
+                            :class="backfillTextClass(props.row)">
+              {{ props.row.last_date }}
+            </b-table-column>
+            <b-table-column field="target_date"
+                            label="Target Date">
+              {{ props.row.target_date }}
+            </b-table-column>
+            <b-table-column label="Actions">
+              <b-button type="is-primary"
+                        icon-pack="fas"
+                        icon-left="arrow-circle-right"
+                        @click="backfillTaskLaunch(props.row)">
+                Launch
+              </b-button>
+            </b-table-column>
+          </template>
+          <template #empty>
+            <p class="block">No tasks defined.</p>
+          </template>
+        </b-table>
+
+        <b-modal has-modal-card
+                 :active.sync="backfillTaskShowLaunchDialog">
+          <div class="modal-card">
+
+            <header class="modal-card-head">
+              <p class="modal-card-title">Launch Backfill Task</p>
+            </header>
+
+            <section class="modal-card-body"
+                     v-if="backfillTask">
+
+              <p class="block has-text-weight-bold">
+                {{ backfillTask.description }}
+                (goes {{ backfillTask.forward ? "FORWARD" : "BACKWARD" }})
+              </p>
+
+              <b-field grouped>
+                <b-field label="Last Date">
+                  {{ backfillTask.last_date || "n/a" }}
+                </b-field>
+                <b-field label="Target Date">
+                  {{ backfillTask.target_date || "n/a" }}
+                </b-field>
+              </b-field>
+
+              <b-field grouped>
+
+                <b-field label="Start Date"
+                         :type="backfillTaskStartDate ? null : 'is-danger'">
+                  <tailbone-datepicker v-model="backfillTaskStartDate">
+                  </tailbone-datepicker>
+                </b-field>
+
+                <b-field label="End Date"
+                         :type="backfillTaskEndDate ? null : 'is-danger'">
+                  <tailbone-datepicker v-model="backfillTaskEndDate">
+                  </tailbone-datepicker>
+                </b-field>
+
+              </b-field>
+
+            </section>
+
+            <footer class="modal-card-foot">
+              <b-button @click="backfillTaskShowLaunchDialog = false">
+                Cancel
+              </b-button>
+              <b-button type="is-primary"
+                        icon-pack="fas"
+                        icon-left="arrow-circle-right"
+                        @click="backfillTaskLaunchSubmit()"
+                        :disabled="backfillTaskLaunching || !backfillTaskStartDate || !backfillTaskEndDate">
+                {{ backfillTaskLaunching ? "Working, please wait..." : "Launch" }}
+              </b-button>
+            </footer>
+          </div>
+        </b-modal>
+
     % endif
 
   </div>
@@ -63,63 +191,114 @@
 
 <%def name="modify_this_page_vars()">
   ${parent.modify_this_page_vars()}
-  % if master.has_perm('restart_scheduler'):
   <script type="text/javascript">
 
+    % if master.has_perm('restart_scheduler'):
+
         ThisPageData.restartSchedulerFormSubmitting = false
 
         ThisPage.methods.submitRestartSchedulerForm = function() {
             this.restartSchedulerFormSubmitting = true
         }
 
-      </script>
     % endif
-</%def>
 
-<%def name="finalize_this_page_vars()">
+    % if master.has_perm('launch_overnight'):
-  ${parent.finalize_this_page_vars()}
-  % if master.has_perm('launch'):
-      <script type="text/javascript">
 
-        const LaunchJob = {
+        ThisPageData.overnightTasks = ${json.dumps(overnight_tasks)|n}
-            template: '#launch-job-template',
+        ThisPageData.overnightTaskLaunching = false
-            props: {
+
-                jobName: String,
+        ThisPage.methods.overnightTextClass = function(task) {
-                buttonText: String,
+            let yesterday = '${rattail_app.today() - datetime.timedelta(days=1)}'
-            },
+            if (task.last_date) {
-            data() {
+                if (task.last_date == yesterday) {
-                return {
+                    return 'has-text-success'
-                    formSubmitting: false,
+                } else {
+                    return 'has-text-warning'
+                }
+            } else {
+                return 'has-text-warning'
             }
-            },
-            methods: {
-                submitForm() {
-                    this.formSubmitting = true
-                },
-            },
         }
 
-        Vue.component('launch-job', LaunchJob)
+        ThisPage.methods.overnightTaskLaunch = function(task) {
+            this.overnightTaskLaunching = task.key
+
+            let url = '${url('{}.launch_overnight'.format(route_prefix))}'
+            let params = {key: task.key}
+
+            this.submitForm(url, params, response => {
+                this.$buefy.toast.open({
+                    message: "Task has been scheduled for immediate launch!",
+                    type: 'is-success',
+                    duration: 5000, // 5 seconds
+                })
+                this.overnightTaskLaunching = false
+            })
+        }
+
+    % endif
+
+    % if master.has_perm('launch_backfill'):
+
+        ThisPageData.backfillTasks = ${json.dumps(backfill_tasks)|n}
+        ThisPageData.backfillTask = null
+        ThisPageData.backfillTaskStartDate = null
+        ThisPageData.backfillTaskEndDate = null
+        ThisPageData.backfillTaskShowLaunchDialog = false
+        ThisPageData.backfillTaskLaunching = false
+
+        ThisPage.methods.backfillTextClass = function(task) {
+            if (task.target_date) {
+                if (task.last_date) {
+                    if (task.forward) {
+                        if (task.last_date >= task.target_date) {
+                            return 'has-text-success'
+                        } else {
+                            return 'has-text-warning'
+                        }
+                    } else {
+                        if (task.last_date <= task.target_date) {
+                            return 'has-text-success'
+                        } else {
+                            return 'has-text-warning'
+                        }
+                    }
+                }
+            }
+        }
+
+        ThisPage.methods.backfillTaskLaunch = function(task) {
+            this.backfillTask = task
+            this.backfillTaskStartDate = null
+            this.backfillTaskEndDate = null
+            this.backfillTaskShowLaunchDialog = true
+        }
+
+        ThisPage.methods.backfillTaskLaunchSubmit = function() {
+            this.backfillTaskLaunching = true
+
+            let url = '${url('{}.launch_backfill'.format(route_prefix))}'
+            let params = {
+                key: this.backfillTask.key,
+                start_date: this.backfillTaskStartDate,
+                end_date: this.backfillTaskEndDate,
+            }
+
+            this.submitForm(url, params, response => {
+                this.$buefy.toast.open({
+                    message: "Task has been scheduled for immediate launch!",
+                    type: 'is-success',
+                    duration: 5000, // 5 seconds
+                })
+                this.backfillTaskLaunching = false
+                this.backfillTaskShowLaunchDialog = false
+            })
+        }
+
+    % endif
 
   </script>
-  % endif
-</%def>
-
-<%def name="render_this_page_template()">
-  ${parent.render_this_page_template()}
-  % if master.has_perm('launch'):
-      <script type="text/x-template" id="launch-job-template">
-        ${h.form(url('{}.launch'.format(route_prefix)), method='post', **{'@submit': 'submitForm'})}
-        ${h.csrf_token(request)}
-        <input type="hidden" name="job" v-model="jobName" />
-        <b-button type="is-primary"
-                  native-type="submit"
-                  :disabled="formSubmitting">
-          {{ formSubmitting ? "Working, please wait..." : buttonText }}
-        </b-button>
-        ${h.end_form()}
-      </script>
-  % endif
 </%def>
 
 

tailbone/views/luigi.py

@@ -27,19 +27,29 @@ Views for Luigi
 from __future__ import unicode_literals, absolute_import
 
 import json
+import logging
+import os
+import re
+import shlex
+
+import six
+import sqlalchemy as sa
 
 from rattail.util import simple_error
 
 from tailbone.views import MasterView
 
 
-class LuigiJobView(MasterView):
+log = logging.getLogger(__name__)
+
+
+class LuigiTaskView(MasterView):
     """
-    Simple views for Luigi jobs.
+    Simple views for Luigi tasks.
     """
-    normalized_model_name = 'luigijobs'
+    normalized_model_name = 'luigitasks'
-    model_key = 'jobname'
+    model_key = 'key'
-    model_title = "Luigi Job"
+    model_title = "Luigi Task"
     route_prefix = 'luigi'
     url_prefix = '/luigi'
 
@@ -50,27 +60,57 @@ class LuigiJobView(MasterView):
     configurable = True
 
     def __init__(self, request, context=None):
-        super(LuigiJobView, self).__init__(request, context=context)
+        super(LuigiTaskView, self).__init__(request, context=context)
         app = self.get_rattail_app()
         self.luigi_handler = app.get_luigi_handler()
 
     def index(self):
-        luigi_url = self.rattail_config.get('luigi', 'url')
+        luigi_url = self.rattail_config.get('rattail.luigi', 'url')
         history_url = '{}/history'.format(luigi_url.rstrip('/')) if luigi_url else None
         return self.render_to_response('index', {
             'use_buefy': self.get_use_buefy(),
             'index_url': None,
             'luigi_url': luigi_url,
             'luigi_history_url': history_url,
-            'overnight_tasks': self.luigi_handler.get_all_overnight_tasks(),
+            'overnight_tasks': self.get_overnight_tasks(),
+            'backfill_tasks': self.get_backfill_tasks(),
         })
 
-    def launch(self):
+    def launch_overnight(self):
-        key = self.request.POST['job']
+        app = self.get_rattail_app()
-        assert key
+        data = self.request.json_body
-        self.luigi_handler.restart_overnight_task(key)
+
-        self.request.session.flash("Scheduled overnight task for immediate launch: {}".format(key))
+        key = data.get('key')
-        return self.redirect(self.get_index_url())
+        task = self.luigi_handler.get_overnight_task(key) if key else None
+        if not task:
+            return self.json_response({'error': "Task not found"})
+
+        try:
+            self.luigi_handler.launch_overnight_task(task, app.yesterday())
+        except Exception as error:
+            log.warning("failed to launch overnight task: %s", task,
+                        exc_info=True)
+            return self.json_response({'error': simple_error(error)})
+        return self.json_response({'ok': True})
+
+    def launch_backfill(self):
+        app = self.get_rattail_app()
+        data = self.request.json_body
+
+        key = data.get('key')
+        task = self.luigi_handler.get_backfill_task(key) if key else None
+        if not task:
+            return self.json_response({'error': "Task not found"})
+
+        start_date = app.parse_date(data['start_date'])
+        end_date = app.parse_date(data['end_date'])
+        try:
+            self.luigi_handler.launch_backfill_task(task, start_date, end_date)
+        except Exception as error:
+            log.warning("failed to launch backfill task: %s", task,
+                        exc_info=True)
+            return self.json_response({'error': simple_error(error)})
+        return self.json_response({'ok': True})
 
     def restart_scheduler(self):
         try:
@@ -87,36 +127,120 @@ class LuigiJobView(MasterView):
         return [
 
             # luigi proper
-            {'section': 'luigi',
+            {'section': 'rattail.luigi',
              'option': 'url'},
-            {'section': 'luigi',
+            {'section': 'rattail.luigi',
              'option': 'scheduler.supervisor_process_name'},
-            {'section': 'luigi',
+            {'section': 'rattail.luigi',
              'option': 'scheduler.restart_command'},
 
         ]
 
     def configure_get_context(self, **kwargs):
-        context = super(LuigiJobView, self).configure_get_context(**kwargs)
+        context = super(LuigiTaskView, self).configure_get_context(**kwargs)
-        context['overnight_tasks'] = self.luigi_handler.get_all_overnight_tasks()
+        context['overnight_tasks'] = self.get_overnight_tasks()
+        context['backfill_tasks'] = self.get_backfill_tasks()
         return context
 
-    def configure_gather_settings(self, data):
+    def get_overnight_tasks(self):
-        settings = super(LuigiJobView, self).configure_gather_settings(data)
+        tasks = self.luigi_handler.get_all_overnight_tasks()
+        for task in tasks:
+            if task['last_date']:
+                task['last_date'] = six.text_type(task['last_date'])
+        return tasks
 
+    def get_backfill_tasks(self):
+        tasks = self.luigi_handler.get_all_backfill_tasks()
+        for task in tasks:
+            if task['last_date']:
+                task['last_date'] = six.text_type(task['last_date'])
+            if task['target_date']:
+                task['target_date'] = six.text_type(task['target_date'])
+        return tasks
+
+    def configure_gather_settings(self, data):
+        settings = super(LuigiTaskView, self).configure_gather_settings(data)
+        app = self.get_rattail_app()
+
+        # overnight tasks
         keys = []
         for task in json.loads(data['overnight_tasks']):
-            keys.append(task['key'])
 
+            key = task['key']
+            if key.startswith('_new_'):
+                key = app.make_uuid()
+
+            key = task['key']
+            if key.startswith('_new_'):
+                cmd = shlex.split(task['script'])
+                script = os.path.basename(cmd[0])
+                root, ext = os.path.splitext(script)
+                key = re.sub(r'\s+', '-', root)
+
+            keys.append(key)
+            settings.extend([
+                {'name': 'rattail.luigi.overnight.{}.description'.format(key),
+                 'value': task['description']},
+                {'name': 'rattail.luigi.overnight.{}.script'.format(key),
+                 'value': task['script']},
+                {'name': 'rattail.luigi.overnight.{}.notes'.format(key),
+                 'value': task['notes']},
+            ])
         if keys:
-            settings.append({'name': 'luigi.overnight_tasks',
+            settings.append({'name': 'rattail.luigi.overnight_tasks',
+                             'value': ', '.join(keys)})
+
+        # backfill tasks
+        keys = []
+        for task in json.loads(data['backfill_tasks']):
+
+            key = task['key']
+            if key.startswith('_new_'):
+                script = os.path.basename(task['script'])
+                root, ext = os.path.splitext(script)
+                key = re.sub(r'\s+', '-', root)
+
+            keys.append(key)
+            settings.extend([
+                {'name': 'rattail.luigi.backfill.{}.description'.format(key),
+                 'value': task['description']},
+                {'name': 'rattail.luigi.backfill.{}.script'.format(key),
+                 'value': task['script']},
+                {'name': 'rattail.luigi.backfill.{}.forward'.format(key),
+                 'value': 'true' if task['forward'] else 'false'},
+                {'name': 'rattail.luigi.backfill.{}.notes'.format(key),
+                 'value': task['notes']},
+                {'name': 'rattail.luigi.backfill.{}.target_date'.format(key),
+                 'value': six.text_type(task['target_date'])},
+            ])
+        if keys:
+            settings.append({'name': 'rattail.luigi.backfill_tasks',
                              'value': ', '.join(keys)})
 
         return settings
 
     def configure_remove_settings(self):
-        super(LuigiJobView, self).configure_remove_settings()
+        super(LuigiTaskView, self).configure_remove_settings()
-        self.luigi_handler.purge_luigi_settings(self.Session())
+        app = self.get_rattail_app()
+        model = self.model
+        session = self.Session()
+
+        to_delete = session.query(model.Setting)\
+                           .filter(sa.or_(
+                               model.Setting.name == 'rattail.luigi.backfill_tasks',
+                               model.Setting.name.like('rattail.luigi.backfill.%.description'),
+                               model.Setting.name.like('rattail.luigi.backfill.%.forward'),
+                               model.Setting.name.like('rattail.luigi.backfill.%.notes'),
+                               model.Setting.name.like('rattail.luigi.backfill.%.script'),
+                               model.Setting.name.like('rattail.luigi.backfill.%.target_date'),
+                               model.Setting.name == 'rattail.luigi.overnight_tasks',
+                               model.Setting.name.like('rattail.luigi.overnight.%.description'),
+                               model.Setting.name.like('rattail.luigi.overnight.%.notes'),
+                               model.Setting.name.like('rattail.luigi.overnight.%.script')))\
+                           .all()
+
+        for setting in to_delete:
+            app.delete_setting(session, setting.name)
 
     @classmethod
     def defaults(cls, config):
@@ -130,16 +254,27 @@ class LuigiJobView(MasterView):
         url_prefix = cls.get_url_prefix()
         model_title_plural = cls.get_model_title_plural()
 
-        # launch job
+        # launch overnight
         config.add_tailbone_permission(permission_prefix,
-                                       '{}.launch'.format(permission_prefix),
+                                       '{}.launch_overnight'.format(permission_prefix),
-                                       label="Launch any Luigi job")
+                                       label="Launch any Overnight Task")
-        config.add_route('{}.launch'.format(route_prefix),
+        config.add_route('{}.launch_overnight'.format(route_prefix),
-                         '{}/launch'.format(url_prefix),
+                         '{}/launch-overnight'.format(url_prefix),
                          request_method='POST')
-        config.add_view(cls, attr='launch',
+        config.add_view(cls, attr='launch_overnight',
-                        route_name='{}.launch'.format(route_prefix),
+                        route_name='{}.launch_overnight'.format(route_prefix),
-                        permission='{}.launch'.format(permission_prefix))
+                        permission='{}.launch_overnight'.format(permission_prefix))
+
+        # launch backfill
+        config.add_tailbone_permission(permission_prefix,
+                                       '{}.launch_backfill'.format(permission_prefix),
+                                       label="Launch any Backfill Task")
+        config.add_route('{}.launch_backfill'.format(route_prefix),
+                         '{}/launch-backfill'.format(url_prefix),
+                         request_method='POST')
+        config.add_view(cls, attr='launch_backfill',
+                        route_name='{}.launch_backfill'.format(route_prefix),
+                        permission='{}.launch_backfill'.format(permission_prefix))
 
         # restart luigid scheduler
         config.add_tailbone_permission(permission_prefix,
@@ -156,8 +291,8 @@ class LuigiJobView(MasterView):
 def defaults(config, **kwargs):
     base = globals()
 
-    LuigiJobView = kwargs.get('LuigiJobView', base['LuigiJobView'])
+    LuigiTaskView = kwargs.get('LuigiTaskView', base['LuigiTaskView'])
-    LuigiJobView.defaults(config)
+    LuigiTaskView.defaults(config)
 
 
 def includeme(config):