Add a command to remove a job by id

If some unexpected error is encountered during the running of
a job, the job may be updated in the database to indicate that
it's running, but never complete or fail. This prevents the job
being retried, and any further attempts to run the job only update
the job scheduled to run after the current one is done. This adds a
command to remove specific jobs by id (identified by calling status
first).

ebmbot/bot.py

@@ -153,6 +153,10 @@ def _listener(event, say):
             handle_status(event, say)
             return
 
+        if text.startswith("remove job id"):
+            handle_remove_job(app, event, say, text)
+            return
+
         for slack_config in config["slack"]:
             if slack_config["regex"].match(text):
                 handle_command(app, event, say, slack_config)
@@ -235,6 +239,24 @@ def handle_status(message, say):
     say(status, thread_ts=message.get("thread_ts"))
 
 
+@log_call
+def handle_remove_job(app, message, say, text):
+    """Remove a job from the database so it can be rerun."""
+    app.client.reactions_add(
+        channel=message["channel"], timestamp=message["ts"], name="crossed_fingers"
+    )
+    job_id = int(text.split("remove job id ")[1])
+    jobs_ids = [job["id"] for job in scheduler.get_jobs()]
+    if job_id not in jobs_ids:
+        say(
+            f"Job id [{job_id}] not found in running or scheduled jobs",
+            thread_ts=message.get("thread_ts"),
+        )
+    else:
+        scheduler.mark_job_done(job_id)
+        say(f"Job id [{job_id}] removed", thread_ts=message.get("thread_ts"))
+
+
 def _build_status():
     running_jobs = []
     scheduled_jobs = []
@@ -410,6 +432,14 @@ def handle_help(message, say, help_configs, include_apology):
     lines.append(
         f"Enter `{prefix}[namespace] help` (eg `{prefix}{random.choice(list(help_configs))} help`) for more help"
     )
+    lines.append(f"Enter `{prefix}status` to see running and scheduled jobs")
+    lines.append(
+        f"""
+        Enter `{prefix}remove job id [id]` to remove a job; this will not cancel
+        jobs that are in progress, but will let you retry a job that appears to
+        have stalled.
+        """
+    )
     say("\n".join(lines), thread_ts=message.get("thread_ts"))
 
 

tests/test_bot.py

@@ -474,6 +474,32 @@ def test_new_channel_created(mock_app):
     ] == {"channel": "C0NEW", "users": "U1234"}
 
 
+def test_remove_job(mock_app):
+    recorder = mock_app.recorder
+    handle_message(mock_app, "<@U1234> test do job 10", reaction_count=1)
+    jobs = scheduler.get_jobs_of_type("test_good_job")
+    assert len(jobs) == 1
+    job_id = jobs[0]["id"]
+    handle_message(mock_app, f"<@U1234> remove job id {job_id}", reaction_count=2)
+    assert not scheduler.get_jobs_of_type("test_good_job")
+
+    post_message = recorder.mock_received_requests_kwargs["/chat.postMessage"][0]
+    assert (
+        "text",
+        "Job id [1] removed",
+    ) in post_message.items()
+
+
+def test_remove_non_existent_job(mock_app):
+    recorder = mock_app.recorder
+    handle_message(mock_app, "<@U1234> remove job id 10", reaction_count=1)
+    post_message = recorder.mock_received_requests_kwargs["/chat.postMessage"][0]
+    assert (
+        "text",
+        "Job id [10] not found in running or scheduled jobs",
+    ) in post_message.items()
+
+
 def handle_message(
     mock_app,
     text,