cap max wait time for new tasks

1 files changed, 6 insertions, 0 deletions

diff --git a/src/ci_driver.rs b/src/ci_driver.rs
index 5855352..f6c1828 100644
--- a/src/ci_driver.rs
+++ b/src/ci_driver.rs
@@ -533,6 +533,8 @@ async fn main() {
 }
 
 async fn find_client_task(dbctx: Arc<DbCtx>, mut candidate: RunnerClient) -> Result<(), String> {
+    let find_client_task_start = std::time::Instant::now();
+
     let (run, job) = 'find_work: loop {
         // try to find a job for this candidate:
         // * start with pending runs - these need *some* client to run them, but do not care which
@@ -568,6 +570,10 @@ async fn find_client_task(dbctx: Arc<DbCtx>, mut candidate: RunnerClient) -> Res
         if candidate.test_connection().await.is_err() {
             return Err("lost client connection".to_string());
         }
+
+        if find_client_task_start.elapsed().as_secs() > 300 {
+            return Err("5min new task deadline elapsed".to_string());
+        }
     };
 
     eprintln!("enqueueing job {} for alternate run under host id {}", job.id, candidate.host_id);