From 0f97747c2e3e6f3178e414adc5b6022ead275601 Mon Sep 17 00:00:00 2001 From: iximeow Date: Tue, 4 Jul 2023 10:33:30 -0700 Subject: cap max wait time for new tasks --- src/ci_driver.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ci_driver.rs b/src/ci_driver.rs index 5855352..f6c1828 100644 --- a/src/ci_driver.rs +++ b/src/ci_driver.rs @@ -533,6 +533,8 @@ async fn main() { } async fn find_client_task(dbctx: Arc, mut candidate: RunnerClient) -> Result<(), String> { + let find_client_task_start = std::time::Instant::now(); + let (run, job) = 'find_work: loop { // try to find a job for this candidate: // * start with pending runs - these need *some* client to run them, but do not care which @@ -568,6 +570,10 @@ async fn find_client_task(dbctx: Arc, mut candidate: RunnerClient) -> Res if candidate.test_connection().await.is_err() { return Err("lost client connection".to_string()); } + + if find_client_task_start.elapsed().as_secs() > 300 { + return Err("5min new task deadline elapsed".to_string()); + } }; eprintln!("enqueueing job {} for alternate run under host id {}", job.id, candidate.host_id); -- cgit v1.1