Hi!

We're encountering a bug that happens when celld deploys a new version while a DO is in the middle of a request. We deploy to our staging environment all the time, so this hits us constantly.

In such cases: * celld tries to move each object to the new version but waits for it to finish, * if the object then makes an outgoing call (`fetch()`, a Worker Loader call, a DO stub call…), celld opens a new small request on that same object to check write safety, * but because the object is "waiting to move", that new request is queued for the new version.

This means that: * the new version needs the old request to finish, * the old request needs the outgoing call to finish, * the outgoing call needs the new version.

Nothing breaks the cycle until `CELLD_DEPLOY_MAX_AGE_S` forces the swap. The request then fails with `500 Worker failed: rejected: Error: The client has disconnected`.

Here's a simple repro. Put `wrangler.jsonc` and `index.js` in `./proj`, then run `CELLD=/path/to/celld ./repro.sh loader` (or `fetch`):

<details><summary>wrangler.jsonc, index.js, repro.sh</summary>

```jsonc { "name": "gate-swap", "main": "index.js", "compatibility_date": "2026-01-01", "worker_loaders": [{ "binding": "LOADER" }], "durable_objects": { "bindings": [{ "name": "CELL", "class_name": "Cell" }] }, "migrations": [{ "tag": "v1", "new_sqlite_classes": ["Cell"] }] } ```

```js import { DurableObject } from 'cloudflare:workers';

const CHILD = `export default { fetch() { return new Response('child ok'); } };`;

export class Cell extends DurableObject { async fetch(request) { const via = new URL(request.url).searchParams.get('via') ?? 'loader'; const t0 = Date.now(); // The redeploy lands while the handler waits here. await new Promise((resolve) => setTimeout(resolve, 5000)); let res; if (via === 'fetch') { res = await fetch(new URL('/child', request.url)); } else { const worker = this.env.LOADER.load({ compatibilityDate: '2026-01-01', mainModule: 'index.js', modules: { 'index.js': CHILD }, globalOutbound: null, }); res = await worker.getEntrypoint().fetch('http://child/'); } return new Response(`${via}: ${await res.text()} after ${Date.now() - t0} ms\n`); } }

export default { fetch(request, env) { if (new URL(request.url).pathname === '/child') return new Response('child ok'); return env.CELL.get(env.CELL.idFromName('a')).fetch(request); }, }; ```

```bash #!/bin/bash # Usage: CELLD=/path/to/celld ./repro.sh [loader|fetch] set -u cd "$(dirname "$0")" VIA=${1:-loader} "${CELLD:-celld}" dev proj --port 8787 --no-watch --logs > dev.log 2>&1 & DEV=$! trap 'kill $DEV; wait $DEV' EXIT until grep -q "ready http" dev.log; do kill -0 $DEV 2>/dev/null || exit 1; sleep 1; done INTERNAL=$(grep -o "internal listening on [0-9.:]*" dev.log | awk '{print $4}') # Warm the cell so it is resident on generation 1. curl -s -m 20 "http://127.0.0.1:8787/?via=$VIA" # The handler sleeps 5 s, then makes its outbound call; redeploy at t=1 s. curl -s -m 120 -w '%{http_code} in %{time_total}s\n' "http://127.0.0.1:8787/?via=$VIA" & CURL=$! sleep 1 curl -s -X POST "http://$INTERNAL/reload"; echo wait $CURL ```

On v0.6.1: `500 in 61.3s` with both `loader` and `fetch`. Expected: `200` in about 5 s. </details>

Here's a proposed patch:

<details><summary>patch against v0.6.1 (4 files, +51/−3)</summary>

```diff diff --git a/crates/celld/actor.rs b/crates/celld/actor.rs index f80ef37..113b7e0 100644 --- a/crates/celld/actor.rs +++ b/crates/celld/actor.rs @@ -440,6 +440,7 @@ pub enum Message { /// quiescing resident. Ordinary requests carry no identity and wait /// for the successor route. websocket: Option<u64>, + gate: bool, reply: oneshot::Sender<Result<Routed, RequestError>>, }, /// A caller disappeared before routing finished. The request identity is @@ -1281,6 +1282,15 @@ impl AppHandle { self.request_with_mode(cell, false, None).await }

+ pub async fn gate_request(&self, cell: String) -> Result<Routed, RequestError> { + match self.request_path(cell, false, None, true).await.1 { + RequestPathOutcome::Returned(result) => result, + RequestPathOutcome::SubmissionFailure | RequestPathOutcome::ReplyChannelClosed => { + Err(RequestError::NodeFenced) + } + } + } + /// Route one test request and retain which asynchronous seam produced the /// result. The shipping method keeps its compact public result, while this /// receipt prevents an inference from `NodeFenced` or an error string. @@ -1289,7 +1299,7 @@ impl AppHandle { &self, cell: String, ) -> RequestPathReceiptForTest { - let (request, outcome) = self.request_path(cell, false, None).await; + let (request, outcome) = self.request_path(cell, false, None, false).await; match outcome { RequestPathOutcome::SubmissionFailure => { RequestPathReceiptForTest::SubmissionFailure { request } @@ -1322,7 +1332,11 @@ impl AppHandle { capacity_handoff: bool, websocket: Option<u64>, ) -> Result<Routed, RequestError> { - match self.request_path(cell, capacity_handoff, websocket).await.1 { + match self + .request_path(cell, capacity_handoff, websocket, false) + .await + .1 + { RequestPathOutcome::Returned(result) => result, RequestPathOutcome::SubmissionFailure | RequestPathOutcome::ReplyChannelClosed => { Err(RequestError::NodeFenced) @@ -1335,6 +1349,7 @@ impl AppHandle { cell: String, capacity_handoff: bool, websocket: Option<u64>, + gate: bool, ) -> (u64, RequestPathOutcome) { let request = crate::asyncrt::next_core_request(); let (reply, receive) = oneshot::channel(); @@ -1346,6 +1361,7 @@ impl AppHandle { capacity_handoff, handoff_accept: None, websocket, + gate, reply, }) .is_err() @@ -1384,6 +1400,7 @@ impl AppHandle { reply: accept_reply, }), websocket: None, + gate: false, reply: route_reply, }) .is_err() @@ -2685,6 +2702,7 @@ impl Actor { capacity_handoff, handoff_accept, websocket, + gate, reply, } => { self.pending.insert(request, reply); @@ -2703,6 +2721,12 @@ impl Actor { now_ms: now_ms(), now_mono_ms: crate::asyncrt::mono_ms(), } + } else if gate { + Event::GateRequestAt { + request, + cell: cell.clone(), + now_mono_ms: crate::asyncrt::mono_ms(), + } } else if ownership_handoff { Event::HandoffRequestAt { request, diff --git a/crates/celld/cell_dispatch.rs b/crates/celld/cell_dispatch.rs index e64de29..e4334ed 100644 --- a/crates/celld/cell_dispatch.rs +++ b/crates/celld/cell_dispatch.rs @@ -46,7 +46,7 @@ impl std::error::Error for RoutedRequestError {} /// decides. `Ok` releases the held effect; `Err` breaks the call, as a routed /// gate failure would. pub async fn dispatch_gate(app: AppHandle, req: crate::js::GateReq) { - let routed = match app.request(req.scope.clone()).await { + let routed = match app.gate_request(req.scope.clone()).await { Ok(routed) => routed, Err(error) => { let _ = req.reply.send(Err(error)); diff --git a/crates/logic/lib.rs b/crates/logic/lib.rs index 5d0f9e6..3f82685 100644 --- a/crates/logic/lib.rs +++ b/crates/logic/lib.rs @@ -3543,6 +3543,21 @@ impl State { } }

+ fn gate_request(&mut self, request: RequestId, id: CellId, effects: &mut Vec<Effect>) { + let admitted = self.node_authoritative() + && !self.draining + && self.cells.get(&id).is_some_and(|cell| { + matches!(cell.phase, Phase::Resident { .. }) + && (self.active_cells.contains_key(&id) + || matches!(cell.alarm, Some(AlarmState::Firing { .. }))) + }); + if admitted { + self.complete_request(&id, request, Ok(Route::Local), effects); + } else { + self.request(request, id, false, false, effects); + } + } + fn request_authorized(&mut self, request: RequestId, id: CellId, effects: &mut Vec<Effect>) { let mut cell = self.cells.remove(&id).unwrap_or_default(); match &cell.phase { @@ -6633,6 +6648,7 @@ fn event_mono_ms(event: &Event) -> Option<u64> { | Event::CapacityRequestAt { now_mono_ms, .. } | Event::HandoffRequestAt { now_mono_ms, .. } | Event::WebSocketRequestAt { now_mono_ms, .. } + | Event::GateRequestAt { now_mono_ms, .. } | Event::OutputAt { now_mono_ms, .. } | Event::WakeHintAt { now_mono_ms, .. } | Event::TimerFired { now_mono_ms, .. } @@ -6718,6 +6734,9 @@ pub fn on_event(state: &mut State, event: Event) -> Vec<Effect> { websocket, .. } => state.websocket_request(request, cell, websocket, &mut effects), + Event::GateRequestAt { request, cell, .. } => { + state.gate_request(request, cell, &mut effects) + } Event::WorkerRequest { request } => state.worker_request(request, &mut effects), Event::BeginPreserve => state.begin_preserve(&mut effects), Event::Cancel { request } => state.cancel(request, &mut effects), diff --git a/crates/logic/types.rs b/crates/logic/types.rs index 6c50514..4141436 100644 --- a/crates/logic/types.rs +++ b/crates/logic/types.rs @@ -600,6 +600,11 @@ pub enum Event { now_ms: u64, now_mono_ms: u64, }, + GateRequestAt { + request: RequestId, + cell: CellId, + now_mono_ms: u64, + }, /// Reserve an idle resident isolate for a top-level Worker request. The /// shell falls back to the stateless pool when no resident is available; /// choosing and pinning a resident is lifecycle policy and therefore ``` </details>

Thanks!