[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10,"_11":12},"actionData","errors","root",{"_180":-5,"_183":184,"_185":186},"routes/shell",{"_43":44,"_179":-5,"_180":-5,"_181":182},"routes/repo",{"_34":-7,"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":32,"_46":32,"_47":32,"_48":32,"_49":32,"_50":32,"_51":52,"_53":52,"_54":32,"_55":-7,"_56":-7,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":65,"_66":67,"_68":69,"_70":71},"routes/repo.blob",{"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":-5,"_26":27,"_28":29,"_29":30,"_31":32,"_33":14},"ref","main","path","crates/logic/drain.rs","oid","fe17afa2201477f3a3a882991c19622b0af798d7","size",7357,"commit","90b43017241f81189453d326d05948f388b34652","mounts",[],"source","plain",{"_13":14,"_15":16},"kind","text","// Copyright 2026 Deno Land Inc. Apache-2.0 license.\n\n//! Sans-I/O policy for the fleet drain token.\n//!\n//! One well-known bucket object serializes concurrent donors: a draining\n//! node claims the token before it releases cells, so simultaneous stop\n//! signals (a node drain, a cluster upgrade, a spot reclaim) hand off one\n//! node at a time instead of flooding the survivors. A fresh node combines the\n//! token with the load in live node leases for its first readiness. The gate\n//! requires successor publication, memory headroom, bounded restoration, and\n//! bounded ownership skew before the fleet loses another node.\n//!\n//! The token is advisory for donor serialization. A donor that cannot claim it\n//! within a bounded wait proceeds unserialized, and a dead holder's claim\n//! lapses by TTL. Readiness stays fail-closed when the token or fleet capacity\n//! is unsettled, so the orchestrator's rollout deadline bounds that wait.\n\n/// The fleet drain token as stored at its well-known bucket key.\n#[derive(Clone, Debug, PartialEq, Eq)]\npub struct DrainToken {\n pub node: String,\n pub expires_ms: u64,\n /// The live nodes' cold-work levels immediately before this donor began.\n /// A released token keeps this snapshot for the replacement readiness gate.\n pub restoration_baseline: Vec,\n}\n\n#[derive(Clone, Debug, PartialEq, Eq)]\npub struct RestorationBaseline {\n pub node: String,\n pub restoring: u64,\n}\n\n/// Whether another node holds a live claim. A claim is live while\n/// `expires_ms > now_ms`, matching the node-lease liveness rule.\npub fn held_by_another(existing: Option<&DrainToken>, node: &str, now_ms: u64) -> bool {\n existing.is_some_and(|token| token.node != node && token.expires_ms > now_ms)\n}\n\n/// Whether a draining node may claim the token: it is absent, expired, or\n/// already its own, so a restarted logical node resumes its earlier claim.\npub fn may_claim(existing: Option<&DrainToken>, node: &str, now_ms: u64) -> bool {\n !held_by_another(existing, node, now_ms)\n}\n\n/// Why the first-readiness gate cannot open yet.\n#[derive(Clone, Debug, PartialEq, Eq)]\npub enum FleetUnsettled {\n Drain,\n Unreadable,\n SelfUnpublished,\n MemoryHeadroom { node: String },\n Restoration { current: u64, maximum: u64 },\n OwnershipSkew { current: usize, maximum: usize },\n}\n\n/// Whether the joining node can receive ownership during the next paced\n/// drain. The node contributes only when its lease is live, it advertises the\n/// handoff protocol, and it owns less than the busiest incumbent.\npub fn joining_contributes_successor_capacity(\n peers: &[crate::CapacityPeer],\n node: &str,\n now_ms: u64,\n) -> bool {\n let live = peers\n .iter()\n .filter(|peer| peer.expires_ms > now_ms)\n .collect::>();\n live.iter()\n .find(|peer| peer.node == node)\n .is_some_and(|joining| {\n joining.paced_handoff\n && live\n .iter()\n .filter(|peer| peer.node != node)\n .map(|peer| peer.resident_cells)\n .max()\n .is_some_and(|maximum| joining.resident_cells < maximum)\n })\n}\n\n/// Evaluate the drain token and the successor capacity published in live node\n/// leases. The joining node must publish itself. Every live peer must have\n/// memory headroom, each live node must return to its pre-donor restore\n/// baseline, and incumbent ownership must fit the successor-share envelope.\npub fn fleet_status(\n existing: Option<&DrainToken>,\n node: &str,\n now_ms: u64,\n peers: &[crate::CapacityPeer],\n max_restoring: u64,\n) -> Result<(), FleetUnsettled> {\n if held_by_another(existing, node, now_ms) {\n return Err(FleetUnsettled::Drain);\n }\n\n let live = peers\n .iter()\n .filter(|peer| peer.expires_ms > now_ms)\n .collect::>();\n if !live.iter().any(|peer| peer.node == node) {\n return Err(FleetUnsettled::SelfUnpublished);\n }\n if let Some(node) = live\n .iter()\n .filter(|peer| peer.pressured || peer.memory_headroom == Some(false))\n .map(|peer| &peer.node)\n .min()\n {\n return Err(FleetUnsettled::MemoryHeadroom { node: node.clone() });\n }\n\n // `restoring` includes ordinary cold activations, and their healthy level\n // can be nonzero under continuous load. With no token there was no donor,\n // so there is no rollout recovery backlog to gate. A donor snapshots the\n // healthy level before it releases ownership. Its replacement gets one\n // local activation budget of slack above each node's baseline; anything\n // more must settle before the fleet spends another donor. An older token\n // has an empty snapshot and therefore retains the conservative zero\n // baseline during a mixed-version rollout.\n let blocked_restoration = existing.and_then(|token| {\n live.iter()\n .filter_map(|peer| {\n let baseline = token\n .restoration_baseline\n .iter()\n .find(|baseline| baseline.node == peer.node)\n .map_or(0, |baseline| baseline.restoring);\n let maximum = baseline.saturating_add(max_restoring);\n (peer.restoring > maximum).then_some((&peer.node, peer.restoring, maximum))\n })\n .min_by(|left, right| left.0.cmp(right.0))\n });\n if let Some((_, current, maximum)) = blocked_restoration {\n return Err(FleetUnsettled::Restoration { current, maximum });\n }\n\n // A paced joining node publishes its peer endpoint before readiness and\n // accepts ownership-only handoffs without restoring a runtime. When that\n // node holds less ownership than the busiest incumbent, it is usable\n // successor capacity for the next donor. Requiring the incumbents to\n // rebalance first creates a closed loop: idle fleets move ownership only\n // during a handoff, while the orchestrator waits for readiness before it\n // starts that handoff.\n let joining_can_absorb_skew = joining_contributes_successor_capacity(peers, node, now_ms);\n\n // An unpaced or already-full joining node cannot repair ownership during\n // the next drain. Its incumbents must therefore remain within one equal\n // successor share above their mean.\n let incumbents = live\n .iter()\n .filter(|peer| peer.node != node)\n .collect::>();\n let count = incumbents.len();\n if count > 1 {\n let total = incumbents\n .iter()\n .fold(0_usize, |sum, peer| sum.saturating_add(peer.resident_cells));\n let count = count as u128;\n let numerator = (total as u128).saturating_mul(count.saturating_add(1));\n let denominator = count.saturating_mul(count);\n let maximum = numerator\n .saturating_add(denominator - 1)\n .checked_div(denominator)\n .unwrap_or_default()\n .min(usize::MAX as u128) as usize;\n let current = incumbents\n .iter()\n .map(|peer| peer.resident_cells)\n .max()\n .unwrap_or_default();\n if current > maximum && !joining_can_absorb_skew {\n return Err(FleetUnsettled::OwnershipSkew { current, maximum });\n }\n }\n Ok(())\n}\n","editable",false,"branch","documentNavigation","repo","denoland/celld","repoId","a23b8a10-92cf-4dc4-a7ab-a04e58298dde","ownerId","org:42048915","organization",true,"site","nexthub","private","admin","settings","maintainer","triage","starred","stars",1,"watchers","watching","pinned","notify","views",[135,136,137,138,139,140,141,142,143,144,145,146,147],"navigation",{"_57":58,"_93":94,"_95":96,"_97":98,"_99":100,"_101":102,"_103":104},"sources",[],"viewer","meta",{"_85":86,"_87":88,"_89":90,"_45":32,"_91":14,"_92":32,"_51":52,"_53":52},"branches",[14],"tags",[72,73,74,75,76,77,78,79,80,81,82,83,84],"cloneUrl","https://git.nexthub.sh/denoland/celld.git","v0.0.1","v0.0.2","v0.1.0","v0.2.0","v0.2.1","v0.3.0","v0.4.0","v0.4.1","v0.5.0","v0.5.1","v0.6.0","v0.6.1","v0.6.2","description","self-hosted, distributed Durable Objects","owner","denoland","section","","head","empty","pins",[118,119,120],"documents",{"_113":114,"_115":116},"code",{"_110":111,"_112":106},"automations",{"_108":52,"_109":-5},"delivery",{"_105":106,"_107":-5},"tag","7gTA52Z9a1vO6FP6eCQPix9zO4O","releases",0,"applications","workflows","deployments","commits",13,"changes","counts",{"_117":116},"total",38,"issue",{"_121":112,"_123":133,"_15":112,"_125":134,"_55":42,"_127":32,"_128":42,"_129":106},{"_121":130,"_123":131,"_15":130,"_125":132,"_55":42,"_127":32,"_128":42,"_129":106},{"_121":122,"_123":124,"_15":122,"_125":126,"_55":42,"_127":32,"_128":42,"_129":116},"id","issues","name","Issues","filters",{},"saved","pinnable","count","tasks","Tasks",{},"Changes",{},{"_121":112,"_123":133,"_15":112,"_125":178,"_55":42,"_127":32,"_128":42},{"_121":130,"_123":131,"_15":130,"_125":177,"_55":42,"_127":32,"_128":42},{"_121":122,"_123":124,"_15":122,"_125":176,"_55":42,"_127":32,"_128":42},{"_121":173,"_123":174,"_15":173,"_125":175,"_55":32,"_127":32,"_128":42},{"_121":170,"_123":171,"_15":170,"_125":172,"_55":32,"_127":32,"_128":42},{"_121":108,"_123":168,"_15":99,"_125":169,"_55":32,"_127":32,"_128":42},{"_121":109,"_123":165,"_15":166,"_125":167,"_55":32,"_127":32,"_128":42},{"_121":105,"_123":163,"_15":105,"_125":164,"_55":32,"_127":32,"_128":42},{"_121":159,"_123":160,"_15":161,"_125":162,"_55":32,"_127":32,"_128":42},{"_121":95,"_123":157,"_15":95,"_125":158,"_55":32,"_127":32,"_128":32},{"_121":154,"_123":155,"_15":154,"_125":156,"_55":32,"_127":32,"_128":32},{"_121":151,"_123":152,"_15":151,"_125":153,"_55":32,"_127":32,"_128":32},{"_121":148,"_123":149,"_15":148,"_125":150,"_55":32,"_127":32,"_128":32},"decisions","Decisions",{},"changelog","Changelog",{},"roadmap","Roadmap",{},"Documents",{},"activity","Activity","insights",{},"Releases",{},"Deployments","automations/deployments",{},"Workflows",{},"discussions","Discussions",{},"goals","Goals",{},{},{},{},"groups","user","agents",[],"now",1794663976704,"sidebar","expanded"]