From 60acd646d7d6b568372838ed45ec1d880e04a6ac Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sat, 17 Oct 2026 13:51:32 -0600 Subject: [PATCH 1/7] Add the session protocol v2 types The client and server now each contribute a session nonce, and the sush nonce becomes their hash, which neither party chose alone. SessionSeq carries the signer's wall clock reading as an opaque, totally ordered sequence number, spoken as three codephrase words. Sequence numbers are not yet included in the session IDs, but will be shortly. Co-Authored-By: Claude Mythos 5 --- common/src/codephrases.rs | 127 +++++++++++++++--------- common/src/hash.rs | 7 ++ common/src/jobs.rs | 203 +++++++++++++++++++++++++++++++++++--- 3 files changed, 273 insertions(+), 64 deletions(-) diff --git a/common/src/codephrases.rs b/common/src/codephrases.rs index 23219f16..78cae843 100644 --- a/common/src/codephrases.rs +++ b/common/src/codephrases.rs @@ -46,6 +46,10 @@ const WORD_SEPARATOR: &str = "-"; /// the strength of the signatures produced over these phrases. const TRUNCATED_BITS: u32 = 88; +/// A three-word phrase carries 33 bits. Not enough for an identifier, +/// but room enough for a small ordered value, like a Unix time. +const SHORT_BITS: u32 = 33; + /// Random code phrase like `abstract misery favorite ordinary moon talk`. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Codephrase(U256); @@ -56,12 +60,20 @@ impl Codephrase { Self(U256::random(&mut OsRng)) } - /// Turn 256 bits of entropy into a reasonably unique code phrase. - /// We use the low-order words of the full codephrase, and pad to - /// the canonical length; these phrases are intended for machine - /// consumption, and are short enough for easy transmission. + /// Turn 256 bits of entropy into a reasonably unique code phrase, + /// around eight words. pub fn truncate(self) -> Self { - let mask = U256::from_u128(2u128.pow(TRUNCATED_BITS) - 1); + self.mask(TRUNCATED_BITS) + } + + /// Chop entropy down to three words; see `SHORT_BITS`. + pub fn shorten(self) -> Self { + self.mask(SHORT_BITS) + } + + /// Keep only the low-order bits of some entropy. + fn mask(self, bits: u32) -> Self { + let mask = U256::from_u128((1 << bits) - 1); Self(self.0.bitand(&mask)) } @@ -70,6 +82,23 @@ impl Codephrase { Self(U256::from_be_bytes(bytes)) } + /// Construct a codephrase from a small integer. + pub fn from_u64(value: u64) -> Self { + Self(U256::from_u64(value)) + } + + /// Return the small integer represented by a codephrase, + /// or `None` if it is wider than 64 bits. + pub fn to_u64(&self) -> Option { + let bytes = self.to_be_bytes(); + if bytes[..24].iter().any(|&byte| byte != 0) { + return None; + } + Some(u64::from_be_bytes( + bytes[24..].try_into().expect("eight bytes"), + )) + } + /// Get the underlying big-endian byte representation of the codephrase. pub fn to_be_bytes(&self) -> [u8; 32] { self.0.to_be_byte_array().into() @@ -77,8 +106,8 @@ impl Codephrase { } impl fmt::Display for Codephrase { + /// Turn 256 bits of entropy into an un-padded big-endian code phrase. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Turn 256 bits of entropy into an un-padded big-endian code phrase. let b = Reciprocal::new(Limb(WORDLIST_LEN as u64)).expect("should have some words"); let mut n = self.0; let mut r; @@ -102,18 +131,17 @@ impl fmt::Debug for Codephrase { impl FromStr for Codephrase { type Err = InvalidCodephrase; + /// Decode a big endian code phrase into 256 bits of entropy. + /// + /// Decoding is non-injective, or "liberal" in the sense that it will + /// accept non-canonical codephrases, e.g., ones with spaces instead of + /// separators, words in mixed case, or without leading zeros. This is + /// for the comfort of humans that may have to transmit such phrases. + /// It rejects phrases that no entropy encodes to: unknown words, more + /// than `PHRASE_WORDS_256` words, or a value of 256 bits or more. + /// The empty phrase decodes to zero. Callers that must distinguish + /// absent input from zero should check before decoding. fn from_str(value: &str) -> Result { - // Decode a big endian code phrase into 256 bits of entropy. - // - // Decoding is non-injective, or "liberal" in the sense that it will - // accept non-canonical codephrases, e.g., ones with spaces instead of - // separators, words in mixed case, or without leading zeros. This is - // for the comfort of humans that may have to transmit such phrases. - // It rejects phrases that no entropy encodes to: unknown words, more - // than [`PHRASE_WORDS_256`] words, or a value of 256 bits or more. - // The empty phrase decodes to zero. Callers that must distinguish - // absent input from zero should check before decoding. - let b = U256::from_u64(WORDLIST_LEN as u64); let mut n = U256::ZERO; let mut words = 0; @@ -158,7 +186,7 @@ impl<'de> Deserialize<'de> for Codephrase { } } -// Treat a codephrase as a string in JSON schemas. +/// Treat a codephrase as a string in JSON schemas. impl JsonSchema for Codephrase { fn schema_name() -> String { String::schema_name() @@ -192,13 +220,9 @@ macro_rules! codephrase_newtype { fn try_from( codephrase: $crate::codephrases::Codephrase, ) -> Result { - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - if codephrase.truncate() != codephrase { - return Err($crate::codephrases::InvalidCodephrase); - } - } + let length = $crate::codephrases::CodephraseLength::$len; + if length.canonical(codephrase) != codephrase { + return Err($crate::codephrases::InvalidCodephrase); } Ok(Self(codephrase)) } @@ -207,38 +231,24 @@ macro_rules! codephrase_newtype { impl $name { #[allow(unused)] $vis fn random() -> Self { - let mut codephrase = $crate::codephrases::Codephrase::random(); - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - codephrase = codephrase.truncate(); - } - } - Self(codephrase) + let length = $crate::codephrases::CodephraseLength::$len; + Self(length.canonical($crate::codephrases::Codephrase::random())) } #[allow(unused)] $vis fn from_hash(hash: $crate::hash::Hash) -> Self { - let mut codephrase = $crate::codephrases::Codephrase::from_be_bytes(*hash.as_bytes()); - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - codephrase = codephrase.truncate(); - } - } - Self(codephrase) + let length = $crate::codephrases::CodephraseLength::$len; + Self(length.canonical( + $crate::codephrases::Codephrase::from_be_bytes(*hash.as_bytes()), + )) } #[allow(unused)] $vis fn from_be_bytes(bytes: [u8; 32]) -> Self { - let mut codephrase = $crate::codephrases::Codephrase::from_be_bytes(bytes); - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - codephrase = codephrase.truncate(); - } - } - Self(codephrase) + let length = $crate::codephrases::CodephraseLength::$len; + Self(length.canonical( + $crate::codephrases::Codephrase::from_be_bytes(bytes), + )) } #[allow(unused)] @@ -273,6 +283,18 @@ macro_rules! codephrase_newtype { pub enum CodephraseLength { Full, Truncated, + Short, +} + +impl CodephraseLength { + /// Construct a codephrase of canonical length by discarding entropy. + pub fn canonical(self, codephrase: Codephrase) -> Codephrase { + match self { + Self::Full => codephrase, + Self::Truncated => codephrase.truncate(), + Self::Short => codephrase.shorten(), + } + } } /// Decoding a phrase failed. @@ -331,6 +353,13 @@ mod test { round_trip(U256::MAX); } + #[test] + fn small_integers() { + assert_eq!(Codephrase::from_u64(0).to_u64(), Some(0)); + assert_eq!(Codephrase::from_u64(u64::MAX).to_u64(), Some(u64::MAX)); + assert_eq!(Codephrase::from_be_bytes([0x01; 32]).to_u64(), None); + } + #[test] fn non_phrases() { // Unknown words, wherever they appear. diff --git a/common/src/hash.rs b/common/src/hash.rs index 2510c847..203160a0 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -87,6 +87,13 @@ impl Hasher { self } + /// Append length-prefixed bytes, so that adjacent updates cannot + /// bleed into one another. + pub fn update_with_len(&mut self, bytes: &[u8]) -> &mut Self { + self.update(&(bytes.len() as u64).to_be_bytes()); + self.update(bytes) + } + pub fn count(&self) -> u64 { self.count } diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 26eeb0e1..c16633b6 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -20,6 +20,7 @@ pub use sled_hardware_types::BaseboardId; use sled_hardware_types::BaseboardIdParseError; use thiserror::Error; +use crate::codephrases::Codephrase; use crate::hash::{Hash, Hasher, hash}; use crate::interactive::InteractiveJobError; use crate::keys::{KeyId, Signed, ToBeSigned, ToBeSignedFields, Verified}; @@ -83,20 +84,14 @@ impl SessionId { signer: SessionSignerNonce, ) -> Self { let mut hasher = Hasher::new(); - let mut hash_with_len = |data: &[u8]| { - hasher.update(&(data.len() as u64).to_be_bytes()); - hasher.update(data); - }; - - hash_with_len(b"SUSH_SESSION_ID_V1"); - hash_with_len(b"baseboard_id"); - hash_with_len(baseboard.part_number.as_bytes()); - hash_with_len(baseboard.serial_number.as_bytes()); - hash_with_len(b"sush_nonce"); - hash_with_len(&sush.to_be_bytes()); - hash_with_len(b"signer_nonce"); - hash_with_len(&signer.to_be_bytes()); - + hasher.update_with_len(b"SUSH_SESSION_ID_V1"); + hasher.update_with_len(b"baseboard_id"); + hasher.update_with_len(baseboard.part_number.as_bytes()); + hasher.update_with_len(baseboard.serial_number.as_bytes()); + hasher.update_with_len(b"sush_nonce"); + hasher.update_with_len(&sush.to_be_bytes()); + hasher.update_with_len(b"signer_nonce"); + hasher.update_with_len(&signer.to_be_bytes()); Self::from_hash(hasher.finalize()) } @@ -115,7 +110,46 @@ impl SessionId { } codephrase_newtype! { - /// The component of the hashed Session ID generated by the Sush server. + /// The session nonce component generated by the sush client + /// (on the operator's machine). + #[derive( + Copy, + Clone, + Deserialize, + Serialize, + JsonSchema, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + )] + pub struct SessionClientNonce = Truncated; +} + +codephrase_newtype! { + /// The session nonce component generated by the sush server + /// (on the rack). The server commits to it *before* it sees + /// the client's nonce, so it cannot grind over the pair. + #[derive( + Copy, + Clone, + Deserialize, + Serialize, + JsonSchema, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + )] + pub struct SessionServerNonce = Truncated; +} + +codephrase_newtype! { + /// The nonce the client submits to the signer, derived by + /// [`SessionSushNonce::compute`] from the client and server + /// nonces. #[derive( Copy, Clone, @@ -131,6 +165,25 @@ codephrase_newtype! { pub struct SessionSushNonce = Truncated; } +impl SessionSushNonce { + /// Hash the client and server nonces together. The server nonce + /// alone could carry anything the rack chose to hide in it, but + /// the hash of the pair carries nothing the rack could steer, + /// because the rack committed to its nonce before the client + /// chose its. An operator in a secure facility should therefore + /// be able to export the result based on the strength of their + /// own machine's nonce. + pub fn compute(client: SessionClientNonce, server: SessionServerNonce) -> Self { + let mut hasher = Hasher::new(); + hasher.update_with_len(b"SUSH_SESSION_NONCE_V2"); + hasher.update_with_len(b"client_nonce"); + hasher.update_with_len(&client.to_be_bytes()); + hasher.update_with_len(b"server_nonce"); + hasher.update_with_len(&server.to_be_bytes()); + Self::from_hash(hasher.finalize()) + } +} + codephrase_newtype! { /// The component of the hashed Session ID generated by the signer server. #[derive( @@ -148,6 +201,52 @@ codephrase_newtype! { pub struct SessionSignerNonce = Truncated; } +codephrase_newtype! { + /// The session's place in the signer-assigned order of sessions. + /// Taken as whole seconds since the Unix epoch according to the + /// *signer's* wall clock; the server's clock can't be trusted, so + /// it never interprets a seq as a time, only as an order. Sessions + /// authorized in the same second share a seq and tie-break by id. + /// Spoken as three codephrase words: 2048³ = 2³³ values, which + /// reaches the year 2242. + #[derive( + Copy, + Clone, + Deserialize, + Serialize, + JsonSchema, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + )] + pub struct SessionSeq = Short; +} + +#[derive(Debug, Error)] +#[error("session sequence number exceeds three-word codephrase range")] +pub struct SeqOutOfRange; + +impl SessionSeq { + /// The largest encodable wall clock reading, `zoo-zoo-zoo` + /// = 2242-03-16T12:56:31Z. + pub const MAX_SECONDS: u64 = (1 << 33) - 1; + + /// Make a sequence number from a wall clock reading. Fails + /// if seconds is out of range. + pub fn from_unix_seconds(seconds: u64) -> Result { + if seconds > Self::MAX_SECONDS { + return Err(SeqOutOfRange); + } + Ok(Self(Codephrase::from_u64(seconds))) + } + + pub fn as_unix_seconds(&self) -> u64 { + self.0.to_u64().expect("a sequence number is three words") + } +} + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub enum LastJob { #[default] @@ -733,6 +832,80 @@ mod test { use crate::keys::{EccR, EccS, EncodedSignature}; use crate::targets::SledId; + /// A sequence number encodes to three words, parses back liberally, + /// and orders numerically. + #[test] + fn seq_phrases() { + let max = SessionSeq::from_unix_seconds(SessionSeq::MAX_SECONDS).unwrap(); + assert_eq!(max.to_string(), "zoo-zoo-zoo"); + assert_eq!("zoo zoo zoo".parse::().unwrap(), max); + + // 2026-10-09T09:33:20Z, a realistic signer clock reading. + // Every wall clock reading since 1970-02-18 fills three words. + let seq = SessionSeq::from_unix_seconds(1_760_000_000).unwrap(); + assert_eq!(seq.as_unix_seconds(), 1_760_000_000); + assert_eq!(seq.to_string(), "crucial-over-abandon"); + assert_eq!(seq.to_string().parse::().unwrap(), seq); + assert!(seq < max); + + // 2³³ seconds exceed the range however they arrive. + assert!(SessionSeq::from_unix_seconds(1 << 33).is_err()); + assert!( + "ability-abandon-abandon-abandon" + .parse::() + .is_err() + ); + } + + /// On the binary wire a seq is codephrase bytes, like the ids and + /// nonces; as text it is words. Wide values refuse to decode. + #[test] + fn seq_serde_forms() { + let seq = SessionSeq::from_unix_seconds(1_760_000_000).unwrap(); + assert_eq!( + serde_json::to_value(seq).unwrap(), + serde_json::json!("crucial-over-abandon") + ); + let mut bytes = Vec::new(); + ciborium::ser::into_writer(&seq, &mut bytes).unwrap(); + assert_eq!( + hex::encode(&bytes), + "5820000000000000000000000000000000000000000000000000\ + 0000000068e77800" + ); + assert_eq!( + ciborium::de::from_reader::(bytes.as_slice()).unwrap(), + seq + ); + + let mut wide = Vec::new(); + ciborium::ser::into_writer(&EccR::from_be_bytes([0xff; 32]), &mut wide).unwrap(); + assert!(ciborium::de::from_reader::(wide.as_slice()).is_err()); + } + + /// Pin the derived sush nonce: the client and the rack both + /// compute it, and the signer hashes it into the session id, so + /// the derivation must never drift. + #[test] + fn pin_sush_nonce_compute() { + let nonce = SessionSushNonce::compute( + SessionClientNonce::from_be_bytes([0x11; 32]), + SessionServerNonce::from_be_bytes([0x22; 32]), + ); + assert_eq!( + nonce.to_string(), + "shop-adapt-belt-pulse-order-try-local-record" + ); + + // The field labels separate the operands: swapping the + // inputs changes the nonce. + let swapped = SessionSushNonce::compute( + SessionClientNonce::from_be_bytes([0x22; 32]), + SessionServerNonce::from_be_bytes([0x11; 32]), + ); + assert_ne!(nonce, swapped); + } + /// A job request is signed, and every sled rebuilds it from the /// wire to check the signature, so two sleds with different /// request shapes disagree about what verifies. This pin freezes From 4615a790673840d4b1099642bcd1412d204888b0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 18 Oct 2026 09:16:34 -0600 Subject: [PATCH 2/7] Run the v2 session ceremony end to end The rack serves a fresh server nonce per fetch, the client mixes in its own nonce and stashes it, the signer's nonce and sequence number come back through the operator, and the rack verifies the v2 session id over them all before gossiping Start(id, seq). The state machine carries the seq but does not yet order by it. Co-Authored-By: Claude Mythos 5 --- api/src/lib.rs | 24 +- client/src/cli.rs | 76 +++- client/src/commands.rs | 134 ++++-- client/src/context.rs | 25 +- client/src/lib.rs | 4 +- client/src/repl.rs | 31 +- common/src/jobs.rs | 8 +- server/src/manager.rs | 36 +- server/src/messages.rs | 18 +- server/src/server.rs | 23 +- server/src/state.rs | 41 +- server/tests/distributed.rs | 236 ++++++---- server/tests/output/session-start-request.bin | Bin 112 -> 147 bytes sush.json | 45 +- tests/src/integration_tests.rs | 84 +--- tests/src/manager_tests.rs | 402 ++++++++++++------ tests/src/test_utils.rs | 57 ++- 17 files changed, 798 insertions(+), 446 deletions(-) diff --git a/api/src/lib.rs b/api/src/lib.rs index 8ddc8932..5c437131 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -21,8 +21,8 @@ use sled_hardware_types::BaseboardId; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobLimits, JobOutputStream, JobStatus, JsonJobStatusMap, Session, SessionId, - SessionSignerNonce, SessionSushNonce, SignedJob, + Access, JobId, JobLimits, JobOutputStream, JobStatus, JsonJobStatusMap, Session, + SessionClientNonce, SessionId, SessionSeq, SessionServerNonce, SessionSignerNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::SledVersion; @@ -74,15 +74,16 @@ pub trait SushApi { headers: Header, ) -> Result, HttpError>; - /// Get the sush server nonce needed to start a new session. + /// Get a fresh server nonce for the session ceremony. /// - /// The nonce will need to be sent to the signer server, which will give back its own nonce. - /// Both nonces combined (along with the baseboard ID) will form the session ID. - #[endpoint { method = GET, path = "/session-nonce" }] - async fn session_start_nonce( + /// The client mixes its own nonce with this one to form the sush + /// nonce it submits to the signer. Fetching replaces the previous + /// server nonce, so only the latest ceremony can complete. + #[endpoint { method = GET, path = "/server-nonce" }] + async fn server_nonce( ctx: RequestContext, headers: Header, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Start a new support session. /// @@ -287,14 +288,11 @@ pub struct JobTargetParams { pub target: String, } -#[derive(Serialize, JsonSchema)] -pub struct SessionStartNonce { - pub nonce: SessionSushNonce, -} - #[derive(Deserialize, JsonSchema)] pub struct SessionStartBody { + pub client_nonce: SessionClientNonce, pub signer_nonce: SessionSignerNonce, + pub seq: SessionSeq, } /// Job parameters _not_ specified in the signed job request. diff --git a/client/src/cli.rs b/client/src/cli.rs index 4587363e..ace7436e 100644 --- a/client/src/cli.rs +++ b/client/src/cli.rs @@ -30,8 +30,9 @@ use xdg::BaseDirectories; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobOutputState, JobOutputStream, JobStatus, JobStatusMap, Session, SessionId, - SessionSignerNonce, SignedJob, job_status_to_json_map, + Access, JobId, JobOutputState, JobOutputStream, JobStatus, JobStatusMap, Session, + SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, + job_status_to_json_map, }; use sush_common::keys::{KeyId, Signature, SshPublicKey}; use sush_common::targets::{MAX_CUBBY, SledHealth, SledId, SledVersion}; @@ -40,13 +41,14 @@ use sush_common::version::VersionInfo; use crate::AuthzSigner; use crate::commands::{CommandError, GlobalArgs}; use crate::context::{CommandContext, OutputFormat, StatusDisplayStyle}; -use crate::types::SessionStartNonce; pub(crate) const PREFIX: &str = "sush"; const SESSION_FILE_NAME: &str = "session.json"; const SESSION_FILE_VERSION: u32 = 1; const TOKEN_FILE_NAME: &str = "permslip-token.json"; const TOKEN_FILE_VERSION: u32 = 1; +const CLIENT_NONCE_FILE_NAME: &str = "client-nonce.json"; +const CLIENT_NONCE_FILE_VERSION: u32 = 1; /// How long a permslip token is trusted for reuse. Staying well /// under the server's 15 minute TTL spares a command from expiring @@ -71,6 +73,16 @@ struct SavedToken { created: DateTime, } +/// The persisted client nonce of the pending session ceremony, +/// versioned for future migrations. One pending ceremony at a time, +/// mirroring the rack's one server nonce slot: `session start-params` +/// overwrites it just as the fetch it makes overwrites the slot. +#[derive(Deserialize, Serialize)] +struct SavedClientNonce { + version: u32, + client_nonce: SessionClientNonce, +} + #[derive(Clone, Debug, Default)] pub struct Cli { globals: Arc>, @@ -80,6 +92,7 @@ pub struct Cli { session: Arc>>, session_file: Option, token_file: Option, + client_nonce_file: Option, credentials: AuthzSigner, } @@ -114,6 +127,10 @@ impl Cli { Ok(path) => self.token_file = Some(path), Err(error) => eprintln!("❗ Signing tokens will not persist: {error}"), } + match BaseDirectories::with_prefix(PREFIX).place_state_file(CLIENT_NONCE_FILE_NAME) { + Ok(path) => self.client_nonce_file = Some(path), + Err(error) => eprintln!("❗ The client nonce will not persist: {error}"), + } } /// Adopt `session` unless one with the same ID is already @@ -313,6 +330,32 @@ impl CommandContext for Cli { } } + fn pending_client_nonce(&self) -> Option { + let json = fs::read(self.client_nonce_file.as_ref()?).ok()?; + match serde_json::from_slice::(&json) { + Ok(SavedClientNonce { + version: CLIENT_NONCE_FILE_VERSION, + client_nonce, + }) => Some(client_nonce), + _ => None, + } + } + + fn save_client_nonce(&self, client_nonce: SessionClientNonce) { + let Some(path) = &self.client_nonce_file else { + return; + }; + let result = serde_json::to_vec_pretty(&SavedClientNonce { + version: CLIENT_NONCE_FILE_VERSION, + client_nonce, + }) + .map_err(io::Error::other) + .and_then(|json| write_private(path, &json)); + if let Err(error) = result { + eprintln!("❗ The client nonce was not saved: {error}"); + } + } + fn session_id(&self) -> Option { self.session .lock() @@ -329,32 +372,45 @@ impl CommandContext for Cli { } } - fn session_start_params(&self, baseboard_id: BaseboardId, nonce: SessionStartNonce) { + fn session_start_params( + &self, + baseboard_id: BaseboardId, + client_nonce: SessionClientNonce, + sush_nonce: SessionSushNonce, + ) { match self.get_output_format() { OutputFormat::Json => println!( "{}", json!({ "baseboard_id": &baseboard_id.to_string(), - "sush_nonce": &nonce.nonce, + "client_nonce": &client_nonce, + "sush_nonce": &sush_nonce, }) ), OutputFormat::Text => { - println!("Baseboard ID: {baseboard_id}"); - println!("Sush nonce: {}", nonce.nonce); + println!("✅ Please relay these parameters to support:"); + println!(" Baseboard ID: {baseboard_id}"); + println!(" Sush nonce: {sush_nonce}"); } } } - fn session_created(&mut self, session: Session, signer_nonce: SessionSignerNonce) { + fn session_created( + &mut self, + session: Session, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, + ) { let session_id = self.adopt_session(session, true); match self.get_output_format() { OutputFormat::Json => println!( "{}", - json!({"session_created": session_id, "signer_nonce": signer_nonce}) + json!({"session_created": session_id, "signer_nonce": signer_nonce, "seq": seq}) ), OutputFormat::Text => { println!("✅ Session is now `{session_id}`"); - println!(" Signer nonce: {signer_nonce}"); + println!(" Signer nonce: {signer_nonce}"); + println!(" Sequence number: {seq}"); } } } diff --git a/client/src/commands.rs b/client/src/commands.rs index 626c9738..5af9a95d 100644 --- a/client/src/commands.rs +++ b/client/src/commands.rs @@ -45,16 +45,15 @@ use sush_api::JobWait; use sush_common::authn::{ AuthnError, Challenge, ChallengeResponse, Credentials, Identity, RequestKey, }; -#[cfg(feature = "permslip")] -use sush_common::codephrases::InvalidCodephrase; use sush_common::interactive::{InteractiveJobError, InteractiveJobMessage}; use sush_common::jobs::JobOutputStream::{self, Stderr, Stdout}; +#[cfg(feature = "permslip")] +use sush_common::jobs::JobStartRequest; use sush_common::jobs::{ Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStatus, JobStatusMap, - Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, job_status_try_from_json_map, + Session, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, + SignedJob, SkipReason, job_status_try_from_json_map, }; -#[cfg(feature = "permslip")] -use sush_common::jobs::{JobStartRequest, SessionSushNonce}; use sush_common::keys::{KeyError, KeyId, Signer as _}; use sush_common::targets::{SledId, Target}; use sush_common::version::VersionInfo; @@ -536,15 +535,14 @@ pub enum SessionCommand { key_id: KeyId, }, - /// Create a session at the signing service from a rack's start - /// parameters. + /// Create a session with the signing service from `session start-params`. #[cfg(feature = "permslip")] Create { /// The baseboard ID of the sled that generated the nonce. #[arg(value_parser = parse_baseboard_id)] baseboard_id: BaseboardId, - /// The rack's session nonce. + /// The sush nonce from `session start-params`. nonce: SessionSushNonce, /// Use `permslip` to sign sessions and jobs with this key name. @@ -566,8 +564,15 @@ pub enum SessionCommand { session_id: Option, /// The signer nonce for the session. + #[arg(requires = "seq")] nonce: Option, + /// The session sequence number assigned by the signer. + seq: Option, + + /// Override the client nonce generated and stashed by `session start-params`. + client_nonce: Option, + /// Use `permslip` to sign sessions and jobs with this key name. #[cfg(feature = "permslip")] #[arg(short, long, env = SUSH_PERMSLIP_KEY, value_name = "KEY_NAME")] @@ -1089,11 +1094,14 @@ async fn session( let (baseboard_id, nonce) = with_login(ctx, client, async || { Ok(( client.target().send().await?.into_inner(), - client.session_start_nonce().send().await?.into_inner(), + client.server_nonce().send().await?.into_inner(), )) }) .await?; - ctx.session_start_params(baseboard_id, nonce); + let client_nonce = SessionClientNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); + ctx.save_client_nonce(client_nonce); + ctx.session_start_params(baseboard_id, client_nonce, sush_nonce); Ok(()) } @@ -1102,6 +1110,8 @@ async fn session( SessionCommand::Start { session_id, nonce, + seq, + client_nonce, permslip, permslip_url, wait, @@ -1110,24 +1120,46 @@ async fn session( ) => { // Creation already announces the session; don't echo it // when the start succeeds. - let (session_id, nonce, show) = if let (Some(session_id), Some(nonce)) = - (session_id, nonce) - { - (session_id, nonce, true) - } else { - let (baseboard_id, nonce) = with_login(ctx, client, async || { - Ok(( - client.target().send().await?.into_inner(), - client.session_start_nonce().send().await?.into_inner(), - )) - }) - .await?; - let (session_id, nonce) = - session_create(ctx, permslip, permslip_url, &baseboard_id, nonce.nonce).await?; - ctx.session_created(Session::new(session_id), nonce); - (session_id, nonce, false) - }; - session_start(ctx, client, session_id, nonce, wait, show).await + let (session_id, body, show) = + if let (Some(session_id), Some(signer_nonce), Some(seq)) = (session_id, nonce, seq) + { + let client_nonce = client_nonce + .or_else(|| ctx.pending_client_nonce()) + .ok_or(CommandError::MissingClientNonce)?; + ( + session_id, + SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + true, + ) + } else { + let (baseboard_id, nonce) = with_login(ctx, client, async || { + Ok(( + client.target().send().await?.into_inner(), + client.server_nonce().send().await?.into_inner(), + )) + }) + .await?; + let client_nonce = SessionClientNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); + let (session_id, signer_nonce, seq) = + session_create(ctx, permslip, permslip_url, &baseboard_id, sush_nonce) + .await?; + ctx.session_created(Session::new(session_id), signer_nonce, seq); + ( + session_id, + SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + false, + ) + }; + session_start(ctx, client, session_id, body, wait, show).await } #[cfg(feature = "permslip")] @@ -1140,9 +1172,9 @@ async fn session( }, _, ) => { - let (session_id, nonce) = + let (session_id, nonce, seq) = session_create(ctx, permslip, permslip_url, &baseboard_id, nonce).await?; - ctx.session_created(Session::new(session_id), nonce); + ctx.session_created(Session::new(session_id), nonce, seq); Ok(()) } @@ -1151,16 +1183,31 @@ async fn session( SessionCommand::Start { session_id, nonce, + seq, + client_nonce, wait, }, Some(client), ) => { - let (session_id, nonce) = if let (Some(session_id), Some(nonce)) = (session_id, nonce) { - (session_id, nonce) - } else { + let (Some(session_id), Some(signer_nonce), Some(seq)) = (session_id, nonce, seq) else { return Err(CommandError::SigningUnavailable); }; - session_start(ctx, client, session_id, nonce, wait, true).await + let client_nonce = client_nonce + .or_else(|| ctx.pending_client_nonce()) + .ok_or(CommandError::MissingClientNonce)?; + session_start( + ctx, + client, + session_id, + SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + wait, + true, + ) + .await } (SessionCommand::Allow { key_id, write }, Some(client)) => { @@ -1728,19 +1775,20 @@ async fn session_create( permslip_url: Option, baseboard_id: &BaseboardId, nonce: SessionSushNonce, -) -> Result<(SessionId, SessionSignerNonce), CommandError> { +) -> Result<(SessionId, SessionSignerNonce, SessionSeq), CommandError> { let Some(permslip_url) = permslip_url else { return Err(CommandError::MissingPermslipUrl); }; let Some(permslip_key) = permslip else { return Err(CommandError::MissingKeyName); }; - let invalid = |e: InvalidCodephrase| CommandError::UnsupportedPermslipResponse(e.to_string()); let signer = permslip_signer(ctx, &permslip_key, &permslip_url).await?; - let created = signer.create_session(baseboard_id, nonce).await?; - Ok(( - created.session_id.to_string().parse().map_err(invalid)?, - created.signer_nonce.to_string().parse().map_err(invalid)?, + let _created = signer.create_session(baseboard_id, nonce).await?; + // The signing service cannot order sessions yet: its response + // carries no seq until the permission-slip side of protocol v2 + // lands. + Err(CommandError::UnsupportedPermslipResponse( + "the signing service returned no session sequence number".to_string(), )) } @@ -1748,7 +1796,7 @@ async fn session_start( ctx: &mut impl CommandContext, client: &Client, session_id: SessionId, - signer_nonce: SessionSignerNonce, + body: SessionStartBody, wait: bool, show: bool, ) -> Result<(), CommandError> { @@ -1758,7 +1806,7 @@ async fn session_start( .session_start() .session_id(session.session_id()) .wait(wait) - .body(SessionStartBody { signer_nonce }) + .body(body.clone()) .send() .await }) @@ -2496,6 +2544,8 @@ pub enum CommandError { Key(#[from] KeyError), #[error("❌ Length mismatch: expected {expected} bytes, received {received}")] LengthMismatch { expected: u64, received: u64 }, + #[error("❌ Missing client nonce: run `session start-params` first, or pass one")] + MissingClientNonce, #[error("❌ Missing job command, try `--help`")] MissingCommand, #[cfg(feature = "permslip")] diff --git a/client/src/context.rs b/client/src/context.rs index 44db4a40..ea2fca4b 100644 --- a/client/src/context.rs +++ b/client/src/context.rs @@ -14,7 +14,8 @@ use x509_cert::Certificate; use sush_common::authn::{BoundRequest, Credentials, Identity, RequestKey}; use sush_common::jobs::{ - Access, JobId, JobOutputStream, JobStatusMap, Session, SessionId, SessionSignerNonce, SignedJob, + Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, + SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::{SledId, SledVersion}; @@ -22,7 +23,6 @@ use sush_common::version::VersionInfo; use crate::AuthzSigner; use crate::commands::{CommandError, GlobalArgs}; -use crate::types::SessionStartNonce; /// Authorization state: the credentials that authenticated us, and the /// ephemeral key that binds each request we make. @@ -113,17 +113,28 @@ pub trait CommandContext: Clone + Send + Sync { fn set_credentials(&mut self, credentials: Option) { self.authz_signer().set(credentials) } - /// The cached permslip token for `url` and the key `fingerprint` - /// names, if it is still fresh. The default caches nothing. fn permslip_token(&self, _url: &str, _fingerprint: &str) -> Option { None } - /// Remember a permslip token for reuse. fn save_permslip_token(&self, _url: &str, _fingerprint: &str, _token: &str) {} + fn pending_client_nonce(&self) -> Option { + None + } + fn save_client_nonce(&self, _client_nonce: SessionClientNonce) {} fn session_id(&self) -> Option; fn next_job_id(&self) -> Result; - fn session_start_params(&self, baseboard_id: BaseboardId, nonce: SessionStartNonce); - fn session_created(&mut self, session: Session, signer_nonce: SessionSignerNonce); + fn session_start_params( + &self, + baseboard_id: BaseboardId, + client_nonce: SessionClientNonce, + sush_nonce: SessionSushNonce, + ); + fn session_created( + &mut self, + session: Session, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, + ); fn session_started(&mut self, session: Session, force: bool); fn session_stopped(&mut self, session_id: &SessionId); fn attach_allowed(&mut self, key_id: &KeyId, access: Access); diff --git a/client/src/lib.rs b/client/src/lib.rs index 1f9aab9a..3b90ca87 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -80,8 +80,10 @@ progenitor::generate_api!( KeyId = sush_common::keys::KeyId, Signature = sush_common::keys::Signature, SignedForJobStartRequest = sush_common::jobs::SignedJob, + SessionClientNonce = sush_common::jobs::SessionClientNonce, + SessionSeq = sush_common::jobs::SessionSeq, + SessionServerNonce = sush_common::jobs::SessionServerNonce, SessionSignerNonce = sush_common::jobs::SessionSignerNonce, - SessionSushNonce = sush_common::jobs::SessionSushNonce, SledVersion = sush_common::targets::SledVersion, JobMode = sush_common::jobs::JobMode, VersionInfo = sush_common::version::VersionInfo, diff --git a/client/src/repl.rs b/client/src/repl.rs index 3591afeb..cf710a4f 100644 --- a/client/src/repl.rs +++ b/client/src/repl.rs @@ -21,7 +21,8 @@ use xdg::BaseDirectories; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobOutputStream, JobStatusMap, Session, SessionId, SessionSignerNonce, SignedJob, + Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, + SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::{SledId, SledVersion}; @@ -33,7 +34,6 @@ use crate::commands::{ SUSH_OUTPUT_FORMAT, SUSH_URL, }; use crate::context::{CommandContext, OutputFormat, StatusDisplayStyle}; -use crate::types::SessionStartNonce; use crate::{AuthzSigner, Client}; const HISTORY_FILE: &str = "history.txt"; @@ -243,12 +243,31 @@ impl CommandContext for Repl { self.cli.next_job_id() } - fn session_start_params(&self, baseboard_id: BaseboardId, nonce: SessionStartNonce) { - self.cli.session_start_params(baseboard_id, nonce) + fn pending_client_nonce(&self) -> Option { + self.cli.pending_client_nonce() } - fn session_created(&mut self, session: Session, signer_nonce: SessionSignerNonce) { - self.cli.session_created(session, signer_nonce) + fn save_client_nonce(&self, client_nonce: SessionClientNonce) { + self.cli.save_client_nonce(client_nonce) + } + + fn session_start_params( + &self, + baseboard_id: BaseboardId, + client_nonce: SessionClientNonce, + sush_nonce: SessionSushNonce, + ) { + self.cli + .session_start_params(baseboard_id, client_nonce, sush_nonce) + } + + fn session_created( + &mut self, + session: Session, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, + ) { + self.cli.session_created(session, signer_nonce, seq) } fn session_started(&mut self, session: Session, force: bool) { diff --git a/common/src/jobs.rs b/common/src/jobs.rs index c16633b6..449c5535 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -82,9 +82,10 @@ impl SessionId { baseboard: &BaseboardId, sush: SessionSushNonce, signer: SessionSignerNonce, + seq: SessionSeq, ) -> Self { let mut hasher = Hasher::new(); - hasher.update_with_len(b"SUSH_SESSION_ID_V1"); + hasher.update_with_len(b"SUSH_SESSION_ID_V2"); hasher.update_with_len(b"baseboard_id"); hasher.update_with_len(baseboard.part_number.as_bytes()); hasher.update_with_len(baseboard.serial_number.as_bytes()); @@ -92,6 +93,8 @@ impl SessionId { hasher.update_with_len(&sush.to_be_bytes()); hasher.update_with_len(b"signer_nonce"); hasher.update_with_len(&signer.to_be_bytes()); + hasher.update_with_len(b"seq"); + hasher.update_with_len(&seq.to_be_bytes()); Self::from_hash(hasher.finalize()) } @@ -129,8 +132,7 @@ codephrase_newtype! { codephrase_newtype! { /// The session nonce component generated by the sush server - /// (on the rack). The server commits to it *before* it sees - /// the client's nonce, so it cannot grind over the pair. + /// (on the rack). #[derive( Copy, Clone, diff --git a/server/src/manager.rs b/server/src/manager.rs index 283818a0..ca0fa5b8 100644 --- a/server/src/manager.rs +++ b/server/src/manager.rs @@ -31,9 +31,9 @@ use sush_common::authn::{ Nonce, RequestVerifier, SeqWindow, }; use sush_common::jobs::{ - Access, JobId, JobStatusMap, Session, SessionId, SessionSignerNonce, SignedJob, + Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, + SessionSeq, SessionServerNonce, SessionSignerNonce, SessionSushNonce, SignedJob, }; -use sush_common::jobs::{JobOutputStream, SessionSushNonce}; use sush_common::keys::{KeyError, KeyId, SshPublicKey}; use sush_common::targets::{Cubbies, SledHealth, SledVersion}; use sush_common::version::LONG_VERSION; @@ -87,7 +87,7 @@ pub struct JobManager { log: Logger, nonces: Arc>>, identities: Arc>>, - session_sush_nonce: Arc>, + session_server_nonce: Arc>, output_dir: JobOutputDir, own_baseboard: BaseboardId, linked: LinkedBaseboards, @@ -144,7 +144,7 @@ impl JobManager { info!(log, "starting sush"; "version" => LONG_VERSION); let (tx_req, rx_req) = mpsc::channel(16); let requests = ReceiverStream::new(rx_req); - let session_sush_nonce = Arc::new(SyncMutex::new(SessionSushNonce::random())); + let session_server_nonce = Arc::new(SyncMutex::new(SessionServerNonce::random())); let boundary = Arc::new(BoundaryStore::new(&log, locker)); boundary.load().await; let (rx_state, join_state) = StateManager::run( @@ -156,7 +156,6 @@ impl JobManager { cubbies, universe, anchors, - session_sush_nonce.clone(), boundary, shutdown, )?; @@ -164,7 +163,7 @@ impl JobManager { log: log.new(o!("component" => "job manager")), nonces: Arc::new(Mutex::new(LruCache::new(MAX_OUTSTANDING_NONCES))), identities: Arc::new(Mutex::new(LruCache::new(MAX_CACHED_IDENTITIES))), - session_sush_nonce, + session_server_nonce, own_baseboard, linked, output_dir, @@ -457,28 +456,35 @@ impl JobManager { Ok(session.into_session_id()) } - pub fn session_sush_nonce(&self) -> SessionSushNonce { - *self.session_sush_nonce.lock().unwrap() + /// Replace the current server nonce with a fresh one and return + /// it. A fetch is the only rotation, so a client must finish its + /// ceremony before the next fetch. + pub fn fresh_server_nonce(&self) -> SessionServerNonce { + let nonce = SessionServerNonce::random(); + *self.session_server_nonce.lock().unwrap() = nonce; + nonce } pub async fn session_start( &self, authn: &Identity, session_id: SessionId, + client_nonce: SessionClientNonce, signer_nonce: SessionSignerNonce, + seq: SessionSeq, wait: bool, ) -> Result<(), JobError> { - // Verify that the session ID provided by the client is meant for - // this instance of Sush, by checking that the signer nonce was - // hashed with our baseboard and nonce. The state machine rotates - // the nonce when a session activates. - let sush_nonce = *self.session_sush_nonce.lock().unwrap(); - let computed = SessionId::compute(self.own_baseboard(), sush_nonce, signer_nonce); + // Verify that the session ID provided by the client is meant + // for this instance of Sush: it must hash over our baseboard + // and current server nonce. + let server_nonce = *self.session_server_nonce.lock().unwrap(); + let sush_nonce = SessionSushNonce::compute(client_nonce, server_nonce); + let computed = SessionId::compute(self.own_baseboard(), sush_nonce, signer_nonce, seq); if computed != session_id { return Err(JobError::InvalidSessionId); } - self.session_request(authn, SessionRequest::Start(session_id)) + self.session_request(authn, SessionRequest::Start(session_id, seq)) .await?; if wait { self.wait_for(self.wait_for_session(session_id)).await?; diff --git a/server/src/messages.rs b/server/src/messages.rs index 8a8a29f7..80bf12e0 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -14,7 +14,9 @@ use thiserror::Error; use sush_api::JobStartParams; use sush_common::authn::SignedLogin; use sush_common::jobs::JobOutputState; -use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob, SkipReason}; +use sush_common::jobs::{ + Access, JobId, ProcessError, SessionId, SessionSeq, SignedJob, SkipReason, +}; use sush_common::keys::KeyId; use sush_common::version::VersionInfo; @@ -194,7 +196,7 @@ pub mod v0 { pub fn kind(&self) -> &'static str { match self { Self::Session(r) => match r.request { - SessionRequest::Start(_) => "session start", + SessionRequest::Start(..) => "session start", SessionRequest::Stop(_) => "session stop", SessionRequest::Skip(..) => "session skip", SessionRequest::AllowAttach(..) => "session allow", @@ -213,7 +215,7 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum SessionRequest { - Start(SessionId), + Start(SessionId, SessionSeq), Stop(SessionId), Skip(SessionId, JobId), AllowAttach(SessionId, KeyId, Access), @@ -344,7 +346,7 @@ mod wire_format { fn snapshot_name(message: &Message) -> &'static str { match message { Message::Request(Request::Session(r)) => match r.request { - SessionRequest::Start(_) => "session-start-request", + SessionRequest::Start(..) => "session-start-request", SessionRequest::Stop(_) => "session-stop-request", SessionRequest::Skip(..) => "session-skip-request", SessionRequest::AllowAttach(..) => "session-allow-attach-request", @@ -455,7 +457,13 @@ mod wire_format { }; vec![ - Message::Request(Request::session(actor(), SessionRequest::Start(session))), + Message::Request(Request::session( + actor(), + SessionRequest::Start( + session, + SessionSeq::from_unix_seconds(1_760_000_000).unwrap(), + ), + )), Message::Request(Request::session(actor(), SessionRequest::Stop(session))), Message::Request(Request::session( actor(), diff --git a/server/src/server.rs b/server/src/server.rs index f1a6d665..565526d0 100644 --- a/server/src/server.rs +++ b/server/src/server.rs @@ -24,11 +24,12 @@ use tokio_tungstenite::tungstenite::protocol::{Role, WebSocketConfig}; use sush_api::{ AccessParam, Authorization, AuthorizedRangeRequest, JobHistoryParams, JobIdParam, JobOutputParams, JobStartParams, JobStopParams, JobTargetParams, KeyIdParam, RoutingParam, - SessionAndJobIds, SessionAndKeyIds, SessionIdParam, SessionStartBody, SessionStartNonce, - SushApi, WaitParam, + SessionAndJobIds, SessionAndKeyIds, SessionIdParam, SessionStartBody, SushApi, WaitParam, }; use sush_common::authn::Identity; -use sush_common::jobs::{JsonJobStatusMap, Session, SignedJob, job_status_to_json_map}; +use sush_common::jobs::{ + JsonJobStatusMap, Session, SessionServerNonce, SignedJob, job_status_to_json_map, +}; use sush_common::keys::{SshPublicKey, pem_cert_chain}; use sush_common::targets::SledVersion; use sush_common::version::VersionInfo; @@ -118,18 +119,16 @@ impl SushApi for ApiServer { } } - async fn session_start_nonce( + async fn server_nonce( ctx: RequestContext, headers: Header, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let mgr = ctx.context(); let Authorization { authorization } = headers.into_inner(); let _authn = mgr .iam(authorization, None, request_line(&ctx.request)) .await?; - Ok(HttpResponseOk(SessionStartNonce { - nonce: mgr.session_sush_nonce(), - })) + Ok(HttpResponseOk(mgr.fresh_server_nonce())) } async fn session_start( @@ -146,8 +145,12 @@ impl SushApi for ApiServer { .await?; let WaitParam { wait } = query.into_inner(); let SessionIdParam { session_id } = params.into_inner(); - let SessionStartBody { signer_nonce } = body.into_inner(); - mgr.session_start(&authn, session_id, signer_nonce, wait) + let SessionStartBody { + client_nonce, + signer_nonce, + seq, + } = body.into_inner(); + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, wait) .await?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/server/src/state.rs b/server/src/state.rs index 5bc68c64..e5a3a623 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -8,7 +8,7 @@ //! messages via the gossip protocol. use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use chrono::{DateTime, Utc}; use futures::{FutureExt as _, Stream, StreamExt}; @@ -23,8 +23,8 @@ use x509_cert::Certificate; use sush_api::JobStartParams; use sush_common::jobs::{ - Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, - SessionSushNonce, SignedJob, SkipReason, VerifiedJob, + Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, SignedJob, + SkipReason, VerifiedJob, }; use sush_common::keys::{KeyError, KeyId}; use sush_common::targets::Cubbies; @@ -403,7 +403,6 @@ enum Output { Ran(JobStatus), Stop(JobId), Gossip(Message), - RotateNonce, } impl Output { @@ -713,7 +712,7 @@ impl State { let mut outputs = Vec::new(); match request { - SessionRequest::Start(session_id) => match &mut self.session { + SessionRequest::Start(session_id, _seq) => match &mut self.session { // Re-announcement of active session; absorb and ignore. Active { starts, session, .. @@ -744,7 +743,6 @@ impl State { session.resume_at(committed.successor); } outputs.extend(self.skip_queued_jobs(now)); - outputs.push(Output::RotateNonce); self.session = Active { starts: self.session.starts() | incoming_version.clone(), started: incoming_version.clone(), @@ -1376,7 +1374,6 @@ fn apply( executor: &mut Executor, rumors: Option<&GossipNetwork>, store: &Arc, - nonce: &Arc>, input: Input, ) { let now = Utc::now(); @@ -1385,14 +1382,13 @@ fn apply( tx_state.send_modify(|state| { outputs = state.step(log, now, network, input); }); - apply_outputs(executor, rumors, store, nonce, outputs); + apply_outputs(executor, rumors, store, outputs); } fn apply_outputs( executor: &mut Executor, rumors: Option<&GossipNetwork>, store: &Arc, - nonce: &Arc>, outputs: Vec, ) { for output in outputs { @@ -1417,9 +1413,6 @@ fn apply_outputs( gossip(rumors, message.into()); } } - Output::RotateNonce => { - *nonce.lock().unwrap() = SessionSushNonce::random(); - } } } } @@ -1433,7 +1426,6 @@ fn drain_ready( executor: &mut Executor, rumors: Option<&GossipNetwork>, store: &Arc, - nonce: &Arc>, ) { while let Some(Some((version, message))) = causal_messages.next().now_or_never() { apply( @@ -1442,7 +1434,6 @@ fn drain_ready( executor, rumors, store, - nonce, Input::Gossip(version, message), ); } @@ -1525,7 +1516,6 @@ impl StateManager { mut cubbies: watch::Receiver, mut universe: watch::Receiver, anchors: &[Certificate], - session_sush_nonce: Arc>, store: Arc, shutdown: CancellationToken, ) -> Result<(watch::Receiver, JoinHandle<()>), KeyError> @@ -1694,7 +1684,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, Input::Gossip(version, message), ); // Settle only after replayed traffic: @@ -1711,7 +1700,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, ); apply( &log, @@ -1719,7 +1707,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, Input::Drained, ); } @@ -1734,7 +1721,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, Input::Cubbies(cubbies.borrow_and_update().clone()), ); }, @@ -1791,7 +1777,6 @@ impl StateManager { &mut executor, Some(&*rumors), &boundary_store, - &session_sush_nonce, ); apply( &log, @@ -1799,7 +1784,6 @@ impl StateManager { &mut executor, Some(&*rumors), &boundary_store, - &session_sush_nonce, Input::Drained, ); } @@ -1816,7 +1800,7 @@ mod test { use rumors::before::Clock; use x509_cert::time::Validity; - use sush_common::jobs::{JobMode, JobOutputState, JobStartRequest}; + use sush_common::jobs::{JobMode, JobOutputState, JobStartRequest, SessionSeq}; use sush_common::keys::{EphemeralKey, KeyType, Signer as _}; use sush_common::targets::Target; @@ -1926,6 +1910,10 @@ mod test { concurrent: Version, } + fn seq(seconds: u64) -> SessionSeq { + SessionSeq::from_unix_seconds(seconds).unwrap() + } + fn versions() -> Versions { let mut a = Clock::seed(); let mut b = a.fork(); @@ -2241,7 +2229,7 @@ mod test { let session_id = SessionId::random(); sim.request(Request::session( root.key_id().clone(), - SessionRequest::Start(session_id), + SessionRequest::Start(session_id, seq(1)), )); (root, sim, session_id, Session::new(session_id)) } @@ -2258,11 +2246,10 @@ mod test { let session_id = SessionId::random(); let session = Session::new(session_id); - let outputs = sim.request(Request::session( + sim.request(Request::session( actor.clone(), - SessionRequest::Start(session_id), + SessionRequest::Start(session_id, seq(1)), )); - assert!(outputs.iter().any(|o| matches!(o, Output::RotateNonce))); assert!(sim.state.session.session().is_some()); let job_id = session.next_job_id(); @@ -2436,7 +2423,7 @@ mod test { let history = vec![ Message::Request(Request::session( actor.clone(), - SessionRequest::Start(session_id), + SessionRequest::Start(session_id, seq(1)), )), Message::Request(Request::job( actor, diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index fd91afcb..b552d4ca 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -24,8 +24,9 @@ use tokio_util::sync::CancellationToken; use chrono::Utc; use sush_api::{JobStartParams, JobWait}; use sush_common::jobs::{ - JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, SessionId, - SessionSignerNonce, SignedJob, SkipReason, + JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, + SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, + SkipReason, }; use sush_common::keys::{EphemeralKey, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, SledHealth, SledId, Target}; @@ -160,15 +161,15 @@ async fn jobs_gossip_between_sleds() { // A session started on sled A becomes B's active session too. let authn_a = fake_identity(&mut root).await; let authn_b = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("session gossips to B", 60, async || { @@ -209,14 +210,14 @@ async fn jobs_gossip_between_sleds() { .await; // A session started on B supersedes A's everywhere. - let successor_nonce = SessionSignerNonce::random(); - let successor = SessionId::compute( - b.mgr.own_baseboard(), - b.mgr.session_sush_nonce(), - successor_nonce, - ); + let Ceremony { + session_id: successor, + client_nonce, + signer_nonce, + seq, + } = ceremony(&b.mgr, 2); b.mgr - .session_start(&authn_b, successor, successor_nonce, true) + .session_start(&authn_b, successor, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("supersession gossips to A", 60, async || { @@ -261,17 +262,31 @@ async fn concurrent_sessions_annihilate() { a.peers.send(BTreeSet::new()).unwrap(); b.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; - let nonce_a = SessionSignerNonce::random(); - let session_a = SessionId::compute(a.mgr.own_baseboard(), a.mgr.session_sush_nonce(), nonce_a); + let a_ceremony = ceremony(&a.mgr, 1); + let session_a = a_ceremony.session_id; let mut session = Session::new(session_a); a.mgr - .session_start(&authn_a, session_a, nonce_a, true) + .session_start( + &authn_a, + session_a, + a_ceremony.client_nonce, + a_ceremony.signer_nonce, + a_ceremony.seq, + true, + ) .await .unwrap(); - let nonce_b = SessionSignerNonce::random(); - let session_b = SessionId::compute(b.mgr.own_baseboard(), b.mgr.session_sush_nonce(), nonce_b); + let b_ceremony = ceremony(&b.mgr, 2); + let session_b = b_ceremony.session_id; b.mgr - .session_start(&authn_b, session_b, nonce_b, true) + .session_start( + &authn_b, + session_b, + b_ceremony.client_nonce, + b_ceremony.signer_nonce, + b_ceremony.seq, + true, + ) .await .unwrap(); @@ -339,15 +354,15 @@ async fn rejoining_replays_without_reexecuting() { // Sled A runs a whole job before B exists. let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -389,15 +404,22 @@ async fn rejoining_replays_without_reexecuting() { // Live traffic still executes everywhere: a fresh session's job runs // on both sleds. sees(&a, &b).await; - let successor_nonce = SessionSignerNonce::random(); - let successor = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), - successor_nonce, - ); + let Ceremony { + session_id: successor, + client_nonce, + signer_nonce: successor_nonce, + seq, + } = ceremony(&a.mgr, 1); let fresh = Session::new(successor); a.mgr - .session_start(&authn_a, successor, successor_nonce, true) + .session_start( + &authn_a, + successor, + client_nonce, + successor_nonce, + seq, + true, + ) .await .unwrap(); let live_job = fresh.next_job_id(); @@ -588,15 +610,15 @@ async fn stragglers_do_not_interrupt_live_jobs() { .await; sees(&a, &b).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -644,6 +666,32 @@ async fn stragglers_do_not_interrupt_live_jobs() { /// Wait until `anchor` has applied `joiner`'s build announcement. A /// session started on `anchor` afterward causally follows everything /// `joiner` held at entry, so it clears the joiner's entry floor. +/// Everything one sled hears at session start: a client nonce, a +/// signer nonce, the signer's clock reading, and the session id over +/// them all. +struct Ceremony { + session_id: SessionId, + client_nonce: SessionClientNonce, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, +} + +/// Build a [`Ceremony`] against one sled. Fetches a fresh server +/// nonce, so earlier ceremonies against that sled go stale. +fn ceremony(mgr: &JobManager, seq: u64) -> Ceremony { + let client_nonce = SessionClientNonce::random(); + let signer_nonce = SessionSignerNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()); + let seq = SessionSeq::from_unix_seconds(seq).unwrap(); + let session_id = SessionId::compute(mgr.own_baseboard(), sush_nonce, signer_nonce, seq); + Ceremony { + session_id, + client_nonce, + signer_nonce, + seq, + } +} + async fn sees(anchor: &Sled, joiner: &Sled) { eventually("the anchor sees the joiner", 60, async || { anchor @@ -709,15 +757,15 @@ async fn lost_suffix_never_reruns() { }) .await; sees(&a, &b).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("the session gossips to B", 120, async || { @@ -816,15 +864,15 @@ async fn lost_suffix_never_reruns() { assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); // A new session serves immediately. - let signer_nonce = SessionSignerNonce::random(); - let session2_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id: session2_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session2 = Session::new(session2_id); a.mgr - .session_start(&authn_a, session2_id, signer_nonce, true) + .session_start(&authn_a, session2_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("the new session gossips to B", 120, async || { @@ -895,15 +943,15 @@ async fn session_resumes_at_stored_successor() { .await; sees(&a, &b).await; sees(&a, &c).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("the session gossips to B and C", 120, async || { @@ -1034,15 +1082,15 @@ async fn universe_flip_flop_raises_floor() { let authn_a = fake_identity(&mut root).await; let d = Sled::start(&log, &dir, 3, &root_pem, &shutdown).await; let authn_d = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session2_id = SessionId::compute( - d.mgr.own_baseboard(), - d.mgr.session_sush_nonce(), + let Ceremony { + session_id: session2_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&d.mgr, 1); let mut session2 = Session::new(session2_id); d.mgr - .session_start(&authn_d, session2_id, signer_nonce, true) + .session_start(&authn_d, session2_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1060,15 +1108,15 @@ async fn universe_flip_flop_raises_floor() { }) .await; sees(&a, &x).await; - let signer_nonce = SessionSignerNonce::random(); - let session1_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id: session1_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session1 = Session::new(session1_id); a.mgr - .session_start(&authn_a, session1_id, signer_nonce, true) + .session_start(&authn_a, session1_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("session one gossips to X", 120, async || { @@ -1185,15 +1233,15 @@ async fn universe_flip_flop_raises_floor() { // A witnessed the refusal, so a session started now begins above // X's floor, and serves X again. - let signer_nonce = SessionSignerNonce::random(); - let session3_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id: session3_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session3 = Session::new(session3_id); a.mgr - .session_start(&authn_a, session3_id, signer_nonce, true) + .session_start(&authn_a, session3_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("session three gossips to X", 120, async || { @@ -1295,15 +1343,15 @@ async fn witnessed_session_survives_restart() { }) .await; sees(&a, &b).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1374,14 +1422,14 @@ async fn bookmarks_survive_restart() { // Sled A holds session history, so it wins every dominance contest. let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1478,15 +1526,15 @@ async fn gossip_survives_bookmark_failure() { }) .await; sees(&a, &b).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); diff --git a/server/tests/output/session-start-request.bin b/server/tests/output/session-start-request.bin index 36e0f284967b95fb7efd2050d27d284bbee0fb2e..10c2a180aa6c65332ac836e5b391bd20c7ae5006 100644 GIT binary patch delta 25 ecmXSz%s9c1xhX JobId { - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + async fn run_job( + mgr: &JobManager, + root: &mut EphemeralKey, + authn: &Identity, + seq: u64, + ) -> JobId { + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(mgr, seq); let session = Session::new(session_id); - mgr.session_start(authn, session_id, signer_nonce, true) + mgr.session_start(authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -1015,7 +1121,7 @@ async fn universe_swap() { // Run a job to completion in the first universe, and leave a // second one running at the swap. - let job_id = run_job(&mgr, &mut root, &authn).await; + let job_id = run_job(&mgr, &mut root, &authn, 1).await; assert!(mgr.job_status(&authn, &job_id).await.is_ok()); let session = mgr.session(&authn).expect("live session"); let lingering = session.next_job_id(); @@ -1047,7 +1153,7 @@ async fn universe_swap() { // The manager serves the new universe. Running a fresh job also // orders this check after the swap has fully settled. - run_job(&mgr, &mut root, &authn).await; + run_job(&mgr, &mut root, &authn, 2).await; // Nothing about the old universe's jobs crossed over: not the // completed job's history, and not the killed job's ending. @@ -1065,11 +1171,15 @@ async fn shutdown() { let log = test_logger(function_name!()); let (mut mgr, mut root, _dir, shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1138,11 +1248,15 @@ async fn attribution() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); assert_eq!( @@ -1229,11 +1343,15 @@ async fn job_targets() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1362,11 +1480,15 @@ async fn attach_grants() { EphemeralKey::new_root(KeyType::P256, ephemeral_test_subject(), validity).unwrap(); let guest = fake_identity(&mut guest_key).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&owner, session_id, signer_nonce, true) + mgr.session_start(&owner, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -1488,11 +1610,15 @@ async fn skip_and_stop_are_starter_only() { EphemeralKey::new_root(KeyType::P256, ephemeral_test_subject(), validity).unwrap(); let guest = fake_identity(&mut guest_key).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let job_id = Session::new(session_id).next_job_id(); - mgr.session_start(&owner, session_id, signer_nonce, true) + mgr.session_start(&owner, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1595,12 +1721,16 @@ async fn skip_racing_start_converges() { let (mgr, mut root, peer, _dir, _shutdown) = manager_test_root_and_peer(log).await; let owner = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut mirror = Session::new(session_id); let job_id = mirror.next_job_id(); - mgr.session_start(&owner, session_id, signer_nonce, true) + mgr.session_start(&owner, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1654,11 +1784,15 @@ async fn too_much_cpu() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -1708,11 +1842,15 @@ async fn too_much_output() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1806,11 +1944,15 @@ async fn output_ranges() { let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -2106,11 +2248,15 @@ async fn job_signal_dispositions() { let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -2151,11 +2297,15 @@ async fn interactive_target_required() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -2177,11 +2327,15 @@ async fn job_json() { let log = test_logger(function_name!()); let (mgr, mut root, dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -2216,11 +2370,15 @@ async fn streaming_validation() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 8733f17c..7be2e7cd 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -32,7 +32,10 @@ use sush_client::context::Authz; use sush_client::{Client, ResponseValue}; use sush_common::authn::{Challenge, ChallengeResponse, Credentials, Identity, Nonce, RequestKey}; use sush_common::codephrases::Codephrase; -use sush_common::jobs::{JobId, JobMode, JobStartRequest, SessionId, VerifiedJob}; +use sush_common::jobs::{ + JobId, JobMode, JobStartRequest, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, + SessionSushNonce, VerifiedJob, +}; use sush_common::keys::{EphemeralKey, KeyType, Signer, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; use sush_server::executor::PathIsolation; @@ -239,6 +242,58 @@ pub async fn manager_and_test_root( (mgr, root, dir, shutdown) } +/// Everything the rack hears at session start: a client nonce, a +/// signer nonce, the signer's clock reading, and the session id over +/// them all. +pub struct SessionCeremony { + pub session_id: SessionId, + pub client_nonce: SessionClientNonce, + pub sush_nonce: SessionSushNonce, + pub signer_nonce: SessionSignerNonce, + pub seq: SessionSeq, +} + +/// Build a [`SessionCeremony`] against `mgr`. Fetches a fresh server +/// nonce, as `GET /session-nonce` would, so earlier ceremonies +/// against `mgr` go stale. +pub fn session_ceremony(mgr: &JobManager, seq: u64) -> SessionCeremony { + let client_nonce = SessionClientNonce::random(); + let signer_nonce = SessionSignerNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()); + let seq = SessionSeq::from_unix_seconds(seq).unwrap(); + let session_id = SessionId::compute(mgr.own_baseboard(), sush_nonce, signer_nonce, seq); + SessionCeremony { + session_id, + client_nonce, + sush_nonce, + signer_nonce, + seq, + } +} + +/// Build a session id and the start body carrying its parts: like +/// [`session_ceremony`], but the server nonce comes from `client`'s +/// rack over HTTP. +pub async fn http_session_ceremony( + client: &Client, + seq: u64, +) -> (SessionId, sush_client::types::SessionStartBody) { + let client_nonce = SessionClientNonce::random(); + let signer_nonce = SessionSignerNonce::random(); + let server_nonce = client.server_nonce().send().await.unwrap().into_inner(); + let sush_nonce = SessionSushNonce::compute(client_nonce, server_nonce); + let seq = SessionSeq::from_unix_seconds(seq).unwrap(); + let session_id = SessionId::compute(&test_baseboard_id(), sush_nonce, signer_nonce, seq); + ( + session_id, + sush_client::types::SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + ) +} + /// Like [`manager_and_test_root`], but also hands back a clone of the /// manager's gossip network, for tests that play a peer. pub async fn manager_test_root_and_peer( From b281b75821f6e7135066d1537ebf0abcfc657b7c Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 18 Oct 2026 12:00:09 -0600 Subject: [PATCH 3/7] Order sessions totally by the signer's sequence number Session starts gossip the id's preimage, so every sled derives the id and the sequence number it commits to; a forged pairing of id and sequence number is inexpressible. Adoption and the execution floor follow the signer-assigned order: a start wins the session slot by exceeding the active (sequence number, id) pair, and a job runs only when its session exceeds the committed sequence number, resumes the committed session, or arrives live on a sled with no record. The order is total and global, so the causal machinery it replaces is gone: the starts/started lattice, concurrent-session annihilation, the executed-version join, the burned-universe Bloom set, session hops, in-memory floors, and the gossiped error events. Boundary records move to v1, with the sequence number in the committed job; v0 records convert with a floor of zero. Co-Authored-By: Claude Mythos 5 --- client/src/commands.rs | 4 +- common/src/jobs.rs | 61 +- .../output/job-start-request-schema.json | 2 +- server/src/bloom.rs | 161 ----- server/src/boundary.rs | 336 ++++----- server/src/lib.rs | 1 - server/src/manager.rs | 19 +- server/src/messages.rs | 63 +- server/src/state.rs | 637 +++++------------- server/tests/distributed.rs | 69 +- server/tests/output/boundary-record-v1.bin | Bin 0 -> 215 bytes .../output/concurrent-sessions-error.bin | Bin 221 -> 0 bytes server/tests/output/session-hop-error.bin | 1 - server/tests/output/session-start-request.bin | Bin 147 -> 270 bytes .../tests/output/universe-flip-flop-error.bin | 1 - tests/src/manager_tests.rs | 45 +- tests/src/test_utils.rs | 35 +- 17 files changed, 503 insertions(+), 932 deletions(-) delete mode 100644 server/src/bloom.rs create mode 100644 server/tests/output/boundary-record-v1.bin delete mode 100644 server/tests/output/concurrent-sessions-error.bin delete mode 100644 server/tests/output/session-hop-error.bin delete mode 100644 server/tests/output/universe-flip-flop-error.bin diff --git a/client/src/commands.rs b/client/src/commands.rs index 5af9a95d..833a5005 100644 --- a/client/src/commands.rs +++ b/client/src/commands.rs @@ -1785,8 +1785,8 @@ async fn session_create( let signer = permslip_signer(ctx, &permslip_key, &permslip_url).await?; let _created = signer.create_session(baseboard_id, nonce).await?; // The signing service cannot order sessions yet: its response - // carries no seq until the permission-slip side of protocol v2 - // lands. + // carries no sequence number until the permission-slip side of + // protocol v2 lands. Err(CommandError::UnsupportedPermslipResponse( "the signing service returned no session sequence number".to_string(), )) diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 449c5535..ce0eb040 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -61,7 +61,8 @@ impl From<&JobId> for JobId { } codephrase_newtype! { - /// A globally unique identifier for a session. + /// A globally unique identifier for a session. Constructed only + /// from a `SessionPreimage`. #[derive( Copy, Clone, @@ -77,27 +78,46 @@ codephrase_newtype! { pub struct SessionId = Truncated; } -impl SessionId { - pub fn compute( - baseboard: &BaseboardId, - sush: SessionSushNonce, - signer: SessionSignerNonce, - seq: SessionSeq, - ) -> Self { +/// The material a session ID hashes over, and the only source +/// of one. The `SessionRequest::Start` message gossips this +/// preimage to prevent a malicious or corrupted server from +/// poisoning the session. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SessionPreimage { + pub baseboard: BaseboardId, + pub sush_nonce: SessionSushNonce, + pub signer_nonce: SessionSignerNonce, + pub seq: SessionSeq, +} + +impl SessionPreimage { + pub fn session_id(&self) -> SessionId { + let Self { + baseboard: + BaseboardId { + part_number, + serial_number, + }, + sush_nonce, + signer_nonce, + seq, + } = self; let mut hasher = Hasher::new(); hasher.update_with_len(b"SUSH_SESSION_ID_V2"); hasher.update_with_len(b"baseboard_id"); - hasher.update_with_len(baseboard.part_number.as_bytes()); - hasher.update_with_len(baseboard.serial_number.as_bytes()); + hasher.update_with_len(part_number.as_bytes()); + hasher.update_with_len(serial_number.as_bytes()); hasher.update_with_len(b"sush_nonce"); - hasher.update_with_len(&sush.to_be_bytes()); + hasher.update_with_len(&sush_nonce.to_be_bytes()); hasher.update_with_len(b"signer_nonce"); - hasher.update_with_len(&signer.to_be_bytes()); + hasher.update_with_len(&signer_nonce.to_be_bytes()); hasher.update_with_len(b"seq"); hasher.update_with_len(&seq.to_be_bytes()); - Self::from_hash(hasher.finalize()) + SessionId::from_hash(hasher.finalize()) } +} +impl SessionId { pub fn first_job_id(&self) -> JobId { JobId::from_hash(hash(&self.0.to_be_bytes())) } @@ -149,9 +169,8 @@ codephrase_newtype! { } codephrase_newtype! { - /// The nonce the client submits to the signer, derived by - /// [`SessionSushNonce::compute`] from the client and server - /// nonces. + /// The nonce the client submits to the signer, + /// derived from the client and server nonces. #[derive( Copy, Clone, @@ -207,8 +226,9 @@ codephrase_newtype! { /// The session's place in the signer-assigned order of sessions. /// Taken as whole seconds since the Unix epoch according to the /// *signer's* wall clock; the server's clock can't be trusted, so - /// it never interprets a seq as a time, only as an order. Sessions - /// authorized in the same second share a seq and tie-break by id. + /// the server never interprets a sequence number as a time, only + /// as an order. Sessions authorized in the same second share a + /// sequence number and tie-break by id. /// Spoken as three codephrase words: 2048³ = 2³³ values, which /// reaches the year 2242. #[derive( @@ -859,8 +879,9 @@ mod test { ); } - /// On the binary wire a seq is codephrase bytes, like the ids and - /// nonces; as text it is words. Wide values refuse to decode. + /// On the binary wire a sequence number is codephrase bytes, like + /// the ids and nonces; as text it is words. A value too wide for + /// three words fails to decode. #[test] fn seq_serde_forms() { let seq = SessionSeq::from_unix_seconds(1_760_000_000).unwrap(); diff --git a/common/tests/output/job-start-request-schema.json b/common/tests/output/job-start-request-schema.json index c5cd765c..71afee34 100644 --- a/common/tests/output/job-start-request-schema.json +++ b/common/tests/output/job-start-request-schema.json @@ -42,7 +42,7 @@ ] }, "SessionId": { - "description": "A globally unique identifier for a session.", + "description": "A globally unique identifier for a session. Constructed only from a `SessionPreimage`.", "type": "string" } } diff --git a/server/src/bloom.rs b/server/src/bloom.rs deleted file mode 100644 index 5ede9cd5..00000000 --- a/server/src/bloom.rs +++ /dev/null @@ -1,161 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! A fixed-size Bloom filter for durable, grow-only sets. -//! -//! Designed for the boundary record's burned networks set. Entries may -//! never be dropped (once burned, a network stays burned), the size must -//! stay bounded under adversarial growth (we must not fill the M.2s), -//! and the filter may err only by claiming a key it never held. The -//! caller must treat that claim as refusal. -//! -//! The layout and hash algorithm determine the on-disk format, and must -//! not be changed without updating the boundary record's magic. Each key -//! probes the table at seven positions, cut as 16-bit words from its -//! SHA3-256 digest. The table is currently sized at 8192 bits, for -//! which seven probes is the optimal count out to about 800 entries -//! (bits * ln 2 / probes). The false positive rate there is about 0.7%, -//! degrading gradually past it. Real occupancy should stay in the tens, -//! where false positives are negligible. - -use std::array::from_fn; -use std::fmt; - -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -use sush_common::hash::{OUT_LEN, hash}; -use sush_common::wire::ExactBytes; - -const BYTES: usize = 1024; -const BITS: u64 = (BYTES * 8) as u64; -const PROBES: usize = 7; - -// The probes must fit in the digest, and the table size must divide -// 2^16, or reducing a 16-bit word to a table slot would favor some -// slots over others. -const _: () = { - assert!(2 * PROBES <= OUT_LEN); - assert!((1u64 << 16).is_multiple_of(BITS)); -}; - -/// A grow-only set of byte-string keys. -#[derive(Clone, Eq, PartialEq)] -pub struct Bloom { - bits: Box<[u8; BYTES]>, -} - -/// The probe positions for `key` are seven 16-bit words cut from -/// the leading bytes of the key's SHA3-256 digest, each reduced to -/// a table slot. -fn probes(key: &[u8]) -> [u64; PROBES] { - let digest = *hash(key).as_bytes(); - from_fn(|i| { - let j = 2 * i; - let word = u16::from_le_bytes(digest[j..j + 2].try_into().expect("two bytes")); - u64::from(word) % BITS - }) -} - -/// The byte index and mask selecting `probe`'s bit. -fn bit(probe: u64) -> (usize, u8) { - ((probe / 8) as usize, 1 << (probe % 8)) -} - -impl Bloom { - pub fn new() -> Self { - Self { - bits: Box::new([0; BYTES]), - } - } - - /// Permanently add `key` to the set. - pub fn insert(&mut self, key: &[u8]) { - for (byte, mask) in probes(key).map(bit) { - self.bits[byte] |= mask; - } - } - - /// Whether `key` may have been inserted. Never returns a false negative, - /// but false positives occur at the documented rate. - pub fn contains(&self, key: &[u8]) -> bool { - probes(key) - .map(bit) - .into_iter() - .all(|(byte, mask)| self.bits[byte] & mask != 0) - } -} - -impl Default for Bloom { - fn default() -> Self { - Self::new() - } -} - -impl fmt::Debug for Bloom { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let set: u32 = self.bits.iter().map(|byte| byte.count_ones()).sum(); - write!(f, "Bloom({set}/{BITS} bits set)") - } -} - -impl Serialize for Bloom { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_bytes(&self.bits[..]) - } -} - -impl<'de> Deserialize<'de> for Bloom { - fn deserialize>(deserializer: D) -> Result { - let bits = deserializer.deserialize_bytes(ExactBytes::)?; - Ok(Self { - bits: Box::new(bits), - }) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn round_trip() { - let mut bloom = Bloom::new(); - assert!(!bloom.contains(b"lost-network")); - bloom.insert(b"lost-network"); - assert!(bloom.contains(b"lost-network")); - assert!(!bloom.contains(b"other-network")); - } - - #[test] - fn serde_preserves_bits() { - let mut bloom = Bloom::new(); - bloom.insert(b"burned"); - let mut bytes = Vec::new(); - ciborium::ser::into_writer(&bloom, &mut bytes).unwrap(); - let back: Bloom = ciborium::de::from_reader(bytes.as_slice()).unwrap(); - assert_eq!(bloom, back); - assert!(back.contains(b"burned")); - } - - #[test] - fn false_positives_are_rare() { - let mut bloom = Bloom::new(); - for i in 0..128 { - bloom.insert(format!("burned-{i}").as_bytes()); - } - let hits = (0..10_000) - .filter(|i| bloom.contains(format!("probe-{i}").as_bytes())) - .count(); - assert_eq!( - hits, 0, - "false positives at plausible occupancy: {hits}/10000" - ); - } - - /// If this test fails, STOP! The on-disk format may have changed! - #[test] - fn pin_probes() { - assert_eq!(probes(b"sush"), [3392, 1007, 8018, 5416, 1335, 7804, 1908]); - } -} diff --git a/server/src/boundary.rs b/server/src/boundary.rs index 825cdd3b..bdfb38b1 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -5,49 +5,30 @@ //! The boundary between jobs we executed and jobs we only heard about. //! //! One record, rewritten before each spawn, carries what this sled -//! last committed to: the job, its session, the chain position after -//! the job, the universe they belong to, and the job's ending once -//! it has one. A universe is one shared gossip history, identified -//! by its network; sleds join universes, leave them, and sometimes -//! return. Alongside the commitment, the record keeps the join -//! of every session start this sled has executed under in that -//! universe, and the set of universes it burned by leaving. The -//! record is this sled's execution watermark: nothing at or below it -//! may run again. +//! last committed to: the job, its session and that session's +//! sequence number, the chain position after the job, the universe +//! they belong to, and the job's ending once it has one. The record +//! is this sled's execution floor: the signer assigns every session +//! a sequence number from its wall clock, so sequence numbers order +//! sessions totally, across restarts and across universes, and no +//! session at or below the committed sequence number may run jobs +//! here again. //! //! After a restart, replayed gossip rebuilds the sessions. When the //! committed session activates, the sled resumes its chain at the //! stored successor. The previous life had already moved the chain //! past every earlier position, so the session's queue never -//! releases them here, and the chain continues from the successor -//! whether its request arrives by replay or by resubmission. A -//! session that starts strictly above the executed join has never -//! run here and is served. A session the record cannot order makes -//! the sled hop: the sled reports the hop to the gossip set as an -//! error and sets a floor, in memory, at the frontier that includes -//! the report. A session that does not start above the floor has -//! its jobs skipped; a session started after the hop is served. The -//! join folds in every session this sled has executed under in this -//! universe, so a session it once ran under can never screen as -//! new: the sled hops, and the session's jobs are skipped rather -//! than re-run. If replay gives the recorded job no status, the sled -//! adjudicates it: it announces the recorded ending when the record -//! holds one, and an interrupted ending when it does not. +//! releases those jobs here, and the chain continues from the +//! successor whether its request arrives by replay or by +//! resubmission. A session whose sequence number exceeds the +//! committed one has never run jobs here and is served; any other +//! session's jobs are skipped. If replay gives the recorded job no +//! status, the sled adjudicates it: it announces the recorded ending +//! when the record holds one, and an interrupted ending when it does +//! not. //! -//! Universes have no order. The record instead keeps a burned set, -//! holding the network of every universe whose watermark it -//! overwrote by moving on. A sled that re-enters a burned universe -//! has flip-flopped, and raises its floor: it reports the flip-flop -//! to the gossip set and sets the floor, in memory, at the frontier -//! that includes the report. No older message can contain a version -//! born at that instant, so a session started before the re-entry -//! never lies above the floor, and its jobs are skipped. A sled that -//! enters a universe with history while holding no record for it -//! raises a floor the same way. The floor is never persisted: the -//! burn, the missing record, or the unordered session is still there -//! after a restart, and raises it again. A floor written to disk -//! could carry a version that died with the life that created it; -//! no later session could ever dominate such a floor. +//! A sled with no record cannot tell replay from re-run, so it +//! serves only sessions that start as live traffic. //! //! A boundary that cannot be written means the job must not run. A //! boundary that cannot be trusted means no job may run at all, since @@ -57,16 +38,14 @@ use std::sync::Mutex as SyncMutex; use std::sync::atomic::{AtomicBool, Ordering}; -use ciborium::ser::into_writer as into_cbor; -use rumors::{Network, Version}; +use rumors::Network; use serde::{Deserialize, Serialize}; use slog::{Logger, o, warn}; use thiserror::Error; -use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId}; +use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId, SessionSeq}; -use crate::bloom::Bloom; -use crate::format::{self, NoFormat, Record, Versioned}; +use crate::format::{self, Record, Versioned}; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOUNDARY: TenantSpec = TenantSpec { @@ -74,97 +53,37 @@ pub const BOUNDARY: TenantSpec = TenantSpec { magic: b"SUSHBOUNDARY", }; -/// The execution boundary: what this sled last committed to, in which -/// universe, and which universes it has left behind ("burned"). -/// -/// Versions do not compare across universes, so `network` scopes every -/// version in the record. `burned` holds the network of every -/// universe whose watermark this record overwrote by moving on: a -/// sled re-entering one raises its floor in memory, and the record on -/// disk stays the displaced universe's true watermark until a commit -/// overwrites it. -/// -/// `executed` is the join of the start versions of every session -/// this sled has executed under in this universe. A single stored -/// start would forget the sessions before it, and a third session -/// could then replay the first session's jobs; the join never -/// forgets. Session starts are witnessed messages, and only those -/// keep their meaning across a crash. A start that only this sled -/// ever saw belongs to a session whose history is lost, and refusal -/// is the right answer there anyway. +/// The execution boundary: what this sled last committed to, and in +/// which universe. The committed job's sequence number is the sled's +/// global execution floor: the signer orders sessions totally and +/// across universes, so every session a previous life could have run +/// jobs under sits at or below this one stored number. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Boundary { pub network: Network, - pub burned: Bloom, - #[serde(with = "version_bytes")] - pub executed: Version, pub job: Option, } /// The last job this sled committed to running, and how far it got. /// We also store the chain position *after* `job`, computed from the /// request's signed bytes at commit time, to allow session resumption. +/// +/// The sequence number itself travels unsigned, in the gossiped +/// session preimage, yet a forged one cannot reach this record: the +/// sled writes the record only when it commits to a signed job, the +/// job's signed bytes name the session id, and the id hashes over +/// the preimage, sequence number included. A preimage with a +/// different sequence number hashes to a different session id, and +/// no signed job names that id. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Committed { pub session: SessionId, + pub seq: SessionSeq, pub job: JobId, pub successor: JobId, pub outcome: JobOutcome, } -impl Boundary { - /// Whether this record burned `network`: left its universe behind - /// and overwrote its watermark. - pub fn is_burned(&self, network: Network) -> bool { - self.burned.contains(&network_key(network)) - } - - /// The burned set for the replacement record, committed in - /// `network`. A replacement in a different universe burns this - /// record's own network. - pub fn burned_for(&self, network: Network) -> Bloom { - let mut burned = self.burned.clone(); - if self.network != network { - burned.insert(&network_key(self.network)); - } - burned - } - - /// The executed-session join for the replacement record, - /// committed in `network` and folding in `started`. Joins never - /// cross universes, so a replacement elsewhere starts its join - /// fresh. - pub fn executed_for(&self, network: Network, started: &Version) -> Version { - if self.network == network { - self.executed.clone() | started.clone() - } else { - started.clone() - } - } -} - -/// A network's Bloom key is its CBOR bytes. -fn network_key(network: Network) -> Vec { - let mut bytes = Vec::new(); - into_cbor(&network, &mut bytes).expect("writing to a Vec cannot fail"); - bytes -} - -mod version_bytes { - use rumors::Version; - use serde::de::Error as _; - use serde::{Deserialize, Deserializer, Serializer}; - - pub fn serialize(version: &Version, serializer: S) -> Result { - serializer.serialize_bytes(&version.encode()) - } - - pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - let bytes = >::deserialize(deserializer)?; - Version::decode(bytes.as_slice()).map_err(D::Error::custom) - } -} - /// How far the boundary job got. #[derive(Clone, Debug, Deserialize, Serialize)] pub enum JobOutcome { @@ -174,21 +93,118 @@ pub enum JobOutcome { Ended(JobStatus), } -/// Version 0 is the shipped baseline. The pinned record snapshot in -/// this module's tests freezes its bytes; see [`crate::format`] for -/// the steps a format change requires. +/// Version 1 replaced the executed-session join and the burned set +/// with the committed sequence number. The pinned record snapshots in this +/// module's tests freeze both versions' bytes; see [`crate::format`] +/// for the steps a format change requires. impl Versioned for Boundary { - const VERSION: u16 = 0; + const VERSION: u16 = 1; } impl Record for Boundary { - type Previous = NoFormat; + type Previous = v0::Boundary; } -impl TryFrom for Boundary { +impl TryFrom for Boundary { type Error = &'static str; - fn try_from(none: NoFormat) -> Result { - match none {} + + /// A v0 record kept a per-universe join of session-start versions + /// instead of a sequence number, and the join yields no sequence + /// number, so the committed job converts with a floor of zero: + /// the session still resumes, and the floor refuses nothing. The + /// zero floor readmits nothing a previous life ran, because every + /// v0 record predates sequence numbers and only sessions that + /// carry one can start now. + fn try_from(old: v0::Boundary) -> Result { + Ok(Self { + network: old.network, + job: old.job.map(|job| Committed { + session: job.session, + seq: SessionSeq::from_unix_seconds(0).expect("zero is encodable"), + job: job.job, + successor: job.successor, + outcome: job.outcome, + }), + }) + } +} + +/// The version 0 record shapes, kept only to decode records written +/// before sessions carried sequence numbers. +pub mod v0 { + use super::{JobId, JobOutcome, Network, SessionId}; + use crate::format::{NoFormat, Record, Versioned}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use sush_common::wire::ExactBytes; + + #[derive(Deserialize, Serialize)] + pub struct Boundary { + pub network: Network, + pub burned: Burned, + #[serde(with = "version_bytes")] + pub executed: rumors::Version, + pub job: Option, + } + + /// The v0 burned set: the Bloom filter's 1024 bytes, decoded but + /// never read. + pub struct Burned(Box<[u8; 1024]>); + + impl Serialize for Burned { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.0[..]) + } + } + + impl<'de> Deserialize<'de> for Burned { + fn deserialize>(deserializer: D) -> Result { + Ok(Self(Box::new( + deserializer.deserialize_bytes(ExactBytes::<1024>)?, + ))) + } + } + + #[derive(Deserialize, Serialize)] + pub struct Committed { + pub session: SessionId, + pub job: JobId, + pub successor: JobId, + pub outcome: JobOutcome, + } + + impl Versioned for Boundary { + const VERSION: u16 = 0; + } + + impl Record for Boundary { + type Previous = NoFormat; + } + + impl TryFrom for Boundary { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } + } + + mod version_bytes { + use rumors::Version; + use serde::de::Error as _; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize( + version: &Version, + serializer: S, + ) -> Result { + serializer.serialize_bytes(&version.encode()) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result { + let bytes = >::deserialize(deserializer)?; + Version::decode(bytes.as_slice()).map_err(D::Error::custom) + } } } @@ -301,8 +317,6 @@ impl BoundaryStore { ..committed.clone() }), network: boundary.network, - burned: boundary.burned.clone(), - executed: boundary.executed.clone(), } }; if let Err(error) = guard.store(&format::encode(&updated)).await { @@ -338,7 +352,7 @@ mod test { use std::fs::create_dir; use camino::Utf8PathBuf; - use rumors::before::Clock; + use ciborium::ser::into_writer as into_cbor; use slog::Discard; use tempfile::TempDir; @@ -368,21 +382,16 @@ mod test { serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() } - /// A deterministic nonempty version: the seed party with two - /// events. - fn executed() -> Version { - let mut clock = Clock::seed(); - clock.tick(); - clock.tick().clone() + fn seq(seconds: u64) -> SessionSeq { + SessionSeq::from_unix_seconds(seconds).unwrap() } fn boundary() -> Boundary { Boundary { network: network(1), - burned: Bloom::new(), - executed: executed(), job: Some(Committed { session: SessionId::random(), + seq: seq(1_760_000_000), job: JobId::random(), successor: JobId::random(), outcome: JobOutcome::Committed, @@ -394,29 +403,39 @@ mod test { boundary.job.as_ref().expect("a committed job").job } - /// The record's bytes are on-disk format, frozen at version 0. If + /// The frozen v0 snapshot decodes and converts: the commitment + /// survives with a floor of zero. Nothing can regenerate the + /// snapshot; the code that wrote it is gone, and git keeps the + /// bytes. + #[test] + fn v0_record_converts() { + let bytes = std::fs::read("tests/output/boundary-record-v0.bin").expect("missing snapshot"); + let decoded: Boundary = format::decode(&bytes).unwrap(); + assert_eq!(decoded.network, network(1)); + let job = decoded.job.unwrap(); + assert_eq!(job.session, "abandon-ability".parse().unwrap()); + assert_eq!(job.successor, "able-about".parse().unwrap()); + assert_eq!(job.seq, seq(0)); + } + + /// The record's bytes are on-disk format, frozen at version 1. If /// this fails, STOP: do not re-pin. Copy the old shape into a /// frozen module and add a new version instead; see /// [`crate::format`]. #[test] - fn pin_record_format_v0() { + fn pin_record_format_v1() { let record = Boundary { network: network(1), - burned: { - let mut burned = Bloom::new(); - burned.insert(&network_key(network(2))); - burned - }, - executed: executed(), job: Some(Committed { session: "abandon-ability".parse().unwrap(), + seq: seq(1_760_000_000), job: "zoo-zero".parse().unwrap(), successor: "able-about".parse().unwrap(), outcome: JobOutcome::Committed, }), }; let bytes = format::encode(&record); - let path = "tests/output/boundary-record-v0.bin"; + let path = "tests/output/boundary-record-v1.bin"; if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { std::fs::write(path, &bytes).unwrap(); } else { @@ -425,12 +444,10 @@ mod test { } let decoded: Boundary = format::decode(&bytes).unwrap(); assert_eq!(decoded.network, record.network); - assert!(decoded.is_burned(network(2))); - assert_eq!(decoded.executed, record.executed); - assert_eq!( - decoded.job.unwrap().successor, - record.job.unwrap().successor - ); + let (decoded, expected) = (decoded.job.unwrap(), record.job.unwrap()); + assert_eq!(decoded.session, expected.session); + assert_eq!(decoded.seq, expected.seq); + assert_eq!(decoded.successor, expected.successor); } /// A record from a newer software version loads as untrusted, and @@ -442,7 +459,7 @@ mod test { #[derive(Serialize)] struct Envelope(u16, #[serde(with = "crate::format::cbor_bytes")] Vec); let mut bytes = Vec::new(); - into_cbor(&Envelope(1, b"from the future".to_vec()), &mut bytes).unwrap(); + into_cbor(&Envelope(2, b"from the future".to_vec()), &mut bytes).unwrap(); let scratch = Locker::new(&test_log(), slots.clone()).unwrap(); scratch.tenant(BOUNDARY).store(&bytes).await.unwrap(); drop(scratch); @@ -451,18 +468,6 @@ mod test { assert!(store.untrusted()); } - /// The burned set's keys are on-disk format: a change to the - /// network's serde shape would silently forget every burn, and a - /// forgotten burn admits a flip-flop instead of refusing it. If - /// this fails, STOP, and see the warning on [`crate::bloom`]. - #[test] - fn pin_network_keys() { - assert_eq!( - network_key(network(1)), - [0x50, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - #[tokio::test] async fn commitments_survive_restarts() { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); @@ -472,7 +477,6 @@ mod test { let (a, mut b) = (boundary(), boundary()); b.network = network(2); - b.burned = a.burned_for(b.network); first.advance(&a).await.unwrap(); first.advance(&b).await.unwrap(); drop(first); @@ -482,11 +486,9 @@ mod test { let recorded = next.boundary().unwrap(); assert_eq!(recorded.network, b.network); assert_eq!(job_of(&recorded), job_of(&b)); - assert!(recorded.is_burned(network(1))); - assert!(!recorded.is_burned(network(3))); - assert_eq!(recorded.executed, b.executed); let (recorded, expected) = (recorded.job.unwrap(), b.job.unwrap()); assert_eq!(recorded.session, expected.session); + assert_eq!(recorded.seq, expected.seq); assert_eq!(recorded.successor, expected.successor); assert!(matches!(recorded.outcome, JobOutcome::Committed)); } diff --git a/server/src/lib.rs b/server/src/lib.rs index 10588c3e..acdf3ab1 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -10,7 +10,6 @@ extern crate function_name; #[cfg(all(feature = "embedded", feature = "test-support"))] compile_error!("`test-support` must not be enabled for an embedded server"); -pub mod bloom; pub mod bookmark; pub mod boundary; pub mod certs; diff --git a/server/src/manager.rs b/server/src/manager.rs index ca0fa5b8..2cbfd495 100644 --- a/server/src/manager.rs +++ b/server/src/manager.rs @@ -32,7 +32,8 @@ use sush_common::authn::{ }; use sush_common::jobs::{ Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, - SessionSeq, SessionServerNonce, SessionSignerNonce, SessionSushNonce, SignedJob, + SessionPreimage, SessionSeq, SessionServerNonce, SessionSignerNonce, SessionSushNonce, + SignedJob, }; use sush_common::keys::{KeyError, KeyId, SshPublicKey}; use sush_common::targets::{Cubbies, SledHealth, SledVersion}; @@ -476,15 +477,21 @@ impl JobManager { ) -> Result<(), JobError> { // Verify that the session ID provided by the client is meant // for this instance of Sush: it must hash over our baseboard - // and current server nonce. + // and current server nonce. The preimage then travels instead + // of the id, so every peer derives for itself both the id and + // the sequence number the id commits to. let server_nonce = *self.session_server_nonce.lock().unwrap(); - let sush_nonce = SessionSushNonce::compute(client_nonce, server_nonce); - let computed = SessionId::compute(self.own_baseboard(), sush_nonce, signer_nonce, seq); - if computed != session_id { + let preimage = SessionPreimage { + baseboard: self.own_baseboard().clone(), + sush_nonce: SessionSushNonce::compute(client_nonce, server_nonce), + signer_nonce, + seq, + }; + if preimage.session_id() != session_id { return Err(JobError::InvalidSessionId); } - self.session_request(authn, SessionRequest::Start(session_id, seq)) + self.session_request(authn, SessionRequest::Start(preimage)) .await?; if wait { self.wait_for(self.wait_for_session(session_id)).await?; diff --git a/server/src/messages.rs b/server/src/messages.rs index 80bf12e0..d0e2b042 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -6,16 +6,14 @@ use chrono::{DateTime, Utc}; use ciborium::{Value, de::from_reader as from_cbor, ser::into_writer as into_cbor}; -use rumors::Version; use serde::{Deserialize, Serialize}; use sled_hardware_types::BaseboardId; -use thiserror::Error; use sush_api::JobStartParams; use sush_common::authn::SignedLogin; use sush_common::jobs::JobOutputState; use sush_common::jobs::{ - Access, JobId, ProcessError, SessionId, SessionSeq, SignedJob, SkipReason, + Access, JobId, ProcessError, SessionId, SessionPreimage, SignedJob, SkipReason, }; use sush_common::keys::KeyId; use sush_common::version::VersionInfo; @@ -215,7 +213,7 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum SessionRequest { - Start(SessionId, SessionSeq), + Start(SessionPreimage), Stop(SessionId), Skip(SessionId, JobId), AllowAttach(SessionId, KeyId, Access), @@ -242,7 +240,6 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum Event { Job(JobEvent), - Error(Error), Version(VersionInfo), } @@ -269,25 +266,6 @@ pub mod v0 { } } } - - #[derive(Clone, Debug, Deserialize, Eq, Error, PartialEq, Serialize)] - pub enum Error { - #[error( - "Concurrent sessions detected: \ - ours is {own_session}@{own_version}, \ - incoming is {incoming_session}@{incoming_version}" - )] - ConcurrentSessions { - own_session: SessionId, - own_version: Box, - incoming_session: SessionId, - incoming_version: Box, - }, - #[error("Sled re-entered a burned universe")] - UniverseFlipFlop, - #[error("Sled hopped into a session its record cannot order")] - SessionHop, - } } #[cfg(test)] @@ -299,7 +277,9 @@ mod wire_format { use sush_api::JobWait; use sush_common::authn::{ChallengeResponse, Nonce, RequestVerifier}; use sush_common::hash::hash; - use sush_common::jobs::{JobMode, JobStartRequest}; + use sush_common::jobs::{ + JobMode, JobStartRequest, SessionSeq, SessionSignerNonce, SessionSushNonce, + }; use sush_common::keys::{EncodedSignature, Signed}; use super::v0::*; @@ -375,9 +355,6 @@ mod wire_format { SkipReason::AlreadyHandled => "already-handled-skip-event", SkipReason::SessionEnded => "session-ended-skip-event", }, - Event::Error(Error::ConcurrentSessions { .. }) => "concurrent-sessions-error", - Event::Error(Error::SessionHop) => "session-hop-error", - Event::Error(Error::UniverseFlipFlop) => "universe-flip-flop-error", }, } } @@ -459,10 +436,12 @@ mod wire_format { vec![ Message::Request(Request::session( actor(), - SessionRequest::Start( - session, - SessionSeq::from_unix_seconds(1_760_000_000).unwrap(), - ), + SessionRequest::Start(SessionPreimage { + baseboard: baseboard.clone(), + sush_nonce: SessionSushNonce::from_be_bytes([0x11; 32]), + signer_nonce: SessionSignerNonce::from_be_bytes([0x22; 32]), + seq: SessionSeq::from_unix_seconds(1_760_000_000).unwrap(), + }), )), Message::Request(Request::session(actor(), SessionRequest::Stop(session))), Message::Request(Request::session( @@ -534,24 +513,13 @@ mod wire_format { )), ), Message::Event( - baseboard.clone(), + baseboard, Event::Job(JobEvent::Skipped( skip_job, skip_when, SkipReason::SessionEnded, )), ), - Message::Event( - baseboard.clone(), - Event::Error(Error::ConcurrentSessions { - own_session: long_session, - own_version: Box::new(Version::new()), - incoming_session: sid("zoo-zoo-zoo-zoo-zoo-zoo-zoo-zebra"), - incoming_version: Box::new(Version::new()), - }), - ), - Message::Event(baseboard.clone(), Event::Error(Error::SessionHop)), - Message::Event(baseboard, Event::Error(Error::UniverseFlipFlop)), ] } @@ -560,9 +528,12 @@ mod wire_format { /// variant cannot leave a stale snapshot behind. #[test] fn wire_formats() { - // The boundary module pins its record format in the same + // The boundary module pins its record formats in the same // directory. - let mut expected = BTreeSet::from(["boundary-record-v0.bin".to_string()]); + let mut expected = BTreeSet::from([ + "boundary-record-v0.bin".to_string(), + "boundary-record-v1.bin".to_string(), + ]); for message in samples() { let name = snapshot_name(&message); assert!( diff --git a/server/src/state.rs b/server/src/state.rs index e5a3a623..312572e4 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -23,14 +23,13 @@ use x509_cert::Certificate; use sush_api::JobStartParams; use sush_common::jobs::{ - Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, SignedJob, - SkipReason, VerifiedJob, + Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, SessionSeq, + SignedJob, SkipReason, VerifiedJob, }; use sush_common::keys::{KeyError, KeyId}; use sush_common::targets::Cubbies; use sush_common::version::VersionInfo; -use crate::bloom::Bloom; use crate::bookmark::SushBookmark; use crate::boundary::{Boundary, BoundaryStore, Committed, JobOutcome}; use crate::certs::Certificates; @@ -40,7 +39,7 @@ use crate::history::JobHistory; use crate::identities::Identities; use crate::job::SocketSender; use crate::locker::Locker; -use crate::messages::v0::{Error, Event, JobEvent, JobRequest, Message, Request, SessionRequest}; +use crate::messages::v0::{Event, JobEvent, JobRequest, Message, Request, SessionRequest}; use crate::messages::{VersionedMessage, VersionedMessage::*}; use crate::output::JobOutputDir; use crate::rack::Rack; @@ -66,18 +65,21 @@ pub struct QueuedJob { pub replayed: bool, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub enum SessionState { - Inactive { - /// Join of all observed session start requests. - starts: Version, - }, + #[default] + Inactive, Active { - /// Join of all observed session start requests. - starts: Version, + /// Where the active session falls in the signer-assigned + /// order. A start wins the slot only by exceeding this + /// sequence number, with ties broken by session id. + seq: SessionSeq, - /// When the active session started. - started: Version, + /// Whether this session's start arrived as live traffic + /// rather than replayed history. A session replayed onto a + /// sled with no boundary record may hold jobs a previous + /// life ran, so [`Past::screen`] refuses it. + live: bool, /// The active session. session: Box, @@ -91,18 +93,10 @@ pub enum SessionState { } impl SessionState { - fn starts(&self) -> Version { - use SessionState::*; - match self { - Inactive { starts } => starts.clone(), - Active { starts, .. } => starts.clone(), - } - } - pub fn session(&self) -> Option<&Session> { use SessionState::*; match self { - Inactive { .. } => None, + Inactive => None, Active { session, .. } => Some(session), } } @@ -128,111 +122,49 @@ impl SessionState { fn queued_jobs(&self) -> Option<&QueuedJobs> { use SessionState::*; match self { - Inactive { .. } => None, + Inactive => None, Active { queued_jobs, .. } => Some(queued_jobs), } } } -impl Default for SessionState { - fn default() -> Self { - Self::Inactive { - starts: Version::new(), - } - } -} - -/// Whether we have a job boundary we can rely on, and if so what the -/// execution floor is. -/// -/// The floor lives in memory only, since a floor written to disk would -/// reach the next life, which might join a different universe; and because -/// versions don't compare across universes, every session would look -/// concurrent with the floor, leading to a permanent lock-out. Instead, -/// we raise a fresh floor at each universe entry from messages already -/// there, so future sessions come to dominate it. +/// Whether we have a job boundary we can rely on. #[derive(Debug)] enum Commitment { - /// Tabula rasa. The floor is raised when we join a universe with - /// history. - Absent { floor: Option }, - - /// We have a valid boundary record. The floor is raised on re-entering - /// a universe, or by a session hop. - Trusted { - boundary: Box, - floor: Option, - }, + /// Tabula rasa: no life has committed to a job here. + Absent, + + /// We have a valid boundary record. + Trusted { boundary: Box }, /// The boundary record is corrupt; no jobs may run here. Untrusted, } impl Commitment { - /// Judge the boundary record at entry into a universe. On a flip-flop - /// (re-entering a burned universe), the error to report is also returned. - fn at_entry( - log: &Logger, - store: &BoundaryStore, - rumors: &GossipNetwork, - ) -> (Self, Option) { + /// Judge the boundary record at startup. + fn at_entry(store: &BoundaryStore) -> Self { if store.untrusted() { - return (Self::Untrusted, None); + return Self::Untrusted; } - - let network = rumors.network(); - let entry = rumors.snapshot().latest().clone(); match store.boundary() { - Some(boundary) if boundary.network != network && boundary.is_burned(network) => { - warn!(log, "re-entered a burned universe"; "committed" => ?boundary.job); - ( - Self::Trusted { - boundary: Box::new(boundary), - floor: Some(entry), - }, - Some(Error::UniverseFlipFlop), - ) - } - Some(boundary) => ( - Self::Trusted { - boundary: Box::new(boundary), - floor: None, - }, - None, - ), - None if entry != Version::new() => { - warn!(log, "no boundary record, but this universe has history"); - (Self::Absent { floor: Some(entry) }, None) - } - None => (Self::Absent { floor: None }, None), - } - } - - /// Raise the execution floor. An untrusted commitment refuses - /// every job already, so it carries no floor to raise. - fn raise(&mut self, to: Version) { - match self { - Self::Absent { floor } | Self::Trusted { floor, .. } => *floor = Some(to), - Self::Untrusted => {} + Some(boundary) => Self::Trusted { + boundary: Box::new(boundary), + }, + None => Self::Absent, } } } -/// What [`Past::screen`] decides about a live job of the active -/// session. +/// What [`Past::screen`] decides about a job of the active session. enum Admission { /// Execute normally. Admit, - /// The session does not start above the execution floor, - /// so skip the job. + /// The session does not pass the execution floor, so skip the + /// job. Refuse, - /// We have a boundary record, but it's incommensurate with the - /// current session. So "hop" into the new session, and raise the - /// execution floor. - Hop, - /// If we cannot trust our boundary record, we must not admit /// new jobs. Untrusted, @@ -252,10 +184,6 @@ pub struct Past { /// life computes its own. birth: Version, - /// The most recently seen message version. Initialized to `birth`, - /// then joined at every state machine update. - frontier: Version, - /// What job, if any, this sled committed to executing. Durable /// across restarts. committed: Commitment, @@ -277,16 +205,9 @@ pub struct Past { adjudicated: Option, } -/// Whether `version` strictly dominates `mark`. Equal and concurrent -/// versions do not. -fn dominates(version: &Version, mark: &Version) -> bool { - version > mark -} - impl Past { fn new(birth: Version, committed: Commitment) -> Self { Past { - frontier: birth.clone(), birth, committed, zombies: BTreeSet::new(), @@ -295,42 +216,29 @@ impl Past { } } - fn observe(&mut self, version: &Version) { - self.frontier |= version; - } - - /// Decide whether the active session, identified by its id and - /// the version of its start message, may run a live job here. - fn screen(&self, network: Network, session: SessionId, started: &Version) -> Admission { + /// Decide whether the active session may run a job here. The + /// signer's sequence numbers order sessions totally and across + /// universes, so the floor needs no universe scoping: a session + /// at or below the committed sequence number may already have run + /// jobs on this sled, and only the committed session itself + /// resumes. + fn screen(&self, session: SessionId, seq: SessionSeq, live: bool) -> Admission { match &self.committed { // Refuse all jobs if our boundary is untrusted. Commitment::Untrusted => Admission::Untrusted, - // We must also refuse anything below our floor, - // since we cannot tell replay from re-run there. - Commitment::Absent { floor: Some(floor) } - | Commitment::Trusted { - floor: Some(floor), .. - } if !dominates(started, floor) => Admission::Refuse, - - // Allow jobs in the absence of any previous commitment. - Commitment::Absent { .. } => Admission::Admit, - - // Jobs cannot cross universes, and anything a previous life - // ran in this universe is below the floor checked above. - Commitment::Trusted { boundary, .. } => { - if boundary.network != network { - Admission::Admit - } else if let Some(job) = &boundary.job - && job.session == session - { - Admission::Admit - } else if dominates(started, &boundary.executed) { - Admission::Admit - } else { - Admission::Hop - } - } + Commitment::Trusted { boundary } => match &boundary.job { + Some(committed) if committed.session == session => Admission::Admit, + Some(committed) if seq > committed.seq => Admission::Admit, + Some(_) => Admission::Refuse, + None if live => Admission::Admit, + None => Admission::Refuse, + }, + + // With no record we cannot tell replay from re-run, so + // only a session that started as live traffic runs jobs. + Commitment::Absent if live => Admission::Admit, + Commitment::Absent => Admission::Refuse, } } @@ -338,17 +246,16 @@ impl Past { fn commit_to( &self, network: Network, - started: &Version, session_id: SessionId, + seq: SessionSeq, job_id: JobId, successor: JobId, ) -> Boundary { Boundary { network, - burned: self.burned_for(network), - executed: self.executed_for(network, started), job: Some(Committed { session: session_id.to_owned(), + seq, job: job_id, successor, outcome: JobOutcome::Committed, @@ -356,25 +263,6 @@ impl Past { } } - /// The burned set for a record committed in `network`; see - /// [`Boundary::burned_for`]. - fn burned_for(&self, network: Network) -> Bloom { - match &self.committed { - Commitment::Trusted { boundary, .. } => boundary.burned_for(network), - _ => Bloom::default(), - } - } - - /// The executed-session join for a record committed in `network` - /// under a session started at `started`; see - /// [`Boundary::executed_for`]. - fn executed_for(&self, network: Network, started: &Version) -> Version { - match &self.committed { - Commitment::Trusted { boundary, .. } => boundary.executed_for(network, started), - _ => started.clone(), - } - } - /// Whether a message at `version` is live traffic rather than /// replayed history; see [`Past::birth`]. Equality admits the /// birth announcement itself. @@ -658,7 +546,6 @@ impl State { incoming_version: &Version, message: &Arc, ) -> Vec { - self.past.observe(incoming_version); match message.as_ref() { V0(Message::Request(request)) => { let actor = request.actor(); @@ -712,86 +599,59 @@ impl State { let mut outputs = Vec::new(); match request { - SessionRequest::Start(session_id, _seq) => match &mut self.session { - // Re-announcement of active session; absorb and ignore. - Active { - starts, session, .. - } if session.session_id() == *session_id => { - *starts |= incoming_version.clone(); - info!(log, "duplicate session start"; "session_id" => %session_id); - } + SessionRequest::Start(preimage) => { + let session_id = preimage.session_id(); + match &self.session { + // Re-announcement of the active session; ignore. + Active { session, .. } if session.session_id() == session_id => { + info!(log, "duplicate session start"; "session_id" => %session_id); + } - // If the incoming new session is in the *strict - // causal future* of everything we have observed, - // adopt it regardless of our current state. Every - // other peer will make the same decision, so it - // will come to dominate the network entirely. - Inactive { starts } | Active { starts, .. } if *incoming_version > *starts => { - info!(log, "session started"; "session_id" => %session_id, "actor" => %actor); - let mut session = Session::started(*session_id, actor.clone()); - if let Commitment::Trusted { boundary, .. } = &self.past.committed - && let Some(committed) = boundary - .job - .as_ref() - .filter(|committed| committed.session == *session_id) + // A start wins the slot only by exceeding the + // active session in the signer-assigned order, + // with ties broken by id. This one does not; + // ignore it. + Active { seq, session, .. } + if (preimage.seq, session_id) <= (*seq, session.session_id()) => { info!( - log, "resuming committed session at its successor"; + log, "stale session start"; "session_id" => %session_id, - "successor" => %committed.successor, + "seq" => %preimage.seq, + "active_seq" => %seq, ); - session.resume_at(committed.successor); } - outputs.extend(self.skip_queued_jobs(now)); - self.session = Active { - starts: self.session.starts() | incoming_version.clone(), - started: incoming_version.clone(), - session: Box::new(session), - queued_jobs: QueuedJobs::new(), - attach_grants: BTreeMap::new(), - }; - } - - // When the incoming session is concurrent, we can't - // establish which is older, so we have to kill both - // sessions. Every peer will make this same decision, - // so as a partition resolves, both sessions will be - // killed everywhere. - Active { - starts, - started, - session, - .. - } if incoming_version.partial_cmp(starts).is_none() => { - let starts = starts.to_owned(); - let error = Error::ConcurrentSessions { - own_session: session.session_id(), - own_version: Box::new(started.clone()), - incoming_session: *session_id, - incoming_version: Box::new(incoming_version.clone()), - }; - outputs.extend(self.skip_queued_jobs(now)); - outputs.push(Output::Gossip(Message::Event( - self.rack.own_baseboard().to_owned(), - Event::Error(error), - ))); - self.session = Inactive { - starts: starts | incoming_version.clone(), - }; - } - // Stale or concurrent session while inactive; OBE. - // Absorb and ignore. - Inactive { starts, .. } | Active { starts, .. } => { - info!( - log, "stale session start"; - "session_id" => %session_id, - "starts" => %starts, - "incoming_version" => %incoming_version, - ); - *starts |= incoming_version.clone(); + // This one does. Every peer applies the same + // total order, so every peer converges on the + // same session. + Inactive | Active { .. } => { + info!(log, "session started"; "session_id" => %session_id, "actor" => %actor); + let mut session = Session::started(session_id, actor.clone()); + if let Commitment::Trusted { boundary } = &self.past.committed + && let Some(committed) = boundary + .job + .as_ref() + .filter(|committed| committed.session == session_id) + { + info!( + log, "resuming committed session at its successor"; + "session_id" => %session_id, + "successor" => %committed.successor, + ); + session.resume_at(committed.successor); + } + outputs.extend(self.skip_queued_jobs(now)); + self.session = Active { + seq: preimage.seq, + live: self.past.is_live(incoming_version), + session: Box::new(session), + queued_jobs: QueuedJobs::new(), + attach_grants: BTreeMap::new(), + }; + } } - }, + } SessionRequest::Stop(session_id) => { // Any session stop request for a session that isn't // ours is silently ignored. Even in the case of @@ -803,9 +663,7 @@ impl State { // ordering above when handling session-start. Together, // this means that we can't fail to react to a stop for // the active session. - if let Active { - starts, session, .. - } = &self.session + if let Active { session, .. } = &self.session && session.session_id() == *session_id { if session.started_by() != Some(actor) { @@ -819,9 +677,7 @@ impl State { "session_id" => %session_id, "actor" => %actor, ); outputs.extend(self.skip_queued_jobs(now)); - self.session = Inactive { - starts: starts.clone(), - }; + self.session = Inactive; } } } @@ -1125,9 +981,6 @@ impl State { outputs.push(Output::Ran(status)); } } - Event::Error(error) => { - error!(log, "session error"; "error" => %error); - } Event::Version(info) => { if self.rack.known_version(baseboard_id.clone(), info.clone()) { info!(log, "sled version"; "version" => %info); @@ -1234,7 +1087,8 @@ impl State { let mut outputs = Vec::new(); let SessionState::Active { session, - started, + seq, + live, queued_jobs, .. } = &mut self.session @@ -1261,20 +1115,7 @@ impl State { break; } - let admission = self.past.screen(network, session.session_id(), started); - if matches!(admission, Admission::Hop) { - warn!( - log, "hopping sessions, raising the execution floor"; - "session_id" => %session.session_id(), "started" => ?started, - ); - self.past.committed.raise(self.past.frontier.clone()); - outputs.push(Output::Gossip(Message::Event( - self.rack.own_baseboard().to_owned(), - Event::Error(Error::SessionHop), - ))); - continue; - } - + let admission = self.past.screen(session.session_id(), *seq, *live); let QueuedJob { job: request, params, @@ -1343,8 +1184,8 @@ impl State { attachment, boundary: self.past.commit_to( network, - started, session.session_id(), + *seq, job_id, session .session_id() @@ -1440,18 +1281,15 @@ fn drain_ready( } /// Enter a universe: judge the boundary record, announce this build, -/// and mark the birth. The announce is this life's first send, and a -/// flip-flop report follows it. The birth is the version of whichever -/// went out last. +/// and mark the birth at the announce's version. fn enter_universe( - log: &Logger, own_baseboard: BaseboardId, anchors: &[Certificate], store: &BoundaryStore, rumors: &GossipNetwork, ) -> Result { - let (committed, report) = Commitment::at_entry(log, store, rumors); - let mut birth = gossip( + let committed = Commitment::at_entry(store); + let birth = gossip( rumors, Message::Event( own_baseboard.clone(), @@ -1459,12 +1297,6 @@ fn enter_universe( ) .into(), ); - if let Some(report) = report { - birth = gossip( - rumors, - Message::Event(own_baseboard.clone(), Event::Error(report)).into(), - ); - } State::new(own_baseboard, anchors, Past::new(birth, committed)) } @@ -1545,7 +1377,7 @@ impl StateManager { let mut initial_state = State::new( own_baseboard.clone(), anchors, - Past::new(Version::new(), Commitment::Absent { floor: None }), + Past::new(Version::new(), Commitment::Absent), )?; initial_state .rack @@ -1577,14 +1409,9 @@ impl StateManager { // processed, so the birth mark splits this life's // traffic from replayed history. if let Some((rumors, _)) = &network { - let entered = enter_universe( - &log, - own_baseboard.clone(), - &anchors, - &boundary_store, - rumors, - ) - .expect("trust anchors validated at startup"); + let entered = + enter_universe(own_baseboard.clone(), &anchors, &boundary_store, rumors) + .expect("trust anchors validated at startup"); tx_state.send_modify(|state| { *state = entered; state.rack.set_cubbies(cubbies.borrow_and_update().clone()); @@ -1757,7 +1584,6 @@ impl StateManager { doomed.insert(job_id); } let entered = enter_universe( - &log, own_baseboard.clone(), &anchors, &boundary_store, @@ -1797,10 +1623,12 @@ impl StateManager { mod test { use std::time::Duration; - use rumors::before::Clock; use x509_cert::time::Validity; - use sush_common::jobs::{JobMode, JobOutputState, JobStartRequest, SessionSeq}; + use sush_common::jobs::{ + JobMode, JobOutputState, JobStartRequest, SessionPreimage, SessionSignerNonce, + SessionSushNonce, + }; use sush_common::keys::{EphemeralKey, KeyType, Signer as _}; use sush_common::targets::Target; @@ -1869,10 +1697,9 @@ mod test { fn committed_boundary(job: JobId) -> Boundary { Boundary { network: network(1), - burned: Bloom::new(), - executed: Version::new(), job: Some(Committed { session: SessionId::random(), + seq: seq(10), job, successor: JobId::random(), outcome: JobOutcome::Committed, @@ -1890,190 +1717,91 @@ mod test { ) } - fn past(committed: Option, floor: Option) -> Past { + fn past(committed: Option) -> Past { let committed = match committed { Some(boundary) => Commitment::Trusted { boundary: Box::new(boundary), - floor, }, - None => Commitment::Absent { floor }, + None => Commitment::Absent, }; Past::new(Version::new(), committed) } - /// Versions with known causal relations: `older < started < - /// newer`, with `concurrent` unordered against all three. - struct Versions { - older: Version, - started: Version, - newer: Version, - concurrent: Version, - } - fn seq(seconds: u64) -> SessionSeq { SessionSeq::from_unix_seconds(seconds).unwrap() } - fn versions() -> Versions { - let mut a = Clock::seed(); - let mut b = a.fork(); - Versions { - older: a.tick().clone(), - started: a.tick().clone(), - newer: a.tick().clone(), - concurrent: b.tick().clone(), + /// A preimage for a fresh session authorized at `seconds`; tests + /// reference the session by the derived id. + fn preimage(seconds: u64) -> SessionPreimage { + SessionPreimage { + baseboard: baseboard(), + sush_nonce: SessionSushNonce::random(), + signer_nonce: SessionSignerNonce::random(), + seq: seq(seconds), } } - /// The admission rules: a sled with no record admits, a record - /// from a foreign universe admits, the committed session admits - /// outright, a session started strictly above the executed join - /// admits, and everything else hops. + /// The admission rules: the committed session resumes even from + /// replay, a session above the committed sequence number admits, + /// and everything at or below it refuses, in any universe. #[test] fn admission_rules() { - let Versions { - older, - started, - newer, - concurrent, - } = versions(); let session = SessionId::random(); - let job = JobId::random(); let committed = Boundary { network: network(1), - burned: Bloom::new(), - executed: started.clone(), job: Some(Committed { session, - job, + seq: seq(10), + job: JobId::random(), successor: JobId::random(), outcome: JobOutcome::Committed, }), }; + let past = past(Some(committed)); - let recordless = past(None, None); - assert!(matches!( - recordless.screen(network(1), session, &started), - Admission::Admit - )); - - let past = past(Some(committed), None); - assert!(matches!( - past.screen(network(2), session, &started), - Admission::Admit - )); - // The committed session admits outright: it resumes at the - // stored successor when it activates, and chain position - // keeps every earlier job from popping. assert!(matches!( - past.screen(network(1), session, &started), + past.screen(session, seq(10), false), Admission::Admit )); assert!(matches!( - past.screen(network(1), SessionId::random(), &newer), + past.screen(SessionId::random(), seq(11), false), Admission::Admit )); - for unordered in [&older, &started, &concurrent] { + for at_or_below in [seq(10), seq(9)] { assert!(matches!( - past.screen(network(1), SessionId::random(), unordered), - Admission::Hop + past.screen(SessionId::random(), at_or_below, true), + Admission::Refuse )); } } - /// A floor refuses every session not started strictly above it, - /// in every universe and under either commitment that carries one: - /// the floor belongs to this life, not to any record. + /// With no record, a live session admits and a replayed one + /// refuses: a sled that has never written a boundary cannot tell + /// replay from re-run. #[test] - fn floors_refuse_below() { - let Versions { - older: below, - started: floor, - newer: above, - concurrent, - } = versions(); - let at: Version = floor.clone(); - let committed = Boundary { - network: network(1), - burned: Bloom::new(), - executed: floor.clone(), - job: None, - }; - - for past in [ - past(None, Some(floor.clone())), - past(Some(committed), Some(floor.clone())), - ] { - for started in [&at, &below, &concurrent] { - for net in [network(1), network(2)] { - assert!(matches!( - past.screen(net, SessionId::random(), started), - Admission::Refuse - )); - } - } - assert!(matches!( - past.screen(network(1), SessionId::random(), &above), - Admission::Admit - )); - } - } - - /// A hop raises the floor to the frontier, so the hopped session - /// screens as a refusal from then on: one hop report per session. - /// A session started strictly above the frontier admits again. - #[test] - fn raised_floor_refuses_hopped_session() { - let mut a = Clock::seed(); - let mut b = a.fork(); - let started = a.tick().clone(); - let executed = b.tick().clone(); - let above = started.clone() | executed.clone() | a.tick().clone(); - let session = SessionId::random(); - let committed = Boundary { - network: network(1), - burned: Bloom::new(), - executed: executed.clone(), - job: None, - }; - - let mut past = past(Some(committed), None); - past.observe(&started); - past.observe(&executed); - assert_eq!(past.frontier, &started | executed.clone()); + fn recordless_admission_is_live_only() { + let past = past(None); assert!(matches!( - past.screen(network(1), session, &started), - Admission::Hop + past.screen(SessionId::random(), seq(1), true), + Admission::Admit )); - - past.committed.raise(past.frontier.clone()); assert!(matches!( - past.screen(network(1), session, &started), + past.screen(SessionId::random(), seq(1), false), Admission::Refuse )); - assert!(matches!( - past.screen(network(1), session, &above), - Admission::Admit - )); } - /// An untrusted record admits nothing in any universe, and a - /// raise leaves it untrusted. + /// An untrusted record admits nothing, live or replayed. #[test] fn untrusted_admits_nothing() { - let Versions { started, .. } = versions(); - let mut past = Past::new(Version::new(), Commitment::Untrusted); - for net in [network(1), network(2)] { + let past = Past::new(Version::new(), Commitment::Untrusted); + for live in [true, false] { assert!(matches!( - past.screen(net, SessionId::random(), &started), + past.screen(SessionId::random(), seq(1), live), Admission::Untrusted )); } - past.committed.raise(started.clone()); - assert!(matches!( - past.screen(network(1), SessionId::random(), &started), - Admission::Untrusted - )); } /// Settle declares a zombie interrupted exactly once. @@ -2082,7 +1810,7 @@ mod test { let log = discard(); let now = Utc::now(); let zombie = JobId::random(); - let mut state = state(past(None, None)); + let mut state = state(past(None)); state.past.zombies.insert(zombie); start_job(&mut state, zombie, now); @@ -2101,7 +1829,7 @@ mod test { let log = discard(); let now = Utc::now(); let job = JobId::random(); - let mut state = state(past(Some(committed_boundary(job)), None)); + let mut state = state(past(Some(committed_boundary(job)))); assert!(state.settle(&log, now, Some(network(2))).is_empty()); let outputs = state.settle(&log, now, Some(network(1))); @@ -2118,7 +1846,7 @@ mod test { let now = Utc::now(); let job = JobId::random(); - let mut witnessed = state(past(Some(committed_boundary(job)), None)); + let mut witnessed = state(past(Some(committed_boundary(job)))); start_job(&mut witnessed, job, now); assert!(witnessed.settle(&log, now, Some(network(1))).is_empty()); assert_eq!(witnessed.past.adjudicated, Some(job)); @@ -2148,7 +1876,7 @@ mod test { } let store = BoundaryStore::new(&log, &Locker::null()); store.load().await; - let state = enter_universe(&log, baseboard(), anchors, &store, &rumors).unwrap(); + let state = enter_universe(baseboard(), anchors, &store, &rumors).unwrap(); let messages = rumors.causal_messages(); Sim { log, @@ -2166,8 +1894,7 @@ mod test { let rumors: GossipNetwork = Seed::grow(&self.log, &Locker::null()).await.into_rumors(); let store = BoundaryStore::new(&self.log, &Locker::null()); store.load().await; - self.state = - enter_universe(&self.log, baseboard(), &self.anchors, &store, &rumors).unwrap(); + self.state = enter_universe(baseboard(), &self.anchors, &store, &rumors).unwrap(); self.messages = rumors.causal_messages(); self.rumors = rumors; } @@ -2226,10 +1953,11 @@ mod test { let root = root(); let anchors = vec![root.cert().clone()]; let mut sim = Sim::new(&anchors, Vec::new()).await; - let session_id = SessionId::random(); + let preimage = preimage(1); + let session_id = preimage.session_id(); sim.request(Request::session( root.key_id().clone(), - SessionRequest::Start(session_id, seq(1)), + SessionRequest::Start(preimage), )); (root, sim, session_id, Session::new(session_id)) } @@ -2244,11 +1972,12 @@ mod test { let actor = root.key_id().clone(); let mut sim = Sim::new(&anchors, Vec::new()).await; - let session_id = SessionId::random(); + let preimage = preimage(1); + let session_id = preimage.session_id(); let session = Session::new(session_id); sim.request(Request::session( actor.clone(), - SessionRequest::Start(session_id, seq(1)), + SessionRequest::Start(preimage), )); assert!(sim.state.session.session().is_some()); @@ -2293,39 +2022,29 @@ mod test { assert!(sim.state.session.session().is_none()); } - /// A session the boundary record cannot order hops: the floor - /// rises, one report goes out, and the session's jobs are skipped - /// below the floor rather than run. A later job in the same - /// session reports no second hop. + /// A session at or below the committed sequence number is + /// refused: its jobs skip below the floor rather than run, and + /// the chain still advances so the session is not wedged. #[tokio::test] - async fn hop_through_step() { + async fn floor_refuses_through_step() { let (mut root, mut sim, session_id, mut session) = sim_with_session().await; let actor = root.key_id().clone(); - // A boundary record whose executed join the session start - // does not dominate: the current frontier, which includes - // the start itself. + // A committed boundary the active session, at sequence + // number 1, does not exceed. sim.state.past.committed = Commitment::Trusted { boundary: Box::new(Boundary { network: sim.rumors.network(), - burned: Bloom::new(), - executed: sim.rumors.snapshot().latest().clone(), - job: None, + job: Some(Committed { + session: SessionId::random(), + seq: seq(10), + job: JobId::random(), + successor: JobId::random(), + outcome: JobOutcome::Committed, + }), }), - floor: None, }; - let hops = |outputs: &[Output]| { - outputs - .iter() - .filter(|o| { - matches!( - o, - Output::Gossip(Message::Event(_, Event::Error(Error::SessionHop))) - ) - }) - .count() - }; let skipped = |outputs: &[Output], job_id: JobId| { outputs.iter().any(|o| { matches!( @@ -2344,7 +2063,6 @@ mod test { actor.clone(), JobRequest::Start(signed.clone(), JobStartParams::default()), )); - assert_eq!(hops(&outputs), 1); assert!(!outputs.iter().any(|o| matches!(o, Output::Run(_)))); assert!(skipped(&outputs, job_id)); @@ -2355,7 +2073,6 @@ mod test { actor, JobRequest::Start(next, JobStartParams::default()), )); - assert_eq!(hops(&outputs), 0); assert!(skipped(&outputs, next_id)); } @@ -2416,14 +2133,15 @@ mod test { let mut root = root(); let anchors = vec![root.cert().clone()]; let actor = root.key_id().clone(); - let session_id = SessionId::random(); + let preimage = preimage(1); + let session_id = preimage.session_id(); let session = Session::new(session_id); let job_id = session.next_job_id(); let signed = signed_job(&mut root, job_id, session_id, "true").await; let history = vec![ Message::Request(Request::session( actor.clone(), - SessionRequest::Start(session_id, seq(1)), + SessionRequest::Start(preimage), )), Message::Request(Request::job( actor, @@ -2457,7 +2175,6 @@ mod test { boundary.network = home; sim.state.past.committed = Commitment::Trusted { boundary: Box::new(boundary), - floor: None, }; }; diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index b552d4ca..5640c314 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -25,8 +25,8 @@ use chrono::Utc; use sush_api::{JobStartParams, JobWait}; use sush_common::jobs::{ JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, - SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, - SkipReason, + SessionClientNonce, SessionId, SessionPreimage, SessionSeq, SessionSignerNonce, + SessionSushNonce, SignedJob, SkipReason, }; use sush_common::keys::{EphemeralKey, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, SledHealth, SledId, Target}; @@ -232,7 +232,7 @@ async fn jobs_gossip_between_sleds() { #[named] #[tokio::test] -async fn concurrent_sessions_annihilate() { +async fn partitioned_sessions_converge_on_higher_seq() { let (_tmp, dir) = pki("sush-concurrent-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); @@ -258,7 +258,8 @@ async fn concurrent_sessions_annihilate() { sees(&b, &a).await; // Partitioned, each sled starts its own session. Neither start - // can hear the other, so the two versions are concurrent. + // can hear the other, but the signer-assigned order still ranks + // them. a.peers.send(BTreeSet::new()).unwrap(); b.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; @@ -291,7 +292,7 @@ async fn concurrent_sessions_annihilate() { .unwrap(); // A job on A sits queued behind a hole in the chain, so it can - // never run before the sessions annihilate. + // never run before B's session supersedes A's. let hole_id = session.next_job_id(); let hole = sign_job_for(&mut root, hole_id, session_a, "true", &a.baseboard).await; session.job_started(hole); @@ -309,14 +310,23 @@ async fn concurrent_sessions_annihilate() { }) .await; - // The partition heals. Each sled hears a session start concurrent - // with its own: both sessions are killed everywhere, and the - // queued job is skipped rather than left queued forever. + // The partition heals. B's session carries the higher sequence + // number, so every sled converges on it, and A's queued job is + // skipped rather than left queued forever. a.peers.send(BTreeSet::from([b.addr])).unwrap(); b.peers.send(BTreeSet::from([a.addr])).unwrap(); - eventually("the sessions annihilate", 120, async || { - a.mgr.session(&authn_a).is_none() && b.mgr.session(&authn_b).is_none() - }) + eventually( + "the higher sequence number wins everywhere", + 120, + async || { + a.mgr + .session(&authn_a) + .is_some_and(|s| s.session_id() == session_b) + && b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session_b) + }, + ) .await; eventually("the queued job is skipped", 60, async || { a.mgr.job_status(&authn_a, &job_id).await.is_ok_and(|map| { @@ -409,7 +419,7 @@ async fn rejoining_replays_without_reexecuting() { client_nonce, signer_nonce: successor_nonce, seq, - } = ceremony(&a.mgr, 1); + } = ceremony(&a.mgr, 2); let fresh = Session::new(successor); a.mgr .session_start( @@ -680,15 +690,17 @@ struct Ceremony { /// nonce, so earlier ceremonies against that sled go stale. fn ceremony(mgr: &JobManager, seq: u64) -> Ceremony { let client_nonce = SessionClientNonce::random(); - let signer_nonce = SessionSignerNonce::random(); - let sush_nonce = SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()); - let seq = SessionSeq::from_unix_seconds(seq).unwrap(); - let session_id = SessionId::compute(mgr.own_baseboard(), sush_nonce, signer_nonce, seq); + let preimage = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce: SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()), + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_seconds(seq).unwrap(), + }; Ceremony { - session_id, + session_id: preimage.session_id(), client_nonce, - signer_nonce, - seq, + signer_nonce: preimage.signer_nonce, + seq: preimage.seq, } } @@ -869,7 +881,7 @@ async fn lost_suffix_never_reruns() { client_nonce, signer_nonce, seq, - } = ceremony(&a.mgr, 1); + } = ceremony(&a.mgr, 2); let session2 = Session::new(session2_id); a.mgr .session_start(&authn_a, session2_id, client_nonce, signer_nonce, seq, true) @@ -1064,7 +1076,7 @@ async fn session_resumes_at_stored_successor() { #[named] #[tokio::test] -async fn universe_flip_flop_raises_floor() { +async fn seq_floor_crosses_universes() { let (_tmp, dir) = pki("sush-flipflop-", 3); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); @@ -1087,7 +1099,7 @@ async fn universe_flip_flop_raises_floor() { client_nonce, signer_nonce, seq, - } = ceremony(&d.mgr, 1); + } = ceremony(&d.mgr, 2); let mut session2 = Session::new(session2_id); d.mgr .session_start(&authn_d, session2_id, client_nonce, signer_nonce, seq, true) @@ -1191,9 +1203,10 @@ async fn universe_flip_flop_raises_floor() { }) .await; - // X dies again and flip-flops back to A's universe. Its record - // burned that universe, so X raises its floor: session one's next - // job is refused, and nothing re-runs. + // X dies again and returns to A's universe. Its record holds + // session two's sequence number, which session one does not + // exceed, even from another universe: session one's next job is + // refused, and nothing re-runs. x_shutdown.cancel(); drop(x); d.peers.send(BTreeSet::new()).unwrap(); @@ -1231,14 +1244,14 @@ async fn universe_flip_flop_raises_floor() { }) .await; - // A witnessed the refusal, so a session started now begins above - // X's floor, and serves X again. + // A session with a sequence number above the committed one serves + // X again. let Ceremony { session_id: session3_id, client_nonce, signer_nonce, seq, - } = ceremony(&a.mgr, 1); + } = ceremony(&a.mgr, 3); let session3 = Session::new(session3_id); a.mgr .session_start(&authn_a, session3_id, client_nonce, signer_nonce, seq, true) diff --git a/server/tests/output/boundary-record-v1.bin b/server/tests/output/boundary-record-v1.bin new file mode 100644 index 0000000000000000000000000000000000000000..e2ea3269242183078530a6398d639b2cd566eb5f GIT binary patch literal 215 zcmZo-jJUigJukJSJijPAfRO@EiTT?&x=q%mBipNCKsm`;#ZaNyn+E_ sD1KE8_W%E87MCU`11-)kBA}K*fH^(Cv?Mt{H#O5aKQ}kCq$D*30LIoJwg3PC literal 0 HcmV?d00001 diff --git a/server/tests/output/concurrent-sessions-error.bin b/server/tests/output/concurrent-sessions-error.bin deleted file mode 100644 index f26b5582b0d836b8b4b4e62637119a5cdb1ea1db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 221 zcmZ3O6lSn6)wL`&ucT>Fc0poMNqk;uZc=Jdwxyx5t^o)bTILp~7G);pz+{|)d`*mu zj0_BnL0XH7@{1N0Ip^mkmlhQPjR{UIF3!x)D_)YFU!E6V4B(-k0|0je2d4l4 diff --git a/server/tests/output/universe-flip-flop-error.bin b/server/tests/output/universe-flip-flop-error.bin deleted file mode 100644 index 2b752144..00000000 --- a/server/tests/output/universe-flip-flop-error.bin +++ /dev/null @@ -1 +0,0 @@ -bV0eEventkpart_numberk913-0000019mserial_numberkBRM42220030eErrorpUniverseFlipFlop \ No newline at end of file diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index 177aa09c..8c6b503e 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -27,7 +27,8 @@ use sush_client::context::Authz; use sush_common::authn::{Challenge, ChallengeResponse, Credentials, Identity, Nonce, RequestKey}; use sush_common::jobs::{ Access, JobId, JobLimits, JobMode, JobOutputState, JobOutputStream::*, JobStartRequest, - JobStatus, ProcessError, Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, + JobStatus, ProcessError, Session, SessionId, SessionPreimage, SessionSeq, SessionSignerNonce, + SignedJob, SkipReason, }; use sush_common::keys::{EphemeralKey, KeyError, KeyType, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; @@ -311,7 +312,7 @@ async fn job_session_enforced() { /// leaves it alone, and so does activation. #[named] #[tokio::test] -async fn session_nonce_rotates() { +async fn fetch_rotates_server_nonce() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; @@ -344,21 +345,20 @@ async fn session_nonce_rotates() { .await .unwrap(); - // Activation leaves it alone too: a second grant over the same + // Activation leaves it alone too: a later grant over the same // sush nonce verifies and supersedes. - let second_signer_nonce = SessionSignerNonce::random(); - let second = SessionId::compute( - mgr.own_baseboard(), - ceremony.sush_nonce, - second_signer_nonce, - ceremony.seq, - ); + let second = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce: ceremony.sush_nonce, + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_seconds(2).unwrap(), + }; mgr.session_start( &authn, - second, + second.session_id(), ceremony.client_nonce, - second_signer_nonce, - ceremony.seq, + second.signer_nonce, + second.seq, true, ) .await @@ -366,20 +366,19 @@ async fn session_nonce_rotates() { // A fetch rotates: everything built over the old nonce is stale. mgr.fresh_server_nonce(); - let stale_signer_nonce = SessionSignerNonce::random(); - let stale = SessionId::compute( - mgr.own_baseboard(), - ceremony.sush_nonce, - stale_signer_nonce, - ceremony.seq, - ); + let stale = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce: ceremony.sush_nonce, + signer_nonce: SessionSignerNonce::random(), + seq: ceremony.seq, + }; assert!(matches!( mgr.session_start( &authn, - stale, + stale.session_id(), ceremony.client_nonce, - stale_signer_nonce, - ceremony.seq, + stale.signer_nonce, + stale.seq, true, ) .await, diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 7be2e7cd..625d92a0 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -33,8 +33,8 @@ use sush_client::{Client, ResponseValue}; use sush_common::authn::{Challenge, ChallengeResponse, Credentials, Identity, Nonce, RequestKey}; use sush_common::codephrases::Codephrase; use sush_common::jobs::{ - JobId, JobMode, JobStartRequest, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, - SessionSushNonce, VerifiedJob, + JobId, JobMode, JobStartRequest, SessionClientNonce, SessionId, SessionPreimage, SessionSeq, + SessionSignerNonce, SessionSushNonce, VerifiedJob, }; use sush_common::keys::{EphemeralKey, KeyType, Signer, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; @@ -258,16 +258,19 @@ pub struct SessionCeremony { /// against `mgr` go stale. pub fn session_ceremony(mgr: &JobManager, seq: u64) -> SessionCeremony { let client_nonce = SessionClientNonce::random(); - let signer_nonce = SessionSignerNonce::random(); let sush_nonce = SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()); - let seq = SessionSeq::from_unix_seconds(seq).unwrap(); - let session_id = SessionId::compute(mgr.own_baseboard(), sush_nonce, signer_nonce, seq); + let preimage = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce, + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_seconds(seq).unwrap(), + }; SessionCeremony { - session_id, + session_id: preimage.session_id(), client_nonce, sush_nonce, - signer_nonce, - seq, + signer_nonce: preimage.signer_nonce, + seq: preimage.seq, } } @@ -279,17 +282,19 @@ pub async fn http_session_ceremony( seq: u64, ) -> (SessionId, sush_client::types::SessionStartBody) { let client_nonce = SessionClientNonce::random(); - let signer_nonce = SessionSignerNonce::random(); let server_nonce = client.server_nonce().send().await.unwrap().into_inner(); - let sush_nonce = SessionSushNonce::compute(client_nonce, server_nonce); - let seq = SessionSeq::from_unix_seconds(seq).unwrap(); - let session_id = SessionId::compute(&test_baseboard_id(), sush_nonce, signer_nonce, seq); + let preimage = SessionPreimage { + baseboard: test_baseboard_id(), + sush_nonce: SessionSushNonce::compute(client_nonce, server_nonce), + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_seconds(seq).unwrap(), + }; ( - session_id, + preimage.session_id(), sush_client::types::SessionStartBody { client_nonce, - signer_nonce, - seq, + signer_nonce: preimage.signer_nonce, + seq: preimage.seq, }, ) } From dbe29b0d85455a7bc2969457edda242994fd50a9 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 18 Oct 2026 16:38:04 -0600 Subject: [PATCH 4/7] Use permslip session credentials Co-Authored-By: Claude Mythos 5 --- Cargo.lock | 99 ++++++++++---- Cargo.toml | 4 +- client/src/cli.rs | 91 ++++--------- client/src/commands.rs | 289 ++++++++++++++++++++++------------------- client/src/context.rs | 9 +- client/src/permslip.rs | 19 +-- client/src/repl.rs | 16 +-- 7 files changed, 274 insertions(+), 253 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e668d409..5c005c2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1173,6 +1173,16 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -2685,6 +2695,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308d96db8debc727c3fd9744aac51751243420e46edf401010908da7f8d5e57c" +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "newtype-uuid" version = "1.4.0" @@ -2799,32 +2815,51 @@ dependencies = [ ] [[package]] -name = "oauth2" -version = "5.0.0" +name = "objc2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" dependencies = [ - "base64 0.22.1", - "chrono", - "getrandom 0.2.17", - "http 1.5.0", - "rand 0.8.7", - "serde", - "serde_json", - "serde_path_to_error", - "sha2", - "thiserror 1.0.69", - "url", + "objc2-encode", ] [[package]] -name = "oauth2-reqwest" -version = "0.1.0-alpha.3" +name = "objc2-app-kit" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234fb5c965bbce983ee5de636a7a51d6a3223da8067ea02f9ab2d2d78ac08be2" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "oauth2", - "reqwest", + "bitflags 2.13.1", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", ] [[package]] @@ -3008,7 +3043,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "permission-slip-client" version = "1.1.0" -source = "git+https://github.com/oxidecomputer/permission-slip?rev=c9d302ef44576e075e35f4f28f8e7ac8345bcd4b#c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" +source = "git+https://github.com/oxidecomputer/permission-slip?rev=94b68e717f2da426ca1450d5ebc9fc5000596bdc#94b68e717f2da426ca1450d5ebc9fc5000596bdc" dependencies = [ "anyhow", "base64 0.22.1", @@ -3022,8 +3057,6 @@ dependencies = [ "hex", "lpc55_sign", "natord", - "oauth2", - "oauth2-reqwest", "once_cell", "permission-slip-common", "progenitor", @@ -3037,6 +3070,7 @@ dependencies = [ "sush-common", "tokio", "uuid", + "webbrowser", "x509-cert", "xdg 2.5.2", ] @@ -3044,7 +3078,7 @@ dependencies = [ [[package]] name = "permission-slip-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/permission-slip?rev=c9d302ef44576e075e35f4f28f8e7ac8345bcd4b#c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" +source = "git+https://github.com/oxidecomputer/permission-slip?rev=94b68e717f2da426ca1450d5ebc9fc5000596bdc#94b68e717f2da426ca1450d5ebc9fc5000596bdc" dependencies = [ "aws-types", "blake3", @@ -4617,7 +4651,7 @@ dependencies = [ [[package]] name = "sshauth" version = "0.1.2" -source = "git+https://github.com/oxidecomputer/sshauth?rev=afd9b4e549d0e9229c2910c94aa7bca76623dfdb#afd9b4e549d0e9229c2910c94aa7bca76623dfdb" +source = "git+https://github.com/oxidecomputer/sshauth#6233c1b25d9e3b03d51ab95b19ed7903bfadfd8f" dependencies = [ "anyhow", "base64 0.22.1", @@ -5587,7 +5621,6 @@ dependencies = [ "idna", "percent-encoding", "serde", - "serde_derive", ] [[package]] @@ -5763,6 +5796,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" +dependencies = [ + "jni", + "log", + "ndk-context", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "url", + "web-sys", +] + [[package]] name = "webpki-root-certs" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index 65872a62..81f90277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,8 +40,8 @@ memmap2 = "0.9" p256 = { version = "0.13", features = ["ecdsa", "pkcs8"] } pem-rfc7468 = { version = "0.7", features = ["std"] } percent-encoding = "2" -permission-slip-client = { git = "https://github.com/oxidecomputer/permission-slip", rev = "c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" } -permission-slip-common = { git = "https://github.com/oxidecomputer/permission-slip", rev = "c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" } +permission-slip-client = { git = "https://github.com/oxidecomputer/permission-slip", rev = "94b68e717f2da426ca1450d5ebc9fc5000596bdc", features = ["sush-types"] } +permission-slip-common = { git = "https://github.com/oxidecomputer/permission-slip", rev = "94b68e717f2da426ca1450d5ebc9fc5000596bdc" } pki-playground = { git = "https://github.com/oxidecomputer/pki-playground", rev = "7600756029ce046a02c6234aa84ce230cc5eaa04" } progenitor = "0.14" progenitor-client = "0.14" diff --git a/client/src/cli.rs b/client/src/cli.rs index ace7436e..14be9cd8 100644 --- a/client/src/cli.rs +++ b/client/src/cli.rs @@ -17,7 +17,7 @@ use anstream::print; use anstyle::{AnsiColor, Style}; use atomicwrites::{AtomicFile, OverwriteBehavior}; use bytesize::ByteSize; -use chrono::{DateTime, TimeDelta, Utc}; +use chrono::TimeDelta; use humantime::format_duration; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use rustix::io::ioctl_fionread; @@ -30,9 +30,9 @@ use xdg::BaseDirectories; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobOutputState, JobOutputStream, JobStatus, JobStatusMap, Session, - SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, - job_status_to_json_map, + Access, JobId, JobOutputState, JobOutputStream, JobStartRequest, JobStatus, JobStatusMap, + Session, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, + SignedJob, job_status_to_json_map, }; use sush_common::keys::{KeyId, Signature, SshPublicKey}; use sush_common::targets::{MAX_CUBBY, SledHealth, SledId, SledVersion}; @@ -45,16 +45,9 @@ use crate::context::{CommandContext, OutputFormat, StatusDisplayStyle}; pub(crate) const PREFIX: &str = "sush"; const SESSION_FILE_NAME: &str = "session.json"; const SESSION_FILE_VERSION: u32 = 1; -const TOKEN_FILE_NAME: &str = "permslip-token.json"; -const TOKEN_FILE_VERSION: u32 = 1; const CLIENT_NONCE_FILE_NAME: &str = "client-nonce.json"; const CLIENT_NONCE_FILE_VERSION: u32 = 1; -/// How long a permslip token is trusted for reuse. Staying well -/// under the server's 15 minute TTL spares a command from expiring -/// mid-flight. -const TOKEN_REUSE: TimeDelta = TimeDelta::minutes(10); - /// The persisted session, versioned for future migrations. #[derive(Deserialize, Serialize)] struct SavedSession { @@ -62,17 +55,6 @@ struct SavedSession { session: Session, } -/// A persisted permslip token, versioned for future migrations. The -/// url and fingerprint pin it to one signing server and one identity. -#[derive(Deserialize, Serialize)] -struct SavedToken { - version: u32, - url: String, - fingerprint: String, - token: String, - created: DateTime, -} - /// The persisted client nonce of the pending session ceremony, /// versioned for future migrations. One pending ceremony at a time, /// mirroring the rack's one server nonce slot: `session start-params` @@ -91,7 +73,6 @@ pub struct Cli { watch: Arc>>, session: Arc>>, session_file: Option, - token_file: Option, client_nonce_file: Option, credentials: AuthzSigner, } @@ -123,10 +104,6 @@ impl Cli { Err(error) => eprintln!("❗ Ignoring the saved session: {error}"), } self.session_file = Some(path); - match BaseDirectories::with_prefix(PREFIX).place_state_file(TOKEN_FILE_NAME) { - Ok(path) => self.token_file = Some(path), - Err(error) => eprintln!("❗ Signing tokens will not persist: {error}"), - } match BaseDirectories::with_prefix(PREFIX).place_state_file(CLIENT_NONCE_FILE_NAME) { Ok(path) => self.client_nonce_file = Some(path), Err(error) => eprintln!("❗ The client nonce will not persist: {error}"), @@ -293,43 +270,6 @@ impl CommandContext for Cli { self.credentials.clone() } - fn permslip_token(&self, url: &str, fingerprint: &str) -> Option { - let json = fs::read(self.token_file.as_ref()?).ok()?; - match serde_json::from_slice::(&json) { - Ok(SavedToken { - version: TOKEN_FILE_VERSION, - url: saved_url, - fingerprint: saved_fingerprint, - token, - created, - }) if saved_url == url - && saved_fingerprint == fingerprint - && (TimeDelta::zero()..TOKEN_REUSE).contains(&(Utc::now() - created)) => - { - Some(token) - } - _ => None, - } - } - - fn save_permslip_token(&self, url: &str, fingerprint: &str, token: &str) { - let Some(path) = &self.token_file else { - return; - }; - let result = serde_json::to_vec_pretty(&SavedToken { - version: TOKEN_FILE_VERSION, - url: url.to_owned(), - fingerprint: fingerprint.to_owned(), - token: token.to_owned(), - created: Utc::now(), - }) - .map_err(io::Error::other) - .and_then(|json| write_private(path, &json)); - if let Err(error) = result { - eprintln!("❗ The token was not saved: {error}"); - } - } - fn pending_client_nonce(&self) -> Option { let json = fs::read(self.client_nonce_file.as_ref()?).ok()?; match serde_json::from_slice::(&json) { @@ -395,6 +335,29 @@ impl CommandContext for Cli { } } + fn job_start_params(&self, request: &JobStartRequest) { + match self.get_output_format() { + OutputFormat::Json => println!( + "{}", + json!({ + "job_id": &request.job_id, + "session_id": &request.session_id, + "command": &request.command, + "mode": &request.mode, + "target": &request.target.to_string(), + }) + ), + OutputFormat::Text => { + println!("✅ Please request a job with these parameters from support:"); + println!(" Job ID: {}", request.job_id); + println!(" Session ID: {}", request.session_id); + println!(" Command: {}", request.command); + println!(" Mode: {}", request.mode.as_str()); + println!(" Target: {}", request.target); + } + } + } + fn session_created( &mut self, session: Session, diff --git a/client/src/commands.rs b/client/src/commands.rs index 833a5005..71252fd3 100644 --- a/client/src/commands.rs +++ b/client/src/commands.rs @@ -7,7 +7,6 @@ //! May be executed via either the main CLI or the interactive REPL. use std::fs::{File, OpenOptions, read}; -#[cfg(feature = "permslip")] use std::io::ErrorKind; use std::io::{Seek as _, SeekFrom, Write as _, stdin}; use std::net::SocketAddr; @@ -47,12 +46,10 @@ use sush_common::authn::{ }; use sush_common::interactive::{InteractiveJobError, InteractiveJobMessage}; use sush_common::jobs::JobOutputStream::{self, Stderr, Stdout}; -#[cfg(feature = "permslip")] -use sush_common::jobs::JobStartRequest; use sush_common::jobs::{ - Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStatus, JobStatusMap, - Session, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, - SignedJob, SkipReason, job_status_try_from_json_map, + Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStartRequest, JobStatus, + JobStatusMap, Session, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, + SessionSushNonce, SignedJob, SkipReason, job_status_try_from_json_map, }; use sush_common::keys::{KeyError, KeyId, Signer as _}; use sush_common::targets::{SledId, Target}; @@ -63,7 +60,7 @@ use crate::context::{Authz, CommandContext, OutputFormat, StatusDisplayStyle}; use crate::identity::{IdentityError, SshAgentConnection}; use crate::interactive::interactive_job; #[cfg(feature = "permslip")] -use crate::permslip::{PermslipError, PermslipSigner, fresh_token}; +use crate::permslip::{PermslipError, PermslipSigner, saved_token}; use crate::repl::Repl; use crate::tls; use crate::tunnel::{Tunnel, TunnelError}; @@ -742,10 +739,11 @@ pub struct JobStartArgs { } impl JobStartArgs { - /// The signing key name in a build that can sign. - fn key_name(&self) -> Option<&str> { + /// The signing service URL in a build that can sign. Its absence + /// means an out-of-band request to the signer (support). + fn signer_url(&self) -> Option<&str> { #[cfg(feature = "permslip")] - return self.permslip.as_deref(); + return self.permslip_url.as_deref(); #[cfg(not(feature = "permslip"))] None } @@ -1090,20 +1088,7 @@ async fn session( Ok(()) } - (SessionCommand::StartParams, Some(client)) => { - let (baseboard_id, nonce) = with_login(ctx, client, async || { - Ok(( - client.target().send().await?.into_inner(), - client.server_nonce().send().await?.into_inner(), - )) - }) - .await?; - let client_nonce = SessionClientNonce::random(); - let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); - ctx.save_client_nonce(client_nonce); - ctx.session_start_params(baseboard_id, client_nonce, sush_nonce); - Ok(()) - } + (SessionCommand::StartParams, Some(client)) => session_start_params(ctx, client).await, #[cfg(feature = "permslip")] ( @@ -1136,6 +1121,12 @@ async fn session( true, ) } else { + let Some(permslip_url) = permslip_url else { + return session_start_params(ctx, client).await; + }; + let Some(permslip_key) = permslip else { + return Err(CommandError::MissingKeyName); + }; let (baseboard_id, nonce) = with_login(ctx, client, async || { Ok(( client.target().send().await?.into_inner(), @@ -1146,7 +1137,7 @@ async fn session( let client_nonce = SessionClientNonce::random(); let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); let (session_id, signer_nonce, seq) = - session_create(ctx, permslip, permslip_url, &baseboard_id, sush_nonce) + session_create(permslip_key, permslip_url, &baseboard_id, sush_nonce) .await?; ctx.session_created(Session::new(session_id), signer_nonce, seq); ( @@ -1172,8 +1163,14 @@ async fn session( }, _, ) => { + let Some(permslip_url) = permslip_url else { + return Err(CommandError::MissingPermslipUrl); + }; + let Some(permslip_key) = permslip else { + return Err(CommandError::MissingKeyName); + }; let (session_id, nonce, seq) = - session_create(ctx, permslip, permslip_url, &baseboard_id, nonce).await?; + session_create(permslip_key, permslip_url, &baseboard_id, nonce).await?; ctx.session_created(Session::new(session_id), nonce, seq); Ok(()) } @@ -1189,8 +1186,10 @@ async fn session( }, Some(client), ) => { + // A build that cannot sign is always in the relay flow: a + // bare start means the start parameters. let (Some(session_id), Some(signer_nonce), Some(seq)) = (session_id, nonce, seq) else { - return Err(CommandError::SigningUnavailable); + return session_start_params(ctx, client).await; }; let client_nonce = client_nonce .or_else(|| ctx.pending_client_nonce()) @@ -1273,8 +1272,10 @@ async fn job( command: JobCommand, ) -> Result<(), CommandError> { match (command, client) { + // Without a signing service, a bare job start takes a job the + // signer already signed. (JobCommand::Start { start_args }, Some(client)) - if start_args.command.is_none() && start_args.key_name().is_none() => + if start_args.command.is_none() && start_args.signer_url().is_none() => { let job = ctx.read_signed_job()?; job_start(ctx, client, job, start_args).await?; @@ -1289,6 +1290,42 @@ async fn job( Some(_), ) => Err(CommandError::MissingCommand), + // Without a signing service, a command's job start prints the + // request parameters for the user to relay to support. + #[cfg(feature = "permslip")] + ( + JobCommand::Start { + start_args: + ref start_args @ JobStartArgs { + command: Some(_), + permslip_url: None, + .. + }, + }, + client, + ) => { + let request = job_start_request(ctx, client.as_ref(), start_args).await?; + ctx.job_start_params(&request); + Ok(()) + } + + // A build that cannot sign prints the request parameters for + // the user to relay to support. + #[cfg(not(feature = "permslip"))] + ( + JobCommand::Start { + start_args: + ref start_args @ JobStartArgs { + command: Some(_), .. + }, + }, + client, + ) => { + let request = job_start_request(ctx, client.as_ref(), start_args).await?; + ctx.job_start_params(&request); + Ok(()) + } + #[cfg(feature = "permslip")] ( JobCommand::Start { @@ -1298,80 +1335,24 @@ async fn job( Some(_), ) => Err(CommandError::MissingKeyName), - #[cfg(not(feature = "permslip"))] - (JobCommand::Start { .. }, Some(_)) => Err(CommandError::SigningUnavailable), - #[cfg(feature = "permslip")] ( JobCommand::Start { start_args: ref start_args @ JobStartArgs { - command: Some(ref command), permslip: Some(ref key_name), - ref permslip_url, - ref interactive, - ref streaming, - ref target, + permslip_url: Some(ref permslip_url), .. }, }, client, ) => { - let Some(session_id) = ctx.session_id() else { - return Err(CommandError::MissingSession); - }; - let Some(permslip_url) = permslip_url else { - return Err(CommandError::MissingPermslipUrl); - }; - let job_id = ctx.next_job_id()?; - let target = match target { - TargetArg::Target(target) => target.clone(), - abbreviated => { - let Some(client) = client.as_ref() else { - return Err(CommandError::Offline); - }; - resolve_target_arg(client, abbreviated).await? - } - }; - // Sign an interactive or streaming job for the sled its - // attachment will land on. - let target = if (*interactive || *streaming) && target.single_baseboard().is_none() { - let Some(client) = client.as_ref() else { - return Err(CommandError::InteractiveTarget); - }; - Target::from(resolve_target(client, &target).await?) - } else { - target - }; - // Catch output file problems before the signing ceremony. - if *streaming { - let Some(path) = &start_args.file else { - return Err(CommandError::StreamingNeedsFile); - }; - if !start_args.force && path.exists() { - return Err(CommandError::io(path, ErrorKind::AlreadyExists.into())); - } - } - if let Some(client) = client.as_ref() { - preflight_target(ctx, client, &target).await?; - } - let mode = if *interactive { - JobMode::Interactive - } else if *streaming { - JobMode::StreamOutput - } else { - JobMode::Batch - }; - let signer = permslip_signer(ctx, key_name, permslip_url).await?; + let request = job_start_request(ctx, client.as_ref(), start_args).await?; + let job_id = request.job_id; + let signer = permslip_signer(key_name, permslip_url).await?; let mut interval = interval(SIGNING_UPDATE_INTERVAL); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - let sign = signer.sign_job_request(JobStartRequest::new( - job_id.to_owned(), - session_id, - command, - mode, - target, - )); + let sign = signer.sign_job_request(request); pin!(sign); ctx.job_signing_started(&job_id); let mut sigint = signal(SignalKind::interrupt())?; @@ -1737,59 +1718,108 @@ fn parse_baseboard_id(value: &str) -> Result { value.parse::().map_err(|e| e.to_string()) } -/// A permslip signer authenticated as the agent key `-s`/`SUSH_KEY_ID` -/// names, or the agent's first. We choose the key rather than letting -/// permslip choose, so that one identity serves both the rack and the -/// signing service. +/// A permslip signer authenticated by the credential that +/// `permslip login` saved. #[cfg(feature = "permslip")] async fn permslip_signer( - ctx: &mut impl CommandContext, key_name: &str, permslip_url: &str, ) -> Result { - let globals = ctx.get_globals(); - let Some(sock) = globals.ssh_auth_sock.clone() else { - return Err(CommandError::MissingSshAuthSock); + let token = saved_token(permslip_url).await?; + Ok(PermslipSigner::new(key_name, permslip_url, &token)?) +} + +/// Fetch the ceremony inputs from the rack and print the parameters +/// to relay to the signer, stashing the client nonce for the +/// eventual start. +async fn session_start_params( + ctx: &mut impl CommandContext, + client: &Client, +) -> Result<(), CommandError> { + let (baseboard_id, nonce) = with_login(ctx, client, async || { + Ok(( + client.target().send().await?.into_inner(), + client.server_nonce().send().await?.into_inner(), + )) + }) + .await?; + let client_nonce = SessionClientNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); + ctx.save_client_nonce(client_nonce); + ctx.session_start_params(baseboard_id, client_nonce, sush_nonce); + Ok(()) +} + +/// Assemble the request a job start submits for signing: the +/// session's next job id, the resolved target, and the mode the +/// flags name. +async fn job_start_request( + ctx: &mut impl CommandContext, + client: Option<&Client>, + start_args: &JobStartArgs, +) -> Result { + let Some(command) = &start_args.command else { + return Err(CommandError::MissingCommand); }; - let key_id = globals.ssh_key_id.clone(); - let mut agent = SshAgentConnection::connect(&sock).await?; - let key = agent.identity(key_id.as_ref()).await?; - let fingerprint = key.fingerprint(); - let token = match ctx.permslip_token(permslip_url, &fingerprint) { - Some(token) => token, - None => { - ctx.please_touch(&key)?; - let token = fresh_token(permslip_url, sock, fingerprint.clone()).await?; - ctx.save_permslip_token(permslip_url, &fingerprint, &token); - token + let Some(session_id) = ctx.session_id() else { + return Err(CommandError::MissingSession); + }; + let job_id = ctx.next_job_id()?; + let (interactive, streaming) = (start_args.interactive, start_args.streaming); + let target = match &start_args.target { + TargetArg::Target(target) => target.clone(), + abbreviated => { + let Some(client) = client else { + return Err(CommandError::Offline); + }; + resolve_target_arg(client, abbreviated).await? } }; - Ok(PermslipSigner::new(key_name, permslip_url, &token)?) + // An interactive or streaming job binds to the single sled its + // attachment will land on. + let target = if (interactive || streaming) && target.single_baseboard().is_none() { + let Some(client) = client else { + return Err(CommandError::InteractiveTarget); + }; + Target::from(resolve_target(client, &target).await?) + } else { + target + }; + // Catch output file problems before anything is signed. + if streaming { + let Some(path) = &start_args.file else { + return Err(CommandError::StreamingNeedsFile); + }; + if !start_args.force && path.exists() { + return Err(CommandError::io(path, ErrorKind::AlreadyExists.into())); + } + } + if let Some(client) = client { + preflight_target(ctx, client, &target).await?; + } + let mode = if interactive { + JobMode::Interactive + } else if streaming { + JobMode::StreamOutput + } else { + JobMode::Batch + }; + Ok(JobStartRequest::new( + job_id, session_id, command, mode, target, + )) } /// Ask the online signing service to create a session. #[cfg(feature = "permslip")] async fn session_create( - ctx: &mut impl CommandContext, - permslip: Option, - permslip_url: Option, + permslip_key: String, + permslip_url: String, baseboard_id: &BaseboardId, nonce: SessionSushNonce, ) -> Result<(SessionId, SessionSignerNonce, SessionSeq), CommandError> { - let Some(permslip_url) = permslip_url else { - return Err(CommandError::MissingPermslipUrl); - }; - let Some(permslip_key) = permslip else { - return Err(CommandError::MissingKeyName); - }; - let signer = permslip_signer(ctx, &permslip_key, &permslip_url).await?; - let _created = signer.create_session(baseboard_id, nonce).await?; - // The signing service cannot order sessions yet: its response - // carries no sequence number until the permission-slip side of - // protocol v2 lands. - Err(CommandError::UnsupportedPermslipResponse( - "the signing service returned no session sequence number".to_string(), - )) + let signer = permslip_signer(&permslip_key, &permslip_url).await?; + let created = signer.create_session(baseboard_id, nonce).await?; + Ok((created.session_id, created.signer_nonce, created.seq)) } async fn session_start( @@ -2393,7 +2423,6 @@ impl FromStr for TargetArg { } /// Confirm before signing a job for a sled that doesn't appear in inventory. -#[cfg(feature = "permslip")] async fn preflight_target( ctx: &mut impl CommandContext, client: &Client, @@ -2596,9 +2625,6 @@ pub enum CommandError { Reqwest(#[from] reqwest::Error), #[error("❌ SSH signature error on key ID `{0}`")] Signature(KeyId), - #[cfg(not(feature = "permslip"))] - #[error("❌ Signing unavailable: built without the `permslip` feature")] - SigningUnavailable, #[error("❌ Streaming jobs need `--file`")] StreamingNeedsFile, #[error("❌ Job `{0}` did not exit cleanly, streamed output is unverified")] @@ -2625,9 +2651,6 @@ pub enum CommandError { Utf8(#[from] std::string::FromUtf8Error), #[error("❌ WebSocket error: {0}")] WebSocket(#[from] WebSocketError), - #[cfg(feature = "permslip")] - #[error("❌ Unsupported permslip response: {0}")] - UnsupportedPermslipResponse(String), } impl CommandError { diff --git a/client/src/context.rs b/client/src/context.rs index ea2fca4b..c5fc942d 100644 --- a/client/src/context.rs +++ b/client/src/context.rs @@ -14,8 +14,8 @@ use x509_cert::Certificate; use sush_common::authn::{BoundRequest, Credentials, Identity, RequestKey}; use sush_common::jobs::{ - Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, - SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, + Access, JobId, JobOutputStream, JobStartRequest, JobStatusMap, Session, SessionClientNonce, + SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::{SledId, SledVersion}; @@ -113,10 +113,6 @@ pub trait CommandContext: Clone + Send + Sync { fn set_credentials(&mut self, credentials: Option) { self.authz_signer().set(credentials) } - fn permslip_token(&self, _url: &str, _fingerprint: &str) -> Option { - None - } - fn save_permslip_token(&self, _url: &str, _fingerprint: &str, _token: &str) {} fn pending_client_nonce(&self) -> Option { None } @@ -129,6 +125,7 @@ pub trait CommandContext: Clone + Send + Sync { client_nonce: SessionClientNonce, sush_nonce: SessionSushNonce, ); + fn job_start_params(&self, request: &JobStartRequest); fn session_created( &mut self, session: Session, diff --git a/client/src/permslip.rs b/client/src/permslip.rs index fca7477c..ca161913 100644 --- a/client/src/permslip.rs +++ b/client/src/permslip.rs @@ -21,19 +21,12 @@ pub struct PermslipSigner { key_name: String, } -/// Get a bearer token by the sshauth challenge, signed with the agent -/// key named by `fingerprint`. -pub async fn fresh_token( - url: &str, - agent_sock: String, - fingerprint: String, -) -> Result { - let tokens = TokenProvider::SshAuth { - fingerprint: Some(fingerprint), - server_url: url.to_owned(), - agent_sock, - }; - let token = tokens.token().await.map_err(PermslipError::token_flow)?; +/// Get the bearer token that `permslip login` saved for `url`. +pub async fn saved_token(url: &str) -> Result { + let token = TokenProvider::SavedCredentials + .token(url) + .await + .map_err(PermslipError::token_flow)?; let value = token.into_header_value().map_err(PermslipError::token)?; value .to_str() diff --git a/client/src/repl.rs b/client/src/repl.rs index cf710a4f..ec65be73 100644 --- a/client/src/repl.rs +++ b/client/src/repl.rs @@ -21,8 +21,8 @@ use xdg::BaseDirectories; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, - SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, + Access, JobId, JobOutputStream, JobStartRequest, JobStatusMap, Session, SessionClientNonce, + SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::{SledId, SledVersion}; @@ -227,14 +227,6 @@ impl CommandContext for Repl { self.cli.authz_signer() } - fn permslip_token(&self, url: &str, fingerprint: &str) -> Option { - self.cli.permslip_token(url, fingerprint) - } - - fn save_permslip_token(&self, url: &str, fingerprint: &str, token: &str) { - self.cli.save_permslip_token(url, fingerprint, token) - } - fn session_id(&self) -> Option { self.cli.session_id() } @@ -261,6 +253,10 @@ impl CommandContext for Repl { .session_start_params(baseboard_id, client_nonce, sush_nonce) } + fn job_start_params(&self, request: &JobStartRequest) { + self.cli.job_start_params(request) + } + fn session_created( &mut self, session: Session, From 6b18ab19d9d07dd1d53704279f90497a636cbd87 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 18 Oct 2026 17:08:41 -0600 Subject: [PATCH 5/7] Regenerate the API document Co-Authored-By: Claude Mythos 5 --- sush.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sush.json b/sush.json index 94056abc..76ab6ecb 100644 --- a/sush.json +++ b/sush.json @@ -1563,11 +1563,11 @@ "type": "string" }, "SessionId": { - "description": "A globally unique identifier for a session.", + "description": "A globally unique identifier for a session. Constructed only from a `SessionPreimage`.", "type": "string" }, "SessionSeq": { - "description": "The session's place in the signer-assigned order of sessions. Taken as whole seconds since the Unix epoch according to the *signer's* wall clock; the server's clock can't be trusted, so it never interprets a seq as a time, only as an order. Sessions authorized in the same second share a seq and tie-break by id. Spoken as three codephrase words: 2048³ = 2³³ values, which reaches the year 2242.", + "description": "The session's place in the signer-assigned order of sessions. Taken as whole seconds since the Unix epoch according to the *signer's* wall clock; the server's clock can't be trusted, so the server never interprets a sequence number as a time, only as an order. Sessions authorized in the same second share a sequence number and tie-break by id. Spoken as three codephrase words: 2048³ = 2³³ values, which reaches the year 2242.", "type": "string" }, "SessionServerNonce": { From 0d3c658c5b91ff1e90df4bb28d4346e4663a2d5f Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Mon, 19 Oct 2026 15:13:31 -0600 Subject: [PATCH 6/7] Generate four-word sequence numbers from millisecond-precision time Co-Authored-By: Claude Mythos 5 --- common/src/codephrases.rs | 9 +-- common/src/jobs.rs | 66 +++++++++--------- server/src/boundary.rs | 10 +-- server/src/messages.rs | 2 +- server/src/state.rs | 10 +-- server/tests/distributed.rs | 2 +- server/tests/output/boundary-record-v1.bin | Bin 215 -> 215 bytes server/tests/output/session-start-request.bin | Bin 270 -> 270 bytes sush.json | 4 +- tests/src/manager_tests.rs | 2 +- tests/src/test_utils.rs | 4 +- 11 files changed, 55 insertions(+), 54 deletions(-) diff --git a/common/src/codephrases.rs b/common/src/codephrases.rs index 78cae843..3fa05929 100644 --- a/common/src/codephrases.rs +++ b/common/src/codephrases.rs @@ -46,9 +46,10 @@ const WORD_SEPARATOR: &str = "-"; /// the strength of the signatures produced over these phrases. const TRUNCATED_BITS: u32 = 88; -/// A three-word phrase carries 33 bits. Not enough for an identifier, -/// but room enough for a small ordered value, like a Unix time. -const SHORT_BITS: u32 = 33; +/// A four-word phrase carries 44 bits. Not enough for an identifier, +/// but room enough for a small ordered value, like a Unix time in +/// milliseconds. +const SHORT_BITS: u32 = 44; /// Random code phrase like `abstract misery favorite ordinary moon talk`. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -66,7 +67,7 @@ impl Codephrase { self.mask(TRUNCATED_BITS) } - /// Chop entropy down to three words; see `SHORT_BITS`. + /// Chop entropy down to four words; see `SHORT_BITS`. pub fn shorten(self) -> Self { self.mask(SHORT_BITS) } diff --git a/common/src/jobs.rs b/common/src/jobs.rs index ce0eb040..0ce040b1 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -206,7 +206,7 @@ impl SessionSushNonce { } codephrase_newtype! { - /// The component of the hashed Session ID generated by the signer server. + /// The component of the hashed Session ID generated by the signing server. #[derive( Copy, Clone, @@ -224,13 +224,13 @@ codephrase_newtype! { codephrase_newtype! { /// The session's place in the signer-assigned order of sessions. - /// Taken as whole seconds since the Unix epoch according to the - /// *signer's* wall clock; the server's clock can't be trusted, so - /// the server never interprets a sequence number as a time, only - /// as an order. Sessions authorized in the same second share a - /// sequence number and tie-break by id. - /// Spoken as three codephrase words: 2048³ = 2³³ values, which - /// reaches the year 2242. + /// Taken as whole milliseconds since the Unix epoch according to + /// the *signer's* wall clock; the server's clock can't be trusted, + /// so the server never interprets a sequence number as a time, + /// only as an order. Sessions authorized in the same millisecond + /// share a sequence number and tie-break by ID. Encoded as four + /// codephrase words: 2048⁴ = 2⁴⁴ values, good through the year + /// 2527. #[derive( Copy, Clone, @@ -247,25 +247,25 @@ codephrase_newtype! { } #[derive(Debug, Error)] -#[error("session sequence number exceeds three-word codephrase range")] +#[error("session sequence number exceeds four-word codephrase range")] pub struct SeqOutOfRange; impl SessionSeq { - /// The largest encodable wall clock reading, `zoo-zoo-zoo` - /// = 2242-03-16T12:56:31Z. - pub const MAX_SECONDS: u64 = (1 << 33) - 1; + /// The largest encodable wall clock reading, `zoo-zoo-zoo-zoo` + /// = 2527-06-23T06:20:44.415Z. + pub const MAX_MILLIS: u64 = (1 << 44) - 1; /// Make a sequence number from a wall clock reading. Fails - /// if seconds is out of range. - pub fn from_unix_seconds(seconds: u64) -> Result { - if seconds > Self::MAX_SECONDS { + /// if millis is out of range. + pub fn from_unix_millis(millis: u64) -> Result { + if millis > Self::MAX_MILLIS { return Err(SeqOutOfRange); } - Ok(Self(Codephrase::from_u64(seconds))) + Ok(Self(Codephrase::from_u64(millis))) } - pub fn as_unix_seconds(&self) -> u64 { - self.0.to_u64().expect("a sequence number is three words") + pub fn as_unix_millis(&self) -> u64 { + self.0.to_u64().expect("a sequence number is four words") } } @@ -854,26 +854,26 @@ mod test { use crate::keys::{EccR, EccS, EncodedSignature}; use crate::targets::SledId; - /// A sequence number encodes to three words, parses back liberally, + /// A sequence number encodes to four words, parses back liberally, /// and orders numerically. #[test] fn seq_phrases() { - let max = SessionSeq::from_unix_seconds(SessionSeq::MAX_SECONDS).unwrap(); - assert_eq!(max.to_string(), "zoo-zoo-zoo"); - assert_eq!("zoo zoo zoo".parse::().unwrap(), max); + let max = SessionSeq::from_unix_millis(SessionSeq::MAX_MILLIS).unwrap(); + assert_eq!(max.to_string(), "zoo-zoo-zoo-zoo"); + assert_eq!("zoo zoo zoo zoo".parse::().unwrap(), max); // 2026-10-09T09:33:20Z, a realistic signer clock reading. - // Every wall clock reading since 1970-02-18 fills three words. - let seq = SessionSeq::from_unix_seconds(1_760_000_000).unwrap(); - assert_eq!(seq.as_unix_seconds(), 1_760_000_000); - assert_eq!(seq.to_string(), "crucial-over-abandon"); + // Every wall clock reading since 1970-04-10 fills four words. + let seq = SessionSeq::from_unix_millis(1_760_000_000_000).unwrap(); + assert_eq!(seq.as_unix_millis(), 1_760_000_000_000); + assert_eq!(seq.to_string(), "book-tomato-reason-abandon"); assert_eq!(seq.to_string().parse::().unwrap(), seq); assert!(seq < max); - // 2³³ seconds exceed the range however they arrive. - assert!(SessionSeq::from_unix_seconds(1 << 33).is_err()); + // 2⁴⁴ milliseconds exceed the range however they arrive. + assert!(SessionSeq::from_unix_millis(1 << 44).is_err()); assert!( - "ability-abandon-abandon-abandon" + "ability-abandon-abandon-abandon-abandon" .parse::() .is_err() ); @@ -881,20 +881,20 @@ mod test { /// On the binary wire a sequence number is codephrase bytes, like /// the ids and nonces; as text it is words. A value too wide for - /// three words fails to decode. + /// four words fails to decode. #[test] fn seq_serde_forms() { - let seq = SessionSeq::from_unix_seconds(1_760_000_000).unwrap(); + let seq = SessionSeq::from_unix_millis(1_760_000_000_000).unwrap(); assert_eq!( serde_json::to_value(seq).unwrap(), - serde_json::json!("crucial-over-abandon") + serde_json::json!("book-tomato-reason-abandon") ); let mut bytes = Vec::new(); ciborium::ser::into_writer(&seq, &mut bytes).unwrap(); assert_eq!( hex::encode(&bytes), "5820000000000000000000000000000000000000000000000000\ - 0000000068e77800" + 00000199c82cc000" ); assert_eq!( ciborium::de::from_reader::(bytes.as_slice()).unwrap(), diff --git a/server/src/boundary.rs b/server/src/boundary.rs index bdfb38b1..94545baa 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -120,7 +120,7 @@ impl TryFrom for Boundary { network: old.network, job: old.job.map(|job| Committed { session: job.session, - seq: SessionSeq::from_unix_seconds(0).expect("zero is encodable"), + seq: SessionSeq::from_unix_millis(0).expect("zero is encodable"), job: job.job, successor: job.successor, outcome: job.outcome, @@ -382,8 +382,8 @@ mod test { serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() } - fn seq(seconds: u64) -> SessionSeq { - SessionSeq::from_unix_seconds(seconds).unwrap() + fn seq(millis: u64) -> SessionSeq { + SessionSeq::from_unix_millis(millis).unwrap() } fn boundary() -> Boundary { @@ -391,7 +391,7 @@ mod test { network: network(1), job: Some(Committed { session: SessionId::random(), - seq: seq(1_760_000_000), + seq: seq(1_760_000_000_000), job: JobId::random(), successor: JobId::random(), outcome: JobOutcome::Committed, @@ -428,7 +428,7 @@ mod test { network: network(1), job: Some(Committed { session: "abandon-ability".parse().unwrap(), - seq: seq(1_760_000_000), + seq: seq(1_760_000_000_000), job: "zoo-zero".parse().unwrap(), successor: "able-about".parse().unwrap(), outcome: JobOutcome::Committed, diff --git a/server/src/messages.rs b/server/src/messages.rs index d0e2b042..2528d961 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -440,7 +440,7 @@ mod wire_format { baseboard: baseboard.clone(), sush_nonce: SessionSushNonce::from_be_bytes([0x11; 32]), signer_nonce: SessionSignerNonce::from_be_bytes([0x22; 32]), - seq: SessionSeq::from_unix_seconds(1_760_000_000).unwrap(), + seq: SessionSeq::from_unix_millis(1_760_000_000_000).unwrap(), }), )), Message::Request(Request::session(actor(), SessionRequest::Stop(session))), diff --git a/server/src/state.rs b/server/src/state.rs index 312572e4..1a05484a 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -1727,18 +1727,18 @@ mod test { Past::new(Version::new(), committed) } - fn seq(seconds: u64) -> SessionSeq { - SessionSeq::from_unix_seconds(seconds).unwrap() + fn seq(millis: u64) -> SessionSeq { + SessionSeq::from_unix_millis(millis).unwrap() } - /// A preimage for a fresh session authorized at `seconds`; tests + /// A preimage for a fresh session authorized at `millis`; tests /// reference the session by the derived id. - fn preimage(seconds: u64) -> SessionPreimage { + fn preimage(millis: u64) -> SessionPreimage { SessionPreimage { baseboard: baseboard(), sush_nonce: SessionSushNonce::random(), signer_nonce: SessionSignerNonce::random(), - seq: seq(seconds), + seq: seq(millis), } } diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index 5640c314..f97e700a 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -694,7 +694,7 @@ fn ceremony(mgr: &JobManager, seq: u64) -> Ceremony { baseboard: mgr.own_baseboard().clone(), sush_nonce: SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()), signer_nonce: SessionSignerNonce::random(), - seq: SessionSeq::from_unix_seconds(seq).unwrap(), + seq: SessionSeq::from_unix_millis(seq).unwrap(), }; Ceremony { session_id: preimage.session_id(), diff --git a/server/tests/output/boundary-record-v1.bin b/server/tests/output/boundary-record-v1.bin index e2ea3269242183078530a6398d639b2cd566eb5f..582b74dffb97646f7b181a4a235a558af02f5e86 100644 GIT binary patch delta 23 fcmcc4c%5-VE*Im>6FLVNlC$!YCI-q)oazJsZM+E# delta 23 fcmcc4c%5-VE*Aqs#`6k>SNl#!N|rq^MuX;1^^x_1TFvo delta 14 VcmeBU>SNl#!N|tIkny~N0RS6G1E&B0 diff --git a/sush.json b/sush.json index 76ab6ecb..905fb383 100644 --- a/sush.json +++ b/sush.json @@ -1567,7 +1567,7 @@ "type": "string" }, "SessionSeq": { - "description": "The session's place in the signer-assigned order of sessions. Taken as whole seconds since the Unix epoch according to the *signer's* wall clock; the server's clock can't be trusted, so the server never interprets a sequence number as a time, only as an order. Sessions authorized in the same second share a sequence number and tie-break by id. Spoken as three codephrase words: 2048³ = 2³³ values, which reaches the year 2242.", + "description": "The session's place in the signer-assigned order of sessions. Taken as whole milliseconds since the Unix epoch according to the *signer's* wall clock; the server's clock can't be trusted, so the server never interprets a sequence number as a time, only as an order. Sessions authorized in the same millisecond share a sequence number and tie-break by ID. Encoded as four codephrase words: 2048⁴ = 2⁴⁴ values, good through the year 2527.", "type": "string" }, "SessionServerNonce": { @@ -1575,7 +1575,7 @@ "type": "string" }, "SessionSignerNonce": { - "description": "The component of the hashed Session ID generated by the signer server.", + "description": "The component of the hashed Session ID generated by the signing server.", "type": "string" }, "SessionStartBody": { diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index 8c6b503e..a8243aad 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -351,7 +351,7 @@ async fn fetch_rotates_server_nonce() { baseboard: mgr.own_baseboard().clone(), sush_nonce: ceremony.sush_nonce, signer_nonce: SessionSignerNonce::random(), - seq: SessionSeq::from_unix_seconds(2).unwrap(), + seq: SessionSeq::from_unix_millis(2).unwrap(), }; mgr.session_start( &authn, diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 625d92a0..1979a657 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -263,7 +263,7 @@ pub fn session_ceremony(mgr: &JobManager, seq: u64) -> SessionCeremony { baseboard: mgr.own_baseboard().clone(), sush_nonce, signer_nonce: SessionSignerNonce::random(), - seq: SessionSeq::from_unix_seconds(seq).unwrap(), + seq: SessionSeq::from_unix_millis(seq).unwrap(), }; SessionCeremony { session_id: preimage.session_id(), @@ -287,7 +287,7 @@ pub async fn http_session_ceremony( baseboard: test_baseboard_id(), sush_nonce: SessionSushNonce::compute(client_nonce, server_nonce), signer_nonce: SessionSignerNonce::random(), - seq: SessionSeq::from_unix_seconds(seq).unwrap(), + seq: SessionSeq::from_unix_millis(seq).unwrap(), }; ( preimage.session_id(), From 7d007680c8e1a64092979e994dd05fb86f552680 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Mon, 19 Oct 2026 20:52:08 -0600 Subject: [PATCH 7/7] Refuse to resume a session in another universe This was previously allowed in the state machine, but not actually supported end-to-end (e.g., the server nonce wouldn't match). But it's semantically problematic to allow, so just forbid it. Co-Authored-By: Claude Mythos 5 --- common/src/jobs.rs | 14 +- server/src/boundary.rs | 69 ++++---- server/src/messages.rs | 13 +- server/src/state.rs | 163 +++++++++++++----- server/tests/distributed.rs | 18 +- .../output/already-handled-skip-event.bin | Bin 149 -> 0 bytes server/tests/output/job-skipped-event.bin | Bin 145 -> 152 bytes sush.json | 25 +-- 8 files changed, 182 insertions(+), 120 deletions(-) delete mode 100644 server/tests/output/already-handled-skip-event.bin diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 0ce040b1..be90e08a 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -255,6 +255,10 @@ impl SessionSeq { /// = 2527-06-23T06:20:44.415Z. pub const MAX_MILLIS: u64 = (1 << 44) - 1; + pub fn zero() -> Self { + Self(Codephrase::from_u64(0)) + } + /// Make a sequence number from a wall clock reading. Fails /// if millis is out of range. pub fn from_unix_millis(millis: u64) -> Result { @@ -522,20 +526,14 @@ pub enum JobStatus { #[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "kebab-case")] pub enum SkipReason { - /// The job's session sits at or below the sled's execution floor, - /// where the sled cannot tell replay from re-run. - BelowFloor, - /// The job's chain position precedes the sled's recorded - /// commitment: a previous life already handled it. - AlreadyHandled, + SessionSuperseded, SessionEnded, } impl fmt::Display for SkipReason { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match self { - Self::BelowFloor => "the session sits below this sled's execution floor", - Self::AlreadyHandled => "a previous life of this sled already handled it", + Self::SessionSuperseded => "another session has superseded this one", Self::SessionEnded => "the session ended before the job could start", }) } diff --git a/server/src/boundary.rs b/server/src/boundary.rs index 94545baa..71748f9b 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -7,12 +7,11 @@ //! One record, rewritten before each spawn, carries what this sled //! last committed to: the job, its session and that session's //! sequence number, the chain position after the job, the universe -//! they belong to, and the job's ending once it has one. The record -//! is this sled's execution floor: the signer assigns every session -//! a sequence number from its wall clock, so sequence numbers order -//! sessions totally, across restarts and across universes, and no -//! session at or below the committed sequence number may run jobs -//! here again. +//! they belong to, and the job's ending once it has one. The signer +//! assigns every session a sequence number from its wall clock, so +//! sequence numbers order sessions totally, across restarts and +//! across universes: no session at or below the committed one may +//! run jobs here again. //! //! After a restart, replayed gossip rebuilds the sessions. When the //! committed session activates, the sled resumes its chain at the @@ -54,10 +53,10 @@ pub const BOUNDARY: TenantSpec = TenantSpec { }; /// The execution boundary: what this sled last committed to, and in -/// which universe. The committed job's sequence number is the sled's -/// global execution floor: the signer orders sessions totally and -/// across universes, so every session a previous life could have run -/// jobs under sits at or below this one stored number. +/// which universe. The committed session supersedes every session at +/// or below its sequence number: the signer orders sessions totally +/// and across universes, so every session a previous life could have +/// run jobs under sits at or below this one stored number. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Boundary { pub network: Network, @@ -93,10 +92,13 @@ pub enum JobOutcome { Ended(JobStatus), } -/// Version 1 replaced the executed-session join and the burned set -/// with the committed sequence number. The pinned record snapshots in this -/// module's tests freeze both versions' bytes; see [`crate::format`] -/// for the steps a format change requires. +/// Record version 1 replaces the v0 executed-session join and burned set +/// with the committed sequence number from the session-v2 protocol. +/// Don't confuse a "session-v2" (one with a sequence number) with a +/// boundary record version 2 (which doesn't yet exist). +/// +/// See [`crate::format`] for instructions on adding this version's +/// successor. impl Versioned for Boundary { const VERSION: u16 = 1; } @@ -108,22 +110,29 @@ impl Record for Boundary { impl TryFrom for Boundary { type Error = &'static str; - /// A v0 record kept a per-universe join of session-start versions - /// instead of a sequence number, and the join yields no sequence - /// number, so the committed job converts with a floor of zero: - /// the session still resumes, and the floor refuses nothing. The - /// zero floor readmits nothing a previous life ran, because every - /// v0 record predates sequence numbers and only sessions that - /// carry one can start now. fn try_from(old: v0::Boundary) -> Result { + let v0::Boundary { + network, + burned: _, // dropped with session-v2 protocol + executed: _, // also dropped with session-v2 + job, + } = old; Ok(Self { - network: old.network, - job: old.job.map(|job| Committed { - session: job.session, - seq: SessionSeq::from_unix_millis(0).expect("zero is encodable"), - job: job.job, - successor: job.successor, - outcome: job.outcome, + network, + job: job.map(|job| { + let v0::Committed { + session, + job, + successor, + outcome, + } = job; + Committed { + session, + seq: SessionSeq::zero(), // any valid session-v2 will supersede this one + job, + successor, + outcome, + } }), }) } @@ -404,8 +413,8 @@ mod test { } /// The frozen v0 snapshot decodes and converts: the commitment - /// survives with a floor of zero. Nothing can regenerate the - /// snapshot; the code that wrote it is gone, and git keeps the + /// survives with a sequence number of zero. Nothing can regenerate + /// the snapshot; the code that wrote it is gone, and git keeps the /// bytes. #[test] fn v0_record_converts() { diff --git a/server/src/messages.rs b/server/src/messages.rs index 2528d961..8dad368e 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -351,8 +351,7 @@ mod wire_format { } Event::Job(JobEvent::Error(..)) => "job-error-event", Event::Job(JobEvent::Skipped(_, _, reason)) => match reason { - SkipReason::BelowFloor => "job-skipped-event", - SkipReason::AlreadyHandled => "already-handled-skip-event", + SkipReason::SessionSuperseded => "job-skipped-event", SkipReason::SessionEnded => "session-ended-skip-event", }, }, @@ -501,15 +500,7 @@ mod wire_format { Event::Job(JobEvent::Skipped( skip_job, skip_when, - SkipReason::BelowFloor, - )), - ), - Message::Event( - baseboard.clone(), - Event::Job(JobEvent::Skipped( - skip_job, - skip_when, - SkipReason::AlreadyHandled, + SkipReason::SessionSuperseded, )), ), Message::Event( diff --git a/server/src/state.rs b/server/src/state.rs index 1a05484a..d7bad03a 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -161,8 +161,7 @@ enum Admission { /// Execute normally. Admit, - /// The session does not pass the execution floor, so skip the - /// job. + /// Refuse to execute the job because the session has been superseded. Refuse, /// If we cannot trust our boundary record, we must not admit @@ -188,6 +187,9 @@ pub struct Past { /// across restarts. committed: Commitment, + /// The gossip universe this life entered. + network: Network, + /// Jobs whose replayed start events say a previous life ran them. /// The end of such a job may replay later and remove it from the /// set, so which jobs truly lack an end is known only when replay @@ -206,37 +208,36 @@ pub struct Past { } impl Past { - fn new(birth: Version, committed: Commitment) -> Self { + fn new(birth: Version, committed: Commitment, network: Network) -> Self { Past { birth, committed, + network, zombies: BTreeSet::new(), reaped: BTreeSet::new(), adjudicated: None, } } - /// Decide whether the active session may run a job here. The - /// signer's sequence numbers order sessions totally and across - /// universes, so the floor needs no universe scoping: a session - /// at or below the committed sequence number may already have run - /// jobs on this sled, and only the committed session itself - /// resumes. + /// Decide whether the active session may run a job here. + /// A committed session may only resume in the network where + /// this sled committed to it, and sessions are superseded + /// by ones with a higher sequence number. fn screen(&self, session: SessionId, seq: SessionSeq, live: bool) -> Admission { match &self.committed { - // Refuse all jobs if our boundary is untrusted. Commitment::Untrusted => Admission::Untrusted, - Commitment::Trusted { boundary } => match &boundary.job { - Some(committed) if committed.session == session => Admission::Admit, - Some(committed) if seq > committed.seq => Admission::Admit, + Some(committed) + if (boundary.network == self.network && committed.session == session) + || seq > committed.seq => + { + Admission::Admit + } Some(_) => Admission::Refuse, None if live => Admission::Admit, None => Admission::Refuse, }, - // With no record we cannot tell replay from re-run, so - // only a session that started as live traffic runs jobs. Commitment::Absent if live => Admission::Admit, Commitment::Absent => Admission::Refuse, } @@ -427,8 +428,8 @@ impl State { network: Option, ) -> Vec { let mut outputs = self.reap_zombies(log, now); - if let Some(network) = network { - outputs.extend(self.adjudicate_boundary(log, now, network)); + if network.is_some() { + outputs.extend(self.adjudicate_boundary(log, now)); } outputs } @@ -462,12 +463,7 @@ impl State { /// other universe never saw the job and hears nothing, and /// re-entering the job's own universe replays the earlier ruling /// as a status, which stops a second announcement. - fn adjudicate_boundary( - &mut self, - log: &Logger, - now: DateTime, - network: Network, - ) -> Vec { + fn adjudicate_boundary(&mut self, log: &Logger, now: DateTime) -> Vec { // An absent or untrusted record commits to no job, so there // is nothing to rule on. let Commitment::Trusted { boundary, .. } = &self.past.committed else { @@ -476,7 +472,7 @@ impl State { let Some(Committed { job, outcome, .. }) = boundary.job.as_ref() else { return Vec::new(); }; - if boundary.network != network { + if boundary.network != self.past.network { return Vec::new(); } let (job_id, outcome) = (*job, outcome.clone()); @@ -629,6 +625,7 @@ impl State { info!(log, "session started"; "session_id" => %session_id, "actor" => %actor); let mut session = Session::started(session_id, actor.clone()); if let Commitment::Trusted { boundary } = &self.past.committed + && boundary.network == self.past.network && let Some(committed) = boundary .job .as_ref() @@ -1017,7 +1014,7 @@ impl State { && queued_jobs.len() >= MAX_QUEUED_JOBS && !queued_jobs.contains_key(&job_id) { - // We have no choice; drop the job on the floor. + // We have no choice; discard the job. warn!(log, "too many jobs queued"; "job_id" => %job_id, "max" => MAX_QUEUED_JOBS); } else { // Insert the job into our queue. Every job joins the @@ -1143,15 +1140,11 @@ impl State { ), )); } else if matches!(admission, Admission::Refuse) { - // Below the floor the queue skips forward in chain - // order. A live job gets the terminal skip status; - // a replayed one is history, and re-reporting it - // on every rejoin would grow the message set. if !replayed { - warn!(log, "skipping job below the execution floor"; "job_id" => %job_id); + warn!(log, "skipping a superseded session's job"; "job_id" => %job_id); outputs.push(Output::job_event( self.rack.own_baseboard(), - JobEvent::Skipped(job_id, now, SkipReason::BelowFloor), + JobEvent::Skipped(job_id, now, SkipReason::SessionSuperseded), )); } } else if self @@ -1297,7 +1290,11 @@ fn enter_universe( ) .into(), ); - State::new(own_baseboard, anchors, Past::new(birth, committed)) + State::new( + own_baseboard, + anchors, + Past::new(birth, committed, rumors.network()), + ) } /// Send a message into the gossip set, returning the version it was @@ -1377,7 +1374,7 @@ impl StateManager { let mut initial_state = State::new( own_baseboard.clone(), anchors, - Past::new(Version::new(), Commitment::Absent), + Past::new(Version::new(), Commitment::Absent, initial.network()), )?; initial_state .rack @@ -1724,7 +1721,7 @@ mod test { }, None => Commitment::Absent, }; - Past::new(Version::new(), committed) + Past::new(Version::new(), committed, network(1)) } fn seq(millis: u64) -> SessionSeq { @@ -1795,7 +1792,7 @@ mod test { /// An untrusted record admits nothing, live or replayed. #[test] fn untrusted_admits_nothing() { - let past = Past::new(Version::new(), Commitment::Untrusted); + let past = Past::new(Version::new(), Commitment::Untrusted, network(1)); for live in [true, false] { assert!(matches!( past.screen(SessionId::random(), seq(1), live), @@ -1829,9 +1826,12 @@ mod test { let log = discard(); let now = Utc::now(); let job = JobId::random(); - let mut state = state(past(Some(committed_boundary(job)))); + // A sled that entered another universe never saw the job. + let mut elsewhere = state(past(Some(committed_boundary(job)))); + elsewhere.past.network = network(2); + assert!(elsewhere.settle(&log, now, Some(network(2))).is_empty()); - assert!(state.settle(&log, now, Some(network(2))).is_empty()); + let mut state = state(past(Some(committed_boundary(job)))); let outputs = state.settle(&log, now, Some(network(1))); assert_eq!(outputs.len(), 1); assert!(interrupted(&outputs[0], job)); @@ -2022,11 +2022,92 @@ mod test { assert!(sim.state.session.session().is_none()); } + /// Sessions must not be resumed across networks. + #[tokio::test] + async fn committed_session_does_not_resume_across_networks() { + let mut root = root(); + let anchors = vec![root.cert().clone()]; + let actor = root.key_id().clone(); + let mut sim = Sim::new(&anchors, Vec::new()).await; + + // A previous life, in another network, committed to job A + // of the session and recorded the chain position after it. + let committed_seq = 5; + let committed_preimage = preimage(committed_seq); + let session_id = committed_preimage.session_id(); + let mut signer = Session::new(session_id); + let job_a_id = signer.next_job_id(); + let job_a = signed_job(&mut root, job_a_id, session_id, "true").await; + signer.job_started(job_a.clone()); + let successor_id = signer.next_job_id(); + let other = network(7); + assert_ne!(sim.rumors.network(), other); + sim.state.past.committed = Commitment::Trusted { + boundary: Box::new(Boundary { + network: other, + job: Some(Committed { + session: session_id, + seq: committed_preimage.seq, + job: job_a_id, + successor: successor_id, + outcome: JobOutcome::Committed, + }), + }), + }; + + // The session's start arrives in this network. + sim.request(Request::session( + actor.clone(), + SessionRequest::Start(committed_preimage), + )); + assert!( + sim.state + .session + .session() + .is_some_and(|s| s.session_id() == session_id) + ); + + // If the committed job arrives again, it skips as superseded + // instead of running. + let outputs = sim.request(Request::job( + actor.clone(), + JobRequest::Start(job_a, JobStartParams::default()), + )); + assert!(!outputs.iter().any(|o| matches!(o, Output::Run(_)))); + assert!(outputs.iter().any(|o| matches!( + o, + Output::Gossip(Message::Event( + _, + Event::Job(JobEvent::Skipped(id, _, SkipReason::SessionSuperseded)), + )) if *id == job_a_id + ))); + + // That job's successor also does not run. + let job_b = signed_job(&mut root, successor_id, session_id, "true").await; + let outputs = sim.request(Request::job( + actor.clone(), + JobRequest::Start(job_b, JobStartParams::default()), + )); + assert!(!outputs.iter().any(|o| matches!(o, Output::Run(_)))); + + // A newer session outranks the committed one and runs. + let next = preimage(committed_seq + 1); + let next_id = next.session_id(); + sim.request(Request::session(actor.clone(), SessionRequest::Start(next))); + let job_c_id = Session::new(next_id).next_job_id(); + let job_c = signed_job(&mut root, job_c_id, next_id, "true").await; + let outputs = sim.request(Request::job( + actor, + JobRequest::Start(job_c, JobStartParams::default()), + )); + assert!(outputs.iter().any(|o| matches!(o, Output::Run(_)))); + } + /// A session at or below the committed sequence number is - /// refused: its jobs skip below the floor rather than run, and - /// the chain still advances so the session is not wedged. + /// refused: its jobs skip as superseded rather than run, and the + /// chain still advances so the session is not wedged. #[tokio::test] - async fn floor_refuses_through_step() { + async fn superseded_session_skips_through_step() { let (mut root, mut sim, session_id, mut session) = sim_with_session().await; let actor = root.key_id().clone(); @@ -2051,7 +2132,7 @@ mod test { o, Output::Gossip(Message::Event( _, - Event::Job(JobEvent::Skipped(id, _, SkipReason::BelowFloor)), + Event::Job(JobEvent::Skipped(id, _, SkipReason::SessionSuperseded)), )) if *id == job_id ) }) diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index f97e700a..fbb4d5ce 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -675,7 +675,7 @@ async fn stragglers_do_not_interrupt_live_jobs() { /// Wait until `anchor` has applied `joiner`'s build announcement. A /// session started on `anchor` afterward causally follows everything -/// `joiner` held at entry, so it clears the joiner's entry floor. +/// `joiner` held at entry, so the joiner serves it. /// Everything one sled hears at session start: a client nonce, a /// signer nonce, the signer's clock reading, and the session id over /// them all. @@ -754,8 +754,8 @@ async fn lost_suffix_never_reruns() { let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; // Sled B keeps its boundary in a locker. It joins before the - // session starts: a record-less sled raises its floor at entry, - // and serves only sessions started after it arrived. + // session starts: a record-less sled serves only sessions + // that started after it arrived. let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); let b_shutdown = CancellationToken::new(); @@ -1076,7 +1076,7 @@ async fn session_resumes_at_stored_successor() { #[named] #[tokio::test] -async fn seq_floor_crosses_universes() { +async fn supersession_crosses_universes() { let (_tmp, dir) = pki("sush-flipflop-", 3); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); @@ -1229,13 +1229,13 @@ async fn seq_floor_crosses_universes() { .job_start(&authn_x, j3, JobStartParams::default()) .await .unwrap(); - eventually("the floor skips the old session", 120, async || { + eventually("the old session is superseded", 120, async || { a.mgr.job_status(&authn_a, &j3_id).await.is_ok_and(|map| { map.get(&x.baseboard).is_some_and(|s| { matches!( s, JobStatus::Skipped { - reason: SkipReason::BelowFloor, + reason: SkipReason::SessionSuperseded, .. } ) @@ -1281,7 +1281,7 @@ async fn seq_floor_crosses_universes() { map.get(&x.baseboard) .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) }), - "a session started above the floor must serve" + "a session above the committed one must serve" ); // X flops back to D's universe a second time. The job X served in @@ -1308,13 +1308,13 @@ async fn seq_floor_crosses_universes() { .job_start(&authn_x, j5, JobStartParams::default()) .await .unwrap(); - eventually("the floor skips session two as well", 120, async || { + eventually("session two is superseded as well", 120, async || { x.mgr.job_status(&authn_x, &j5_id).await.is_ok_and(|map| { map.get(&x.baseboard).is_some_and(|s| { matches!( s, JobStatus::Skipped { - reason: SkipReason::BelowFloor, + reason: SkipReason::SessionSuperseded, .. } ) diff --git a/server/tests/output/already-handled-skip-event.bin b/server/tests/output/already-handled-skip-event.bin deleted file mode 100644 index 7eed01d49db074b43ba0c26ef7873b995d474673..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 149 zcmZ3O6lSn6)wL`&ucT>Fc0poMNqk;uZc=Jdwxyx5t^o)bTILp~7G);pz+{|)d`*mu zj0_Bn4HhPQIQt8h_qYH@L9ex7b|X+bKGlaiVO0EK1>D*ylh delta 19 acmbQiIFWHeD^GS(YEFK+Zdy)$eh~mfdIul? diff --git a/sush.json b/sush.json index 905fb383..1736962b 100644 --- a/sush.json +++ b/sush.json @@ -1619,27 +1619,10 @@ }, "SkipReason": { "description": "Why a sled will never run a job.", - "oneOf": [ - { - "type": "string", - "enum": [ - "session-ended" - ] - }, - { - "description": "The job's session sits at or below the sled's execution floor, where the sled cannot tell replay from re-run.", - "type": "string", - "enum": [ - "below-floor" - ] - }, - { - "description": "The job's chain position precedes the sled's recorded commitment: a previous life already handled it.", - "type": "string", - "enum": [ - "already-handled" - ] - } + "type": "string", + "enum": [ + "session-superseded", + "session-ended" ] }, "SledHealth": {