diff --git a/Cargo.lock b/Cargo.lock index e668d409..5c005c2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1173,6 +1173,16 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -2685,6 +2695,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308d96db8debc727c3fd9744aac51751243420e46edf401010908da7f8d5e57c" +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "newtype-uuid" version = "1.4.0" @@ -2799,32 +2815,51 @@ dependencies = [ ] [[package]] -name = "oauth2" -version = "5.0.0" +name = "objc2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" dependencies = [ - "base64 0.22.1", - "chrono", - "getrandom 0.2.17", - "http 1.5.0", - "rand 0.8.7", - "serde", - "serde_json", - "serde_path_to_error", - "sha2", - "thiserror 1.0.69", - "url", + "objc2-encode", ] [[package]] -name = "oauth2-reqwest" -version = "0.1.0-alpha.3" +name = "objc2-app-kit" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234fb5c965bbce983ee5de636a7a51d6a3223da8067ea02f9ab2d2d78ac08be2" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "oauth2", - "reqwest", + "bitflags 2.13.1", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", ] [[package]] @@ -3008,7 +3043,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "permission-slip-client" version = "1.1.0" -source = "git+https://github.com/oxidecomputer/permission-slip?rev=c9d302ef44576e075e35f4f28f8e7ac8345bcd4b#c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" +source = "git+https://github.com/oxidecomputer/permission-slip?rev=94b68e717f2da426ca1450d5ebc9fc5000596bdc#94b68e717f2da426ca1450d5ebc9fc5000596bdc" dependencies = [ "anyhow", "base64 0.22.1", @@ -3022,8 +3057,6 @@ dependencies = [ "hex", "lpc55_sign", "natord", - "oauth2", - "oauth2-reqwest", "once_cell", "permission-slip-common", "progenitor", @@ -3037,6 +3070,7 @@ dependencies = [ "sush-common", "tokio", "uuid", + "webbrowser", "x509-cert", "xdg 2.5.2", ] @@ -3044,7 +3078,7 @@ dependencies = [ [[package]] name = "permission-slip-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/permission-slip?rev=c9d302ef44576e075e35f4f28f8e7ac8345bcd4b#c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" +source = "git+https://github.com/oxidecomputer/permission-slip?rev=94b68e717f2da426ca1450d5ebc9fc5000596bdc#94b68e717f2da426ca1450d5ebc9fc5000596bdc" dependencies = [ "aws-types", "blake3", @@ -4617,7 +4651,7 @@ dependencies = [ [[package]] name = "sshauth" version = "0.1.2" -source = "git+https://github.com/oxidecomputer/sshauth?rev=afd9b4e549d0e9229c2910c94aa7bca76623dfdb#afd9b4e549d0e9229c2910c94aa7bca76623dfdb" +source = "git+https://github.com/oxidecomputer/sshauth#6233c1b25d9e3b03d51ab95b19ed7903bfadfd8f" dependencies = [ "anyhow", "base64 0.22.1", @@ -5587,7 +5621,6 @@ dependencies = [ "idna", "percent-encoding", "serde", - "serde_derive", ] [[package]] @@ -5763,6 +5796,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" +dependencies = [ + "jni", + "log", + "ndk-context", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "url", + "web-sys", +] + [[package]] name = "webpki-root-certs" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index 65872a62..81f90277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,8 +40,8 @@ memmap2 = "0.9" p256 = { version = "0.13", features = ["ecdsa", "pkcs8"] } pem-rfc7468 = { version = "0.7", features = ["std"] } percent-encoding = "2" -permission-slip-client = { git = "https://github.com/oxidecomputer/permission-slip", rev = "c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" } -permission-slip-common = { git = "https://github.com/oxidecomputer/permission-slip", rev = "c9d302ef44576e075e35f4f28f8e7ac8345bcd4b" } +permission-slip-client = { git = "https://github.com/oxidecomputer/permission-slip", rev = "94b68e717f2da426ca1450d5ebc9fc5000596bdc", features = ["sush-types"] } +permission-slip-common = { git = "https://github.com/oxidecomputer/permission-slip", rev = "94b68e717f2da426ca1450d5ebc9fc5000596bdc" } pki-playground = { git = "https://github.com/oxidecomputer/pki-playground", rev = "7600756029ce046a02c6234aa84ce230cc5eaa04" } progenitor = "0.14" progenitor-client = "0.14" diff --git a/api/src/lib.rs b/api/src/lib.rs index 8ddc8932..5c437131 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -21,8 +21,8 @@ use sled_hardware_types::BaseboardId; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobLimits, JobOutputStream, JobStatus, JsonJobStatusMap, Session, SessionId, - SessionSignerNonce, SessionSushNonce, SignedJob, + Access, JobId, JobLimits, JobOutputStream, JobStatus, JsonJobStatusMap, Session, + SessionClientNonce, SessionId, SessionSeq, SessionServerNonce, SessionSignerNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::SledVersion; @@ -74,15 +74,16 @@ pub trait SushApi { headers: Header, ) -> Result, HttpError>; - /// Get the sush server nonce needed to start a new session. + /// Get a fresh server nonce for the session ceremony. /// - /// The nonce will need to be sent to the signer server, which will give back its own nonce. - /// Both nonces combined (along with the baseboard ID) will form the session ID. - #[endpoint { method = GET, path = "/session-nonce" }] - async fn session_start_nonce( + /// The client mixes its own nonce with this one to form the sush + /// nonce it submits to the signer. Fetching replaces the previous + /// server nonce, so only the latest ceremony can complete. + #[endpoint { method = GET, path = "/server-nonce" }] + async fn server_nonce( ctx: RequestContext, headers: Header, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Start a new support session. /// @@ -287,14 +288,11 @@ pub struct JobTargetParams { pub target: String, } -#[derive(Serialize, JsonSchema)] -pub struct SessionStartNonce { - pub nonce: SessionSushNonce, -} - #[derive(Deserialize, JsonSchema)] pub struct SessionStartBody { + pub client_nonce: SessionClientNonce, pub signer_nonce: SessionSignerNonce, + pub seq: SessionSeq, } /// Job parameters _not_ specified in the signed job request. diff --git a/client/src/cli.rs b/client/src/cli.rs index 4587363e..14be9cd8 100644 --- a/client/src/cli.rs +++ b/client/src/cli.rs @@ -17,7 +17,7 @@ use anstream::print; use anstyle::{AnsiColor, Style}; use atomicwrites::{AtomicFile, OverwriteBehavior}; use bytesize::ByteSize; -use chrono::{DateTime, TimeDelta, Utc}; +use chrono::TimeDelta; use humantime::format_duration; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use rustix::io::ioctl_fionread; @@ -30,8 +30,9 @@ use xdg::BaseDirectories; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobOutputState, JobOutputStream, JobStatus, JobStatusMap, Session, SessionId, - SessionSignerNonce, SignedJob, job_status_to_json_map, + Access, JobId, JobOutputState, JobOutputStream, JobStartRequest, JobStatus, JobStatusMap, + Session, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, + SignedJob, job_status_to_json_map, }; use sush_common::keys::{KeyId, Signature, SshPublicKey}; use sush_common::targets::{MAX_CUBBY, SledHealth, SledId, SledVersion}; @@ -40,18 +41,12 @@ use sush_common::version::VersionInfo; use crate::AuthzSigner; use crate::commands::{CommandError, GlobalArgs}; use crate::context::{CommandContext, OutputFormat, StatusDisplayStyle}; -use crate::types::SessionStartNonce; pub(crate) const PREFIX: &str = "sush"; const SESSION_FILE_NAME: &str = "session.json"; const SESSION_FILE_VERSION: u32 = 1; -const TOKEN_FILE_NAME: &str = "permslip-token.json"; -const TOKEN_FILE_VERSION: u32 = 1; - -/// How long a permslip token is trusted for reuse. Staying well -/// under the server's 15 minute TTL spares a command from expiring -/// mid-flight. -const TOKEN_REUSE: TimeDelta = TimeDelta::minutes(10); +const CLIENT_NONCE_FILE_NAME: &str = "client-nonce.json"; +const CLIENT_NONCE_FILE_VERSION: u32 = 1; /// The persisted session, versioned for future migrations. #[derive(Deserialize, Serialize)] @@ -60,15 +55,14 @@ struct SavedSession { session: Session, } -/// A persisted permslip token, versioned for future migrations. The -/// url and fingerprint pin it to one signing server and one identity. +/// The persisted client nonce of the pending session ceremony, +/// versioned for future migrations. One pending ceremony at a time, +/// mirroring the rack's one server nonce slot: `session start-params` +/// overwrites it just as the fetch it makes overwrites the slot. #[derive(Deserialize, Serialize)] -struct SavedToken { +struct SavedClientNonce { version: u32, - url: String, - fingerprint: String, - token: String, - created: DateTime, + client_nonce: SessionClientNonce, } #[derive(Clone, Debug, Default)] @@ -79,7 +73,7 @@ pub struct Cli { watch: Arc>>, session: Arc>>, session_file: Option, - token_file: Option, + client_nonce_file: Option, credentials: AuthzSigner, } @@ -110,9 +104,9 @@ impl Cli { Err(error) => eprintln!("❗ Ignoring the saved session: {error}"), } self.session_file = Some(path); - match BaseDirectories::with_prefix(PREFIX).place_state_file(TOKEN_FILE_NAME) { - Ok(path) => self.token_file = Some(path), - Err(error) => eprintln!("❗ Signing tokens will not persist: {error}"), + match BaseDirectories::with_prefix(PREFIX).place_state_file(CLIENT_NONCE_FILE_NAME) { + Ok(path) => self.client_nonce_file = Some(path), + Err(error) => eprintln!("❗ The client nonce will not persist: {error}"), } } @@ -276,40 +270,29 @@ impl CommandContext for Cli { self.credentials.clone() } - fn permslip_token(&self, url: &str, fingerprint: &str) -> Option { - let json = fs::read(self.token_file.as_ref()?).ok()?; - match serde_json::from_slice::(&json) { - Ok(SavedToken { - version: TOKEN_FILE_VERSION, - url: saved_url, - fingerprint: saved_fingerprint, - token, - created, - }) if saved_url == url - && saved_fingerprint == fingerprint - && (TimeDelta::zero()..TOKEN_REUSE).contains(&(Utc::now() - created)) => - { - Some(token) - } + fn pending_client_nonce(&self) -> Option { + let json = fs::read(self.client_nonce_file.as_ref()?).ok()?; + match serde_json::from_slice::(&json) { + Ok(SavedClientNonce { + version: CLIENT_NONCE_FILE_VERSION, + client_nonce, + }) => Some(client_nonce), _ => None, } } - fn save_permslip_token(&self, url: &str, fingerprint: &str, token: &str) { - let Some(path) = &self.token_file else { + fn save_client_nonce(&self, client_nonce: SessionClientNonce) { + let Some(path) = &self.client_nonce_file else { return; }; - let result = serde_json::to_vec_pretty(&SavedToken { - version: TOKEN_FILE_VERSION, - url: url.to_owned(), - fingerprint: fingerprint.to_owned(), - token: token.to_owned(), - created: Utc::now(), + let result = serde_json::to_vec_pretty(&SavedClientNonce { + version: CLIENT_NONCE_FILE_VERSION, + client_nonce, }) .map_err(io::Error::other) .and_then(|json| write_private(path, &json)); if let Err(error) = result { - eprintln!("❗ The token was not saved: {error}"); + eprintln!("❗ The client nonce was not saved: {error}"); } } @@ -329,32 +312,68 @@ impl CommandContext for Cli { } } - fn session_start_params(&self, baseboard_id: BaseboardId, nonce: SessionStartNonce) { + fn session_start_params( + &self, + baseboard_id: BaseboardId, + client_nonce: SessionClientNonce, + sush_nonce: SessionSushNonce, + ) { match self.get_output_format() { OutputFormat::Json => println!( "{}", json!({ "baseboard_id": &baseboard_id.to_string(), - "sush_nonce": &nonce.nonce, + "client_nonce": &client_nonce, + "sush_nonce": &sush_nonce, }) ), OutputFormat::Text => { - println!("Baseboard ID: {baseboard_id}"); - println!("Sush nonce: {}", nonce.nonce); + println!("✅ Please relay these parameters to support:"); + println!(" Baseboard ID: {baseboard_id}"); + println!(" Sush nonce: {sush_nonce}"); } } } - fn session_created(&mut self, session: Session, signer_nonce: SessionSignerNonce) { + fn job_start_params(&self, request: &JobStartRequest) { + match self.get_output_format() { + OutputFormat::Json => println!( + "{}", + json!({ + "job_id": &request.job_id, + "session_id": &request.session_id, + "command": &request.command, + "mode": &request.mode, + "target": &request.target.to_string(), + }) + ), + OutputFormat::Text => { + println!("✅ Please request a job with these parameters from support:"); + println!(" Job ID: {}", request.job_id); + println!(" Session ID: {}", request.session_id); + println!(" Command: {}", request.command); + println!(" Mode: {}", request.mode.as_str()); + println!(" Target: {}", request.target); + } + } + } + + fn session_created( + &mut self, + session: Session, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, + ) { let session_id = self.adopt_session(session, true); match self.get_output_format() { OutputFormat::Json => println!( "{}", - json!({"session_created": session_id, "signer_nonce": signer_nonce}) + json!({"session_created": session_id, "signer_nonce": signer_nonce, "seq": seq}) ), OutputFormat::Text => { println!("✅ Session is now `{session_id}`"); - println!(" Signer nonce: {signer_nonce}"); + println!(" Signer nonce: {signer_nonce}"); + println!(" Sequence number: {seq}"); } } } diff --git a/client/src/commands.rs b/client/src/commands.rs index 626c9738..71252fd3 100644 --- a/client/src/commands.rs +++ b/client/src/commands.rs @@ -7,7 +7,6 @@ //! May be executed via either the main CLI or the interactive REPL. use std::fs::{File, OpenOptions, read}; -#[cfg(feature = "permslip")] use std::io::ErrorKind; use std::io::{Seek as _, SeekFrom, Write as _, stdin}; use std::net::SocketAddr; @@ -45,16 +44,13 @@ use sush_api::JobWait; use sush_common::authn::{ AuthnError, Challenge, ChallengeResponse, Credentials, Identity, RequestKey, }; -#[cfg(feature = "permslip")] -use sush_common::codephrases::InvalidCodephrase; use sush_common::interactive::{InteractiveJobError, InteractiveJobMessage}; use sush_common::jobs::JobOutputStream::{self, Stderr, Stdout}; use sush_common::jobs::{ - Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStatus, JobStatusMap, - Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, job_status_try_from_json_map, + Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStartRequest, JobStatus, + JobStatusMap, Session, SessionClientNonce, SessionId, SessionSeq, SessionSignerNonce, + SessionSushNonce, SignedJob, SkipReason, job_status_try_from_json_map, }; -#[cfg(feature = "permslip")] -use sush_common::jobs::{JobStartRequest, SessionSushNonce}; use sush_common::keys::{KeyError, KeyId, Signer as _}; use sush_common::targets::{SledId, Target}; use sush_common::version::VersionInfo; @@ -64,7 +60,7 @@ use crate::context::{Authz, CommandContext, OutputFormat, StatusDisplayStyle}; use crate::identity::{IdentityError, SshAgentConnection}; use crate::interactive::interactive_job; #[cfg(feature = "permslip")] -use crate::permslip::{PermslipError, PermslipSigner, fresh_token}; +use crate::permslip::{PermslipError, PermslipSigner, saved_token}; use crate::repl::Repl; use crate::tls; use crate::tunnel::{Tunnel, TunnelError}; @@ -536,15 +532,14 @@ pub enum SessionCommand { key_id: KeyId, }, - /// Create a session at the signing service from a rack's start - /// parameters. + /// Create a session with the signing service from `session start-params`. #[cfg(feature = "permslip")] Create { /// The baseboard ID of the sled that generated the nonce. #[arg(value_parser = parse_baseboard_id)] baseboard_id: BaseboardId, - /// The rack's session nonce. + /// The sush nonce from `session start-params`. nonce: SessionSushNonce, /// Use `permslip` to sign sessions and jobs with this key name. @@ -566,8 +561,15 @@ pub enum SessionCommand { session_id: Option, /// The signer nonce for the session. + #[arg(requires = "seq")] nonce: Option, + /// The session sequence number assigned by the signer. + seq: Option, + + /// Override the client nonce generated and stashed by `session start-params`. + client_nonce: Option, + /// Use `permslip` to sign sessions and jobs with this key name. #[cfg(feature = "permslip")] #[arg(short, long, env = SUSH_PERMSLIP_KEY, value_name = "KEY_NAME")] @@ -737,10 +739,11 @@ pub struct JobStartArgs { } impl JobStartArgs { - /// The signing key name in a build that can sign. - fn key_name(&self) -> Option<&str> { + /// The signing service URL in a build that can sign. Its absence + /// means an out-of-band request to the signer (support). + fn signer_url(&self) -> Option<&str> { #[cfg(feature = "permslip")] - return self.permslip.as_deref(); + return self.permslip_url.as_deref(); #[cfg(not(feature = "permslip"))] None } @@ -1085,23 +1088,15 @@ async fn session( Ok(()) } - (SessionCommand::StartParams, Some(client)) => { - let (baseboard_id, nonce) = with_login(ctx, client, async || { - Ok(( - client.target().send().await?.into_inner(), - client.session_start_nonce().send().await?.into_inner(), - )) - }) - .await?; - ctx.session_start_params(baseboard_id, nonce); - Ok(()) - } + (SessionCommand::StartParams, Some(client)) => session_start_params(ctx, client).await, #[cfg(feature = "permslip")] ( SessionCommand::Start { session_id, nonce, + seq, + client_nonce, permslip, permslip_url, wait, @@ -1110,24 +1105,52 @@ async fn session( ) => { // Creation already announces the session; don't echo it // when the start succeeds. - let (session_id, nonce, show) = if let (Some(session_id), Some(nonce)) = - (session_id, nonce) - { - (session_id, nonce, true) - } else { - let (baseboard_id, nonce) = with_login(ctx, client, async || { - Ok(( - client.target().send().await?.into_inner(), - client.session_start_nonce().send().await?.into_inner(), - )) - }) - .await?; - let (session_id, nonce) = - session_create(ctx, permslip, permslip_url, &baseboard_id, nonce.nonce).await?; - ctx.session_created(Session::new(session_id), nonce); - (session_id, nonce, false) - }; - session_start(ctx, client, session_id, nonce, wait, show).await + let (session_id, body, show) = + if let (Some(session_id), Some(signer_nonce), Some(seq)) = (session_id, nonce, seq) + { + let client_nonce = client_nonce + .or_else(|| ctx.pending_client_nonce()) + .ok_or(CommandError::MissingClientNonce)?; + ( + session_id, + SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + true, + ) + } else { + let Some(permslip_url) = permslip_url else { + return session_start_params(ctx, client).await; + }; + let Some(permslip_key) = permslip else { + return Err(CommandError::MissingKeyName); + }; + let (baseboard_id, nonce) = with_login(ctx, client, async || { + Ok(( + client.target().send().await?.into_inner(), + client.server_nonce().send().await?.into_inner(), + )) + }) + .await?; + let client_nonce = SessionClientNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); + let (session_id, signer_nonce, seq) = + session_create(permslip_key, permslip_url, &baseboard_id, sush_nonce) + .await?; + ctx.session_created(Session::new(session_id), signer_nonce, seq); + ( + session_id, + SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + false, + ) + }; + session_start(ctx, client, session_id, body, wait, show).await } #[cfg(feature = "permslip")] @@ -1140,9 +1163,15 @@ async fn session( }, _, ) => { - let (session_id, nonce) = - session_create(ctx, permslip, permslip_url, &baseboard_id, nonce).await?; - ctx.session_created(Session::new(session_id), nonce); + let Some(permslip_url) = permslip_url else { + return Err(CommandError::MissingPermslipUrl); + }; + let Some(permslip_key) = permslip else { + return Err(CommandError::MissingKeyName); + }; + let (session_id, nonce, seq) = + session_create(permslip_key, permslip_url, &baseboard_id, nonce).await?; + ctx.session_created(Session::new(session_id), nonce, seq); Ok(()) } @@ -1151,16 +1180,33 @@ async fn session( SessionCommand::Start { session_id, nonce, + seq, + client_nonce, wait, }, Some(client), ) => { - let (session_id, nonce) = if let (Some(session_id), Some(nonce)) = (session_id, nonce) { - (session_id, nonce) - } else { - return Err(CommandError::SigningUnavailable); + // A build that cannot sign is always in the relay flow: a + // bare start means the start parameters. + let (Some(session_id), Some(signer_nonce), Some(seq)) = (session_id, nonce, seq) else { + return session_start_params(ctx, client).await; }; - session_start(ctx, client, session_id, nonce, wait, true).await + let client_nonce = client_nonce + .or_else(|| ctx.pending_client_nonce()) + .ok_or(CommandError::MissingClientNonce)?; + session_start( + ctx, + client, + session_id, + SessionStartBody { + client_nonce, + signer_nonce, + seq, + }, + wait, + true, + ) + .await } (SessionCommand::Allow { key_id, write }, Some(client)) => { @@ -1226,8 +1272,10 @@ async fn job( command: JobCommand, ) -> Result<(), CommandError> { match (command, client) { + // Without a signing service, a bare job start takes a job the + // signer already signed. (JobCommand::Start { start_args }, Some(client)) - if start_args.command.is_none() && start_args.key_name().is_none() => + if start_args.command.is_none() && start_args.signer_url().is_none() => { let job = ctx.read_signed_job()?; job_start(ctx, client, job, start_args).await?; @@ -1242,6 +1290,42 @@ async fn job( Some(_), ) => Err(CommandError::MissingCommand), + // Without a signing service, a command's job start prints the + // request parameters for the user to relay to support. + #[cfg(feature = "permslip")] + ( + JobCommand::Start { + start_args: + ref start_args @ JobStartArgs { + command: Some(_), + permslip_url: None, + .. + }, + }, + client, + ) => { + let request = job_start_request(ctx, client.as_ref(), start_args).await?; + ctx.job_start_params(&request); + Ok(()) + } + + // A build that cannot sign prints the request parameters for + // the user to relay to support. + #[cfg(not(feature = "permslip"))] + ( + JobCommand::Start { + start_args: + ref start_args @ JobStartArgs { + command: Some(_), .. + }, + }, + client, + ) => { + let request = job_start_request(ctx, client.as_ref(), start_args).await?; + ctx.job_start_params(&request); + Ok(()) + } + #[cfg(feature = "permslip")] ( JobCommand::Start { @@ -1251,80 +1335,24 @@ async fn job( Some(_), ) => Err(CommandError::MissingKeyName), - #[cfg(not(feature = "permslip"))] - (JobCommand::Start { .. }, Some(_)) => Err(CommandError::SigningUnavailable), - #[cfg(feature = "permslip")] ( JobCommand::Start { start_args: ref start_args @ JobStartArgs { - command: Some(ref command), permslip: Some(ref key_name), - ref permslip_url, - ref interactive, - ref streaming, - ref target, + permslip_url: Some(ref permslip_url), .. }, }, client, ) => { - let Some(session_id) = ctx.session_id() else { - return Err(CommandError::MissingSession); - }; - let Some(permslip_url) = permslip_url else { - return Err(CommandError::MissingPermslipUrl); - }; - let job_id = ctx.next_job_id()?; - let target = match target { - TargetArg::Target(target) => target.clone(), - abbreviated => { - let Some(client) = client.as_ref() else { - return Err(CommandError::Offline); - }; - resolve_target_arg(client, abbreviated).await? - } - }; - // Sign an interactive or streaming job for the sled its - // attachment will land on. - let target = if (*interactive || *streaming) && target.single_baseboard().is_none() { - let Some(client) = client.as_ref() else { - return Err(CommandError::InteractiveTarget); - }; - Target::from(resolve_target(client, &target).await?) - } else { - target - }; - // Catch output file problems before the signing ceremony. - if *streaming { - let Some(path) = &start_args.file else { - return Err(CommandError::StreamingNeedsFile); - }; - if !start_args.force && path.exists() { - return Err(CommandError::io(path, ErrorKind::AlreadyExists.into())); - } - } - if let Some(client) = client.as_ref() { - preflight_target(ctx, client, &target).await?; - } - let mode = if *interactive { - JobMode::Interactive - } else if *streaming { - JobMode::StreamOutput - } else { - JobMode::Batch - }; - let signer = permslip_signer(ctx, key_name, permslip_url).await?; + let request = job_start_request(ctx, client.as_ref(), start_args).await?; + let job_id = request.job_id; + let signer = permslip_signer(key_name, permslip_url).await?; let mut interval = interval(SIGNING_UPDATE_INTERVAL); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - let sign = signer.sign_job_request(JobStartRequest::new( - job_id.to_owned(), - session_id, - command, - mode, - target, - )); + let sign = signer.sign_job_request(request); pin!(sign); ctx.job_signing_started(&job_id); let mut sigint = signal(SignalKind::interrupt())?; @@ -1690,65 +1718,115 @@ fn parse_baseboard_id(value: &str) -> Result { value.parse::().map_err(|e| e.to_string()) } -/// A permslip signer authenticated as the agent key `-s`/`SUSH_KEY_ID` -/// names, or the agent's first. We choose the key rather than letting -/// permslip choose, so that one identity serves both the rack and the -/// signing service. +/// A permslip signer authenticated by the credential that +/// `permslip login` saved. #[cfg(feature = "permslip")] async fn permslip_signer( - ctx: &mut impl CommandContext, key_name: &str, permslip_url: &str, ) -> Result { - let globals = ctx.get_globals(); - let Some(sock) = globals.ssh_auth_sock.clone() else { - return Err(CommandError::MissingSshAuthSock); + let token = saved_token(permslip_url).await?; + Ok(PermslipSigner::new(key_name, permslip_url, &token)?) +} + +/// Fetch the ceremony inputs from the rack and print the parameters +/// to relay to the signer, stashing the client nonce for the +/// eventual start. +async fn session_start_params( + ctx: &mut impl CommandContext, + client: &Client, +) -> Result<(), CommandError> { + let (baseboard_id, nonce) = with_login(ctx, client, async || { + Ok(( + client.target().send().await?.into_inner(), + client.server_nonce().send().await?.into_inner(), + )) + }) + .await?; + let client_nonce = SessionClientNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, nonce); + ctx.save_client_nonce(client_nonce); + ctx.session_start_params(baseboard_id, client_nonce, sush_nonce); + Ok(()) +} + +/// Assemble the request a job start submits for signing: the +/// session's next job id, the resolved target, and the mode the +/// flags name. +async fn job_start_request( + ctx: &mut impl CommandContext, + client: Option<&Client>, + start_args: &JobStartArgs, +) -> Result { + let Some(command) = &start_args.command else { + return Err(CommandError::MissingCommand); }; - let key_id = globals.ssh_key_id.clone(); - let mut agent = SshAgentConnection::connect(&sock).await?; - let key = agent.identity(key_id.as_ref()).await?; - let fingerprint = key.fingerprint(); - let token = match ctx.permslip_token(permslip_url, &fingerprint) { - Some(token) => token, - None => { - ctx.please_touch(&key)?; - let token = fresh_token(permslip_url, sock, fingerprint.clone()).await?; - ctx.save_permslip_token(permslip_url, &fingerprint, &token); - token + let Some(session_id) = ctx.session_id() else { + return Err(CommandError::MissingSession); + }; + let job_id = ctx.next_job_id()?; + let (interactive, streaming) = (start_args.interactive, start_args.streaming); + let target = match &start_args.target { + TargetArg::Target(target) => target.clone(), + abbreviated => { + let Some(client) = client else { + return Err(CommandError::Offline); + }; + resolve_target_arg(client, abbreviated).await? } }; - Ok(PermslipSigner::new(key_name, permslip_url, &token)?) + // An interactive or streaming job binds to the single sled its + // attachment will land on. + let target = if (interactive || streaming) && target.single_baseboard().is_none() { + let Some(client) = client else { + return Err(CommandError::InteractiveTarget); + }; + Target::from(resolve_target(client, &target).await?) + } else { + target + }; + // Catch output file problems before anything is signed. + if streaming { + let Some(path) = &start_args.file else { + return Err(CommandError::StreamingNeedsFile); + }; + if !start_args.force && path.exists() { + return Err(CommandError::io(path, ErrorKind::AlreadyExists.into())); + } + } + if let Some(client) = client { + preflight_target(ctx, client, &target).await?; + } + let mode = if interactive { + JobMode::Interactive + } else if streaming { + JobMode::StreamOutput + } else { + JobMode::Batch + }; + Ok(JobStartRequest::new( + job_id, session_id, command, mode, target, + )) } /// Ask the online signing service to create a session. #[cfg(feature = "permslip")] async fn session_create( - ctx: &mut impl CommandContext, - permslip: Option, - permslip_url: Option, + permslip_key: String, + permslip_url: String, baseboard_id: &BaseboardId, nonce: SessionSushNonce, -) -> Result<(SessionId, SessionSignerNonce), CommandError> { - let Some(permslip_url) = permslip_url else { - return Err(CommandError::MissingPermslipUrl); - }; - let Some(permslip_key) = permslip else { - return Err(CommandError::MissingKeyName); - }; - let invalid = |e: InvalidCodephrase| CommandError::UnsupportedPermslipResponse(e.to_string()); - let signer = permslip_signer(ctx, &permslip_key, &permslip_url).await?; +) -> Result<(SessionId, SessionSignerNonce, SessionSeq), CommandError> { + let signer = permslip_signer(&permslip_key, &permslip_url).await?; let created = signer.create_session(baseboard_id, nonce).await?; - Ok(( - created.session_id.to_string().parse().map_err(invalid)?, - created.signer_nonce.to_string().parse().map_err(invalid)?, - )) + Ok((created.session_id, created.signer_nonce, created.seq)) } async fn session_start( ctx: &mut impl CommandContext, client: &Client, session_id: SessionId, - signer_nonce: SessionSignerNonce, + body: SessionStartBody, wait: bool, show: bool, ) -> Result<(), CommandError> { @@ -1758,7 +1836,7 @@ async fn session_start( .session_start() .session_id(session.session_id()) .wait(wait) - .body(SessionStartBody { signer_nonce }) + .body(body.clone()) .send() .await }) @@ -2345,7 +2423,6 @@ impl FromStr for TargetArg { } /// Confirm before signing a job for a sled that doesn't appear in inventory. -#[cfg(feature = "permslip")] async fn preflight_target( ctx: &mut impl CommandContext, client: &Client, @@ -2496,6 +2573,8 @@ pub enum CommandError { Key(#[from] KeyError), #[error("❌ Length mismatch: expected {expected} bytes, received {received}")] LengthMismatch { expected: u64, received: u64 }, + #[error("❌ Missing client nonce: run `session start-params` first, or pass one")] + MissingClientNonce, #[error("❌ Missing job command, try `--help`")] MissingCommand, #[cfg(feature = "permslip")] @@ -2546,9 +2625,6 @@ pub enum CommandError { Reqwest(#[from] reqwest::Error), #[error("❌ SSH signature error on key ID `{0}`")] Signature(KeyId), - #[cfg(not(feature = "permslip"))] - #[error("❌ Signing unavailable: built without the `permslip` feature")] - SigningUnavailable, #[error("❌ Streaming jobs need `--file`")] StreamingNeedsFile, #[error("❌ Job `{0}` did not exit cleanly, streamed output is unverified")] @@ -2575,9 +2651,6 @@ pub enum CommandError { Utf8(#[from] std::string::FromUtf8Error), #[error("❌ WebSocket error: {0}")] WebSocket(#[from] WebSocketError), - #[cfg(feature = "permslip")] - #[error("❌ Unsupported permslip response: {0}")] - UnsupportedPermslipResponse(String), } impl CommandError { diff --git a/client/src/context.rs b/client/src/context.rs index 44db4a40..c5fc942d 100644 --- a/client/src/context.rs +++ b/client/src/context.rs @@ -14,7 +14,8 @@ use x509_cert::Certificate; use sush_common::authn::{BoundRequest, Credentials, Identity, RequestKey}; use sush_common::jobs::{ - Access, JobId, JobOutputStream, JobStatusMap, Session, SessionId, SessionSignerNonce, SignedJob, + Access, JobId, JobOutputStream, JobStartRequest, JobStatusMap, Session, SessionClientNonce, + SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::{SledId, SledVersion}; @@ -22,7 +23,6 @@ use sush_common::version::VersionInfo; use crate::AuthzSigner; use crate::commands::{CommandError, GlobalArgs}; -use crate::types::SessionStartNonce; /// Authorization state: the credentials that authenticated us, and the /// ephemeral key that binds each request we make. @@ -113,17 +113,25 @@ pub trait CommandContext: Clone + Send + Sync { fn set_credentials(&mut self, credentials: Option) { self.authz_signer().set(credentials) } - /// The cached permslip token for `url` and the key `fingerprint` - /// names, if it is still fresh. The default caches nothing. - fn permslip_token(&self, _url: &str, _fingerprint: &str) -> Option { + fn pending_client_nonce(&self) -> Option { None } - /// Remember a permslip token for reuse. - fn save_permslip_token(&self, _url: &str, _fingerprint: &str, _token: &str) {} + fn save_client_nonce(&self, _client_nonce: SessionClientNonce) {} fn session_id(&self) -> Option; fn next_job_id(&self) -> Result; - fn session_start_params(&self, baseboard_id: BaseboardId, nonce: SessionStartNonce); - fn session_created(&mut self, session: Session, signer_nonce: SessionSignerNonce); + fn session_start_params( + &self, + baseboard_id: BaseboardId, + client_nonce: SessionClientNonce, + sush_nonce: SessionSushNonce, + ); + fn job_start_params(&self, request: &JobStartRequest); + fn session_created( + &mut self, + session: Session, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, + ); fn session_started(&mut self, session: Session, force: bool); fn session_stopped(&mut self, session_id: &SessionId); fn attach_allowed(&mut self, key_id: &KeyId, access: Access); diff --git a/client/src/lib.rs b/client/src/lib.rs index 1f9aab9a..3b90ca87 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -80,8 +80,10 @@ progenitor::generate_api!( KeyId = sush_common::keys::KeyId, Signature = sush_common::keys::Signature, SignedForJobStartRequest = sush_common::jobs::SignedJob, + SessionClientNonce = sush_common::jobs::SessionClientNonce, + SessionSeq = sush_common::jobs::SessionSeq, + SessionServerNonce = sush_common::jobs::SessionServerNonce, SessionSignerNonce = sush_common::jobs::SessionSignerNonce, - SessionSushNonce = sush_common::jobs::SessionSushNonce, SledVersion = sush_common::targets::SledVersion, JobMode = sush_common::jobs::JobMode, VersionInfo = sush_common::version::VersionInfo, diff --git a/client/src/permslip.rs b/client/src/permslip.rs index fca7477c..ca161913 100644 --- a/client/src/permslip.rs +++ b/client/src/permslip.rs @@ -21,19 +21,12 @@ pub struct PermslipSigner { key_name: String, } -/// Get a bearer token by the sshauth challenge, signed with the agent -/// key named by `fingerprint`. -pub async fn fresh_token( - url: &str, - agent_sock: String, - fingerprint: String, -) -> Result { - let tokens = TokenProvider::SshAuth { - fingerprint: Some(fingerprint), - server_url: url.to_owned(), - agent_sock, - }; - let token = tokens.token().await.map_err(PermslipError::token_flow)?; +/// Get the bearer token that `permslip login` saved for `url`. +pub async fn saved_token(url: &str) -> Result { + let token = TokenProvider::SavedCredentials + .token(url) + .await + .map_err(PermslipError::token_flow)?; let value = token.into_header_value().map_err(PermslipError::token)?; value .to_str() diff --git a/client/src/repl.rs b/client/src/repl.rs index 3591afeb..ec65be73 100644 --- a/client/src/repl.rs +++ b/client/src/repl.rs @@ -21,7 +21,8 @@ use xdg::BaseDirectories; use sush_common::authn::Identity; use sush_common::jobs::{ - Access, JobId, JobOutputStream, JobStatusMap, Session, SessionId, SessionSignerNonce, SignedJob, + Access, JobId, JobOutputStream, JobStartRequest, JobStatusMap, Session, SessionClientNonce, + SessionId, SessionSeq, SessionSignerNonce, SessionSushNonce, SignedJob, }; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::targets::{SledId, SledVersion}; @@ -33,7 +34,6 @@ use crate::commands::{ SUSH_OUTPUT_FORMAT, SUSH_URL, }; use crate::context::{CommandContext, OutputFormat, StatusDisplayStyle}; -use crate::types::SessionStartNonce; use crate::{AuthzSigner, Client}; const HISTORY_FILE: &str = "history.txt"; @@ -227,14 +227,6 @@ impl CommandContext for Repl { self.cli.authz_signer() } - fn permslip_token(&self, url: &str, fingerprint: &str) -> Option { - self.cli.permslip_token(url, fingerprint) - } - - fn save_permslip_token(&self, url: &str, fingerprint: &str, token: &str) { - self.cli.save_permslip_token(url, fingerprint, token) - } - fn session_id(&self) -> Option { self.cli.session_id() } @@ -243,12 +235,35 @@ impl CommandContext for Repl { self.cli.next_job_id() } - fn session_start_params(&self, baseboard_id: BaseboardId, nonce: SessionStartNonce) { - self.cli.session_start_params(baseboard_id, nonce) + fn pending_client_nonce(&self) -> Option { + self.cli.pending_client_nonce() } - fn session_created(&mut self, session: Session, signer_nonce: SessionSignerNonce) { - self.cli.session_created(session, signer_nonce) + fn save_client_nonce(&self, client_nonce: SessionClientNonce) { + self.cli.save_client_nonce(client_nonce) + } + + fn session_start_params( + &self, + baseboard_id: BaseboardId, + client_nonce: SessionClientNonce, + sush_nonce: SessionSushNonce, + ) { + self.cli + .session_start_params(baseboard_id, client_nonce, sush_nonce) + } + + fn job_start_params(&self, request: &JobStartRequest) { + self.cli.job_start_params(request) + } + + fn session_created( + &mut self, + session: Session, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, + ) { + self.cli.session_created(session, signer_nonce, seq) } fn session_started(&mut self, session: Session, force: bool) { diff --git a/common/src/codephrases.rs b/common/src/codephrases.rs index 23219f16..3fa05929 100644 --- a/common/src/codephrases.rs +++ b/common/src/codephrases.rs @@ -46,6 +46,11 @@ const WORD_SEPARATOR: &str = "-"; /// the strength of the signatures produced over these phrases. const TRUNCATED_BITS: u32 = 88; +/// A four-word phrase carries 44 bits. Not enough for an identifier, +/// but room enough for a small ordered value, like a Unix time in +/// milliseconds. +const SHORT_BITS: u32 = 44; + /// Random code phrase like `abstract misery favorite ordinary moon talk`. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Codephrase(U256); @@ -56,12 +61,20 @@ impl Codephrase { Self(U256::random(&mut OsRng)) } - /// Turn 256 bits of entropy into a reasonably unique code phrase. - /// We use the low-order words of the full codephrase, and pad to - /// the canonical length; these phrases are intended for machine - /// consumption, and are short enough for easy transmission. + /// Turn 256 bits of entropy into a reasonably unique code phrase, + /// around eight words. pub fn truncate(self) -> Self { - let mask = U256::from_u128(2u128.pow(TRUNCATED_BITS) - 1); + self.mask(TRUNCATED_BITS) + } + + /// Chop entropy down to four words; see `SHORT_BITS`. + pub fn shorten(self) -> Self { + self.mask(SHORT_BITS) + } + + /// Keep only the low-order bits of some entropy. + fn mask(self, bits: u32) -> Self { + let mask = U256::from_u128((1 << bits) - 1); Self(self.0.bitand(&mask)) } @@ -70,6 +83,23 @@ impl Codephrase { Self(U256::from_be_bytes(bytes)) } + /// Construct a codephrase from a small integer. + pub fn from_u64(value: u64) -> Self { + Self(U256::from_u64(value)) + } + + /// Return the small integer represented by a codephrase, + /// or `None` if it is wider than 64 bits. + pub fn to_u64(&self) -> Option { + let bytes = self.to_be_bytes(); + if bytes[..24].iter().any(|&byte| byte != 0) { + return None; + } + Some(u64::from_be_bytes( + bytes[24..].try_into().expect("eight bytes"), + )) + } + /// Get the underlying big-endian byte representation of the codephrase. pub fn to_be_bytes(&self) -> [u8; 32] { self.0.to_be_byte_array().into() @@ -77,8 +107,8 @@ impl Codephrase { } impl fmt::Display for Codephrase { + /// Turn 256 bits of entropy into an un-padded big-endian code phrase. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Turn 256 bits of entropy into an un-padded big-endian code phrase. let b = Reciprocal::new(Limb(WORDLIST_LEN as u64)).expect("should have some words"); let mut n = self.0; let mut r; @@ -102,18 +132,17 @@ impl fmt::Debug for Codephrase { impl FromStr for Codephrase { type Err = InvalidCodephrase; + /// Decode a big endian code phrase into 256 bits of entropy. + /// + /// Decoding is non-injective, or "liberal" in the sense that it will + /// accept non-canonical codephrases, e.g., ones with spaces instead of + /// separators, words in mixed case, or without leading zeros. This is + /// for the comfort of humans that may have to transmit such phrases. + /// It rejects phrases that no entropy encodes to: unknown words, more + /// than `PHRASE_WORDS_256` words, or a value of 256 bits or more. + /// The empty phrase decodes to zero. Callers that must distinguish + /// absent input from zero should check before decoding. fn from_str(value: &str) -> Result { - // Decode a big endian code phrase into 256 bits of entropy. - // - // Decoding is non-injective, or "liberal" in the sense that it will - // accept non-canonical codephrases, e.g., ones with spaces instead of - // separators, words in mixed case, or without leading zeros. This is - // for the comfort of humans that may have to transmit such phrases. - // It rejects phrases that no entropy encodes to: unknown words, more - // than [`PHRASE_WORDS_256`] words, or a value of 256 bits or more. - // The empty phrase decodes to zero. Callers that must distinguish - // absent input from zero should check before decoding. - let b = U256::from_u64(WORDLIST_LEN as u64); let mut n = U256::ZERO; let mut words = 0; @@ -158,7 +187,7 @@ impl<'de> Deserialize<'de> for Codephrase { } } -// Treat a codephrase as a string in JSON schemas. +/// Treat a codephrase as a string in JSON schemas. impl JsonSchema for Codephrase { fn schema_name() -> String { String::schema_name() @@ -192,13 +221,9 @@ macro_rules! codephrase_newtype { fn try_from( codephrase: $crate::codephrases::Codephrase, ) -> Result { - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - if codephrase.truncate() != codephrase { - return Err($crate::codephrases::InvalidCodephrase); - } - } + let length = $crate::codephrases::CodephraseLength::$len; + if length.canonical(codephrase) != codephrase { + return Err($crate::codephrases::InvalidCodephrase); } Ok(Self(codephrase)) } @@ -207,38 +232,24 @@ macro_rules! codephrase_newtype { impl $name { #[allow(unused)] $vis fn random() -> Self { - let mut codephrase = $crate::codephrases::Codephrase::random(); - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - codephrase = codephrase.truncate(); - } - } - Self(codephrase) + let length = $crate::codephrases::CodephraseLength::$len; + Self(length.canonical($crate::codephrases::Codephrase::random())) } #[allow(unused)] $vis fn from_hash(hash: $crate::hash::Hash) -> Self { - let mut codephrase = $crate::codephrases::Codephrase::from_be_bytes(*hash.as_bytes()); - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - codephrase = codephrase.truncate(); - } - } - Self(codephrase) + let length = $crate::codephrases::CodephraseLength::$len; + Self(length.canonical( + $crate::codephrases::Codephrase::from_be_bytes(*hash.as_bytes()), + )) } #[allow(unused)] $vis fn from_be_bytes(bytes: [u8; 32]) -> Self { - let mut codephrase = $crate::codephrases::Codephrase::from_be_bytes(bytes); - match $crate::codephrases::CodephraseLength::$len { - $crate::codephrases::CodephraseLength::Full => {} - $crate::codephrases::CodephraseLength::Truncated => { - codephrase = codephrase.truncate(); - } - } - Self(codephrase) + let length = $crate::codephrases::CodephraseLength::$len; + Self(length.canonical( + $crate::codephrases::Codephrase::from_be_bytes(bytes), + )) } #[allow(unused)] @@ -273,6 +284,18 @@ macro_rules! codephrase_newtype { pub enum CodephraseLength { Full, Truncated, + Short, +} + +impl CodephraseLength { + /// Construct a codephrase of canonical length by discarding entropy. + pub fn canonical(self, codephrase: Codephrase) -> Codephrase { + match self { + Self::Full => codephrase, + Self::Truncated => codephrase.truncate(), + Self::Short => codephrase.shorten(), + } + } } /// Decoding a phrase failed. @@ -331,6 +354,13 @@ mod test { round_trip(U256::MAX); } + #[test] + fn small_integers() { + assert_eq!(Codephrase::from_u64(0).to_u64(), Some(0)); + assert_eq!(Codephrase::from_u64(u64::MAX).to_u64(), Some(u64::MAX)); + assert_eq!(Codephrase::from_be_bytes([0x01; 32]).to_u64(), None); + } + #[test] fn non_phrases() { // Unknown words, wherever they appear. diff --git a/common/src/hash.rs b/common/src/hash.rs index 2510c847..203160a0 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -87,6 +87,13 @@ impl Hasher { self } + /// Append length-prefixed bytes, so that adjacent updates cannot + /// bleed into one another. + pub fn update_with_len(&mut self, bytes: &[u8]) -> &mut Self { + self.update(&(bytes.len() as u64).to_be_bytes()); + self.update(bytes) + } + pub fn count(&self) -> u64 { self.count } diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 26eeb0e1..be90e08a 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -20,6 +20,7 @@ pub use sled_hardware_types::BaseboardId; use sled_hardware_types::BaseboardIdParseError; use thiserror::Error; +use crate::codephrases::Codephrase; use crate::hash::{Hash, Hasher, hash}; use crate::interactive::InteractiveJobError; use crate::keys::{KeyId, Signed, ToBeSigned, ToBeSignedFields, Verified}; @@ -60,7 +61,8 @@ impl From<&JobId> for JobId { } codephrase_newtype! { - /// A globally unique identifier for a session. + /// A globally unique identifier for a session. Constructed only + /// from a `SessionPreimage`. #[derive( Copy, Clone, @@ -76,30 +78,46 @@ codephrase_newtype! { pub struct SessionId = Truncated; } -impl SessionId { - pub fn compute( - baseboard: &BaseboardId, - sush: SessionSushNonce, - signer: SessionSignerNonce, - ) -> Self { - let mut hasher = Hasher::new(); - let mut hash_with_len = |data: &[u8]| { - hasher.update(&(data.len() as u64).to_be_bytes()); - hasher.update(data); - }; - - hash_with_len(b"SUSH_SESSION_ID_V1"); - hash_with_len(b"baseboard_id"); - hash_with_len(baseboard.part_number.as_bytes()); - hash_with_len(baseboard.serial_number.as_bytes()); - hash_with_len(b"sush_nonce"); - hash_with_len(&sush.to_be_bytes()); - hash_with_len(b"signer_nonce"); - hash_with_len(&signer.to_be_bytes()); +/// The material a session ID hashes over, and the only source +/// of one. The `SessionRequest::Start` message gossips this +/// preimage to prevent a malicious or corrupted server from +/// poisoning the session. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SessionPreimage { + pub baseboard: BaseboardId, + pub sush_nonce: SessionSushNonce, + pub signer_nonce: SessionSignerNonce, + pub seq: SessionSeq, +} - Self::from_hash(hasher.finalize()) +impl SessionPreimage { + pub fn session_id(&self) -> SessionId { + let Self { + baseboard: + BaseboardId { + part_number, + serial_number, + }, + sush_nonce, + signer_nonce, + seq, + } = self; + let mut hasher = Hasher::new(); + hasher.update_with_len(b"SUSH_SESSION_ID_V2"); + hasher.update_with_len(b"baseboard_id"); + hasher.update_with_len(part_number.as_bytes()); + hasher.update_with_len(serial_number.as_bytes()); + hasher.update_with_len(b"sush_nonce"); + hasher.update_with_len(&sush_nonce.to_be_bytes()); + hasher.update_with_len(b"signer_nonce"); + hasher.update_with_len(&signer_nonce.to_be_bytes()); + hasher.update_with_len(b"seq"); + hasher.update_with_len(&seq.to_be_bytes()); + SessionId::from_hash(hasher.finalize()) } +} +impl SessionId { pub fn first_job_id(&self) -> JobId { JobId::from_hash(hash(&self.0.to_be_bytes())) } @@ -115,7 +133,44 @@ impl SessionId { } codephrase_newtype! { - /// The component of the hashed Session ID generated by the Sush server. + /// The session nonce component generated by the sush client + /// (on the operator's machine). + #[derive( + Copy, + Clone, + Deserialize, + Serialize, + JsonSchema, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + )] + pub struct SessionClientNonce = Truncated; +} + +codephrase_newtype! { + /// The session nonce component generated by the sush server + /// (on the rack). + #[derive( + Copy, + Clone, + Deserialize, + Serialize, + JsonSchema, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + )] + pub struct SessionServerNonce = Truncated; +} + +codephrase_newtype! { + /// The nonce the client submits to the signer, + /// derived from the client and server nonces. #[derive( Copy, Clone, @@ -131,8 +186,27 @@ codephrase_newtype! { pub struct SessionSushNonce = Truncated; } +impl SessionSushNonce { + /// Hash the client and server nonces together. The server nonce + /// alone could carry anything the rack chose to hide in it, but + /// the hash of the pair carries nothing the rack could steer, + /// because the rack committed to its nonce before the client + /// chose its. An operator in a secure facility should therefore + /// be able to export the result based on the strength of their + /// own machine's nonce. + pub fn compute(client: SessionClientNonce, server: SessionServerNonce) -> Self { + let mut hasher = Hasher::new(); + hasher.update_with_len(b"SUSH_SESSION_NONCE_V2"); + hasher.update_with_len(b"client_nonce"); + hasher.update_with_len(&client.to_be_bytes()); + hasher.update_with_len(b"server_nonce"); + hasher.update_with_len(&server.to_be_bytes()); + Self::from_hash(hasher.finalize()) + } +} + codephrase_newtype! { - /// The component of the hashed Session ID generated by the signer server. + /// The component of the hashed Session ID generated by the signing server. #[derive( Copy, Clone, @@ -148,6 +222,57 @@ codephrase_newtype! { pub struct SessionSignerNonce = Truncated; } +codephrase_newtype! { + /// The session's place in the signer-assigned order of sessions. + /// Taken as whole milliseconds since the Unix epoch according to + /// the *signer's* wall clock; the server's clock can't be trusted, + /// so the server never interprets a sequence number as a time, + /// only as an order. Sessions authorized in the same millisecond + /// share a sequence number and tie-break by ID. Encoded as four + /// codephrase words: 2048⁴ = 2⁴⁴ values, good through the year + /// 2527. + #[derive( + Copy, + Clone, + Deserialize, + Serialize, + JsonSchema, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + )] + pub struct SessionSeq = Short; +} + +#[derive(Debug, Error)] +#[error("session sequence number exceeds four-word codephrase range")] +pub struct SeqOutOfRange; + +impl SessionSeq { + /// The largest encodable wall clock reading, `zoo-zoo-zoo-zoo` + /// = 2527-06-23T06:20:44.415Z. + pub const MAX_MILLIS: u64 = (1 << 44) - 1; + + pub fn zero() -> Self { + Self(Codephrase::from_u64(0)) + } + + /// Make a sequence number from a wall clock reading. Fails + /// if millis is out of range. + pub fn from_unix_millis(millis: u64) -> Result { + if millis > Self::MAX_MILLIS { + return Err(SeqOutOfRange); + } + Ok(Self(Codephrase::from_u64(millis))) + } + + pub fn as_unix_millis(&self) -> u64 { + self.0.to_u64().expect("a sequence number is four words") + } +} + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub enum LastJob { #[default] @@ -401,20 +526,14 @@ pub enum JobStatus { #[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "kebab-case")] pub enum SkipReason { - /// The job's session sits at or below the sled's execution floor, - /// where the sled cannot tell replay from re-run. - BelowFloor, - /// The job's chain position precedes the sled's recorded - /// commitment: a previous life already handled it. - AlreadyHandled, + SessionSuperseded, SessionEnded, } impl fmt::Display for SkipReason { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match self { - Self::BelowFloor => "the session sits below this sled's execution floor", - Self::AlreadyHandled => "a previous life of this sled already handled it", + Self::SessionSuperseded => "another session has superseded this one", Self::SessionEnded => "the session ended before the job could start", }) } @@ -733,6 +852,81 @@ mod test { use crate::keys::{EccR, EccS, EncodedSignature}; use crate::targets::SledId; + /// A sequence number encodes to four words, parses back liberally, + /// and orders numerically. + #[test] + fn seq_phrases() { + let max = SessionSeq::from_unix_millis(SessionSeq::MAX_MILLIS).unwrap(); + assert_eq!(max.to_string(), "zoo-zoo-zoo-zoo"); + assert_eq!("zoo zoo zoo zoo".parse::().unwrap(), max); + + // 2026-10-09T09:33:20Z, a realistic signer clock reading. + // Every wall clock reading since 1970-04-10 fills four words. + let seq = SessionSeq::from_unix_millis(1_760_000_000_000).unwrap(); + assert_eq!(seq.as_unix_millis(), 1_760_000_000_000); + assert_eq!(seq.to_string(), "book-tomato-reason-abandon"); + assert_eq!(seq.to_string().parse::().unwrap(), seq); + assert!(seq < max); + + // 2⁴⁴ milliseconds exceed the range however they arrive. + assert!(SessionSeq::from_unix_millis(1 << 44).is_err()); + assert!( + "ability-abandon-abandon-abandon-abandon" + .parse::() + .is_err() + ); + } + + /// On the binary wire a sequence number is codephrase bytes, like + /// the ids and nonces; as text it is words. A value too wide for + /// four words fails to decode. + #[test] + fn seq_serde_forms() { + let seq = SessionSeq::from_unix_millis(1_760_000_000_000).unwrap(); + assert_eq!( + serde_json::to_value(seq).unwrap(), + serde_json::json!("book-tomato-reason-abandon") + ); + let mut bytes = Vec::new(); + ciborium::ser::into_writer(&seq, &mut bytes).unwrap(); + assert_eq!( + hex::encode(&bytes), + "5820000000000000000000000000000000000000000000000000\ + 00000199c82cc000" + ); + assert_eq!( + ciborium::de::from_reader::(bytes.as_slice()).unwrap(), + seq + ); + + let mut wide = Vec::new(); + ciborium::ser::into_writer(&EccR::from_be_bytes([0xff; 32]), &mut wide).unwrap(); + assert!(ciborium::de::from_reader::(wide.as_slice()).is_err()); + } + + /// Pin the derived sush nonce: the client and the rack both + /// compute it, and the signer hashes it into the session id, so + /// the derivation must never drift. + #[test] + fn pin_sush_nonce_compute() { + let nonce = SessionSushNonce::compute( + SessionClientNonce::from_be_bytes([0x11; 32]), + SessionServerNonce::from_be_bytes([0x22; 32]), + ); + assert_eq!( + nonce.to_string(), + "shop-adapt-belt-pulse-order-try-local-record" + ); + + // The field labels separate the operands: swapping the + // inputs changes the nonce. + let swapped = SessionSushNonce::compute( + SessionClientNonce::from_be_bytes([0x22; 32]), + SessionServerNonce::from_be_bytes([0x11; 32]), + ); + assert_ne!(nonce, swapped); + } + /// A job request is signed, and every sled rebuilds it from the /// wire to check the signature, so two sleds with different /// request shapes disagree about what verifies. This pin freezes diff --git a/common/tests/output/job-start-request-schema.json b/common/tests/output/job-start-request-schema.json index c5cd765c..71afee34 100644 --- a/common/tests/output/job-start-request-schema.json +++ b/common/tests/output/job-start-request-schema.json @@ -42,7 +42,7 @@ ] }, "SessionId": { - "description": "A globally unique identifier for a session.", + "description": "A globally unique identifier for a session. Constructed only from a `SessionPreimage`.", "type": "string" } } diff --git a/server/src/bloom.rs b/server/src/bloom.rs deleted file mode 100644 index 5ede9cd5..00000000 --- a/server/src/bloom.rs +++ /dev/null @@ -1,161 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! A fixed-size Bloom filter for durable, grow-only sets. -//! -//! Designed for the boundary record's burned networks set. Entries may -//! never be dropped (once burned, a network stays burned), the size must -//! stay bounded under adversarial growth (we must not fill the M.2s), -//! and the filter may err only by claiming a key it never held. The -//! caller must treat that claim as refusal. -//! -//! The layout and hash algorithm determine the on-disk format, and must -//! not be changed without updating the boundary record's magic. Each key -//! probes the table at seven positions, cut as 16-bit words from its -//! SHA3-256 digest. The table is currently sized at 8192 bits, for -//! which seven probes is the optimal count out to about 800 entries -//! (bits * ln 2 / probes). The false positive rate there is about 0.7%, -//! degrading gradually past it. Real occupancy should stay in the tens, -//! where false positives are negligible. - -use std::array::from_fn; -use std::fmt; - -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -use sush_common::hash::{OUT_LEN, hash}; -use sush_common::wire::ExactBytes; - -const BYTES: usize = 1024; -const BITS: u64 = (BYTES * 8) as u64; -const PROBES: usize = 7; - -// The probes must fit in the digest, and the table size must divide -// 2^16, or reducing a 16-bit word to a table slot would favor some -// slots over others. -const _: () = { - assert!(2 * PROBES <= OUT_LEN); - assert!((1u64 << 16).is_multiple_of(BITS)); -}; - -/// A grow-only set of byte-string keys. -#[derive(Clone, Eq, PartialEq)] -pub struct Bloom { - bits: Box<[u8; BYTES]>, -} - -/// The probe positions for `key` are seven 16-bit words cut from -/// the leading bytes of the key's SHA3-256 digest, each reduced to -/// a table slot. -fn probes(key: &[u8]) -> [u64; PROBES] { - let digest = *hash(key).as_bytes(); - from_fn(|i| { - let j = 2 * i; - let word = u16::from_le_bytes(digest[j..j + 2].try_into().expect("two bytes")); - u64::from(word) % BITS - }) -} - -/// The byte index and mask selecting `probe`'s bit. -fn bit(probe: u64) -> (usize, u8) { - ((probe / 8) as usize, 1 << (probe % 8)) -} - -impl Bloom { - pub fn new() -> Self { - Self { - bits: Box::new([0; BYTES]), - } - } - - /// Permanently add `key` to the set. - pub fn insert(&mut self, key: &[u8]) { - for (byte, mask) in probes(key).map(bit) { - self.bits[byte] |= mask; - } - } - - /// Whether `key` may have been inserted. Never returns a false negative, - /// but false positives occur at the documented rate. - pub fn contains(&self, key: &[u8]) -> bool { - probes(key) - .map(bit) - .into_iter() - .all(|(byte, mask)| self.bits[byte] & mask != 0) - } -} - -impl Default for Bloom { - fn default() -> Self { - Self::new() - } -} - -impl fmt::Debug for Bloom { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let set: u32 = self.bits.iter().map(|byte| byte.count_ones()).sum(); - write!(f, "Bloom({set}/{BITS} bits set)") - } -} - -impl Serialize for Bloom { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_bytes(&self.bits[..]) - } -} - -impl<'de> Deserialize<'de> for Bloom { - fn deserialize>(deserializer: D) -> Result { - let bits = deserializer.deserialize_bytes(ExactBytes::)?; - Ok(Self { - bits: Box::new(bits), - }) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn round_trip() { - let mut bloom = Bloom::new(); - assert!(!bloom.contains(b"lost-network")); - bloom.insert(b"lost-network"); - assert!(bloom.contains(b"lost-network")); - assert!(!bloom.contains(b"other-network")); - } - - #[test] - fn serde_preserves_bits() { - let mut bloom = Bloom::new(); - bloom.insert(b"burned"); - let mut bytes = Vec::new(); - ciborium::ser::into_writer(&bloom, &mut bytes).unwrap(); - let back: Bloom = ciborium::de::from_reader(bytes.as_slice()).unwrap(); - assert_eq!(bloom, back); - assert!(back.contains(b"burned")); - } - - #[test] - fn false_positives_are_rare() { - let mut bloom = Bloom::new(); - for i in 0..128 { - bloom.insert(format!("burned-{i}").as_bytes()); - } - let hits = (0..10_000) - .filter(|i| bloom.contains(format!("probe-{i}").as_bytes())) - .count(); - assert_eq!( - hits, 0, - "false positives at plausible occupancy: {hits}/10000" - ); - } - - /// If this test fails, STOP! The on-disk format may have changed! - #[test] - fn pin_probes() { - assert_eq!(probes(b"sush"), [3392, 1007, 8018, 5416, 1335, 7804, 1908]); - } -} diff --git a/server/src/boundary.rs b/server/src/boundary.rs index 825cdd3b..71748f9b 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -5,49 +5,29 @@ //! The boundary between jobs we executed and jobs we only heard about. //! //! One record, rewritten before each spawn, carries what this sled -//! last committed to: the job, its session, the chain position after -//! the job, the universe they belong to, and the job's ending once -//! it has one. A universe is one shared gossip history, identified -//! by its network; sleds join universes, leave them, and sometimes -//! return. Alongside the commitment, the record keeps the join -//! of every session start this sled has executed under in that -//! universe, and the set of universes it burned by leaving. The -//! record is this sled's execution watermark: nothing at or below it -//! may run again. +//! last committed to: the job, its session and that session's +//! sequence number, the chain position after the job, the universe +//! they belong to, and the job's ending once it has one. The signer +//! assigns every session a sequence number from its wall clock, so +//! sequence numbers order sessions totally, across restarts and +//! across universes: no session at or below the committed one may +//! run jobs here again. //! //! After a restart, replayed gossip rebuilds the sessions. When the //! committed session activates, the sled resumes its chain at the //! stored successor. The previous life had already moved the chain //! past every earlier position, so the session's queue never -//! releases them here, and the chain continues from the successor -//! whether its request arrives by replay or by resubmission. A -//! session that starts strictly above the executed join has never -//! run here and is served. A session the record cannot order makes -//! the sled hop: the sled reports the hop to the gossip set as an -//! error and sets a floor, in memory, at the frontier that includes -//! the report. A session that does not start above the floor has -//! its jobs skipped; a session started after the hop is served. The -//! join folds in every session this sled has executed under in this -//! universe, so a session it once ran under can never screen as -//! new: the sled hops, and the session's jobs are skipped rather -//! than re-run. If replay gives the recorded job no status, the sled -//! adjudicates it: it announces the recorded ending when the record -//! holds one, and an interrupted ending when it does not. +//! releases those jobs here, and the chain continues from the +//! successor whether its request arrives by replay or by +//! resubmission. A session whose sequence number exceeds the +//! committed one has never run jobs here and is served; any other +//! session's jobs are skipped. If replay gives the recorded job no +//! status, the sled adjudicates it: it announces the recorded ending +//! when the record holds one, and an interrupted ending when it does +//! not. //! -//! Universes have no order. The record instead keeps a burned set, -//! holding the network of every universe whose watermark it -//! overwrote by moving on. A sled that re-enters a burned universe -//! has flip-flopped, and raises its floor: it reports the flip-flop -//! to the gossip set and sets the floor, in memory, at the frontier -//! that includes the report. No older message can contain a version -//! born at that instant, so a session started before the re-entry -//! never lies above the floor, and its jobs are skipped. A sled that -//! enters a universe with history while holding no record for it -//! raises a floor the same way. The floor is never persisted: the -//! burn, the missing record, or the unordered session is still there -//! after a restart, and raises it again. A floor written to disk -//! could carry a version that died with the life that created it; -//! no later session could ever dominate such a floor. +//! A sled with no record cannot tell replay from re-run, so it +//! serves only sessions that start as live traffic. //! //! A boundary that cannot be written means the job must not run. A //! boundary that cannot be trusted means no job may run at all, since @@ -57,16 +37,14 @@ use std::sync::Mutex as SyncMutex; use std::sync::atomic::{AtomicBool, Ordering}; -use ciborium::ser::into_writer as into_cbor; -use rumors::{Network, Version}; +use rumors::Network; use serde::{Deserialize, Serialize}; use slog::{Logger, o, warn}; use thiserror::Error; -use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId}; +use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId, SessionSeq}; -use crate::bloom::Bloom; -use crate::format::{self, NoFormat, Record, Versioned}; +use crate::format::{self, Record, Versioned}; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOUNDARY: TenantSpec = TenantSpec { @@ -74,97 +52,37 @@ pub const BOUNDARY: TenantSpec = TenantSpec { magic: b"SUSHBOUNDARY", }; -/// The execution boundary: what this sled last committed to, in which -/// universe, and which universes it has left behind ("burned"). -/// -/// Versions do not compare across universes, so `network` scopes every -/// version in the record. `burned` holds the network of every -/// universe whose watermark this record overwrote by moving on: a -/// sled re-entering one raises its floor in memory, and the record on -/// disk stays the displaced universe's true watermark until a commit -/// overwrites it. -/// -/// `executed` is the join of the start versions of every session -/// this sled has executed under in this universe. A single stored -/// start would forget the sessions before it, and a third session -/// could then replay the first session's jobs; the join never -/// forgets. Session starts are witnessed messages, and only those -/// keep their meaning across a crash. A start that only this sled -/// ever saw belongs to a session whose history is lost, and refusal -/// is the right answer there anyway. +/// The execution boundary: what this sled last committed to, and in +/// which universe. The committed session supersedes every session at +/// or below its sequence number: the signer orders sessions totally +/// and across universes, so every session a previous life could have +/// run jobs under sits at or below this one stored number. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Boundary { pub network: Network, - pub burned: Bloom, - #[serde(with = "version_bytes")] - pub executed: Version, pub job: Option, } /// The last job this sled committed to running, and how far it got. /// We also store the chain position *after* `job`, computed from the /// request's signed bytes at commit time, to allow session resumption. +/// +/// The sequence number itself travels unsigned, in the gossiped +/// session preimage, yet a forged one cannot reach this record: the +/// sled writes the record only when it commits to a signed job, the +/// job's signed bytes name the session id, and the id hashes over +/// the preimage, sequence number included. A preimage with a +/// different sequence number hashes to a different session id, and +/// no signed job names that id. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Committed { pub session: SessionId, + pub seq: SessionSeq, pub job: JobId, pub successor: JobId, pub outcome: JobOutcome, } -impl Boundary { - /// Whether this record burned `network`: left its universe behind - /// and overwrote its watermark. - pub fn is_burned(&self, network: Network) -> bool { - self.burned.contains(&network_key(network)) - } - - /// The burned set for the replacement record, committed in - /// `network`. A replacement in a different universe burns this - /// record's own network. - pub fn burned_for(&self, network: Network) -> Bloom { - let mut burned = self.burned.clone(); - if self.network != network { - burned.insert(&network_key(self.network)); - } - burned - } - - /// The executed-session join for the replacement record, - /// committed in `network` and folding in `started`. Joins never - /// cross universes, so a replacement elsewhere starts its join - /// fresh. - pub fn executed_for(&self, network: Network, started: &Version) -> Version { - if self.network == network { - self.executed.clone() | started.clone() - } else { - started.clone() - } - } -} - -/// A network's Bloom key is its CBOR bytes. -fn network_key(network: Network) -> Vec { - let mut bytes = Vec::new(); - into_cbor(&network, &mut bytes).expect("writing to a Vec cannot fail"); - bytes -} - -mod version_bytes { - use rumors::Version; - use serde::de::Error as _; - use serde::{Deserialize, Deserializer, Serializer}; - - pub fn serialize(version: &Version, serializer: S) -> Result { - serializer.serialize_bytes(&version.encode()) - } - - pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - let bytes = >::deserialize(deserializer)?; - Version::decode(bytes.as_slice()).map_err(D::Error::custom) - } -} - /// How far the boundary job got. #[derive(Clone, Debug, Deserialize, Serialize)] pub enum JobOutcome { @@ -174,21 +92,128 @@ pub enum JobOutcome { Ended(JobStatus), } -/// Version 0 is the shipped baseline. The pinned record snapshot in -/// this module's tests freezes its bytes; see [`crate::format`] for -/// the steps a format change requires. +/// Record version 1 replaces the v0 executed-session join and burned set +/// with the committed sequence number from the session-v2 protocol. +/// Don't confuse a "session-v2" (one with a sequence number) with a +/// boundary record version 2 (which doesn't yet exist). +/// +/// See [`crate::format`] for instructions on adding this version's +/// successor. impl Versioned for Boundary { - const VERSION: u16 = 0; + const VERSION: u16 = 1; } impl Record for Boundary { - type Previous = NoFormat; + type Previous = v0::Boundary; } -impl TryFrom for Boundary { +impl TryFrom for Boundary { type Error = &'static str; - fn try_from(none: NoFormat) -> Result { - match none {} + + fn try_from(old: v0::Boundary) -> Result { + let v0::Boundary { + network, + burned: _, // dropped with session-v2 protocol + executed: _, // also dropped with session-v2 + job, + } = old; + Ok(Self { + network, + job: job.map(|job| { + let v0::Committed { + session, + job, + successor, + outcome, + } = job; + Committed { + session, + seq: SessionSeq::zero(), // any valid session-v2 will supersede this one + job, + successor, + outcome, + } + }), + }) + } +} + +/// The version 0 record shapes, kept only to decode records written +/// before sessions carried sequence numbers. +pub mod v0 { + use super::{JobId, JobOutcome, Network, SessionId}; + use crate::format::{NoFormat, Record, Versioned}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use sush_common::wire::ExactBytes; + + #[derive(Deserialize, Serialize)] + pub struct Boundary { + pub network: Network, + pub burned: Burned, + #[serde(with = "version_bytes")] + pub executed: rumors::Version, + pub job: Option, + } + + /// The v0 burned set: the Bloom filter's 1024 bytes, decoded but + /// never read. + pub struct Burned(Box<[u8; 1024]>); + + impl Serialize for Burned { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.0[..]) + } + } + + impl<'de> Deserialize<'de> for Burned { + fn deserialize>(deserializer: D) -> Result { + Ok(Self(Box::new( + deserializer.deserialize_bytes(ExactBytes::<1024>)?, + ))) + } + } + + #[derive(Deserialize, Serialize)] + pub struct Committed { + pub session: SessionId, + pub job: JobId, + pub successor: JobId, + pub outcome: JobOutcome, + } + + impl Versioned for Boundary { + const VERSION: u16 = 0; + } + + impl Record for Boundary { + type Previous = NoFormat; + } + + impl TryFrom for Boundary { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } + } + + mod version_bytes { + use rumors::Version; + use serde::de::Error as _; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize( + version: &Version, + serializer: S, + ) -> Result { + serializer.serialize_bytes(&version.encode()) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result { + let bytes = >::deserialize(deserializer)?; + Version::decode(bytes.as_slice()).map_err(D::Error::custom) + } } } @@ -301,8 +326,6 @@ impl BoundaryStore { ..committed.clone() }), network: boundary.network, - burned: boundary.burned.clone(), - executed: boundary.executed.clone(), } }; if let Err(error) = guard.store(&format::encode(&updated)).await { @@ -338,7 +361,7 @@ mod test { use std::fs::create_dir; use camino::Utf8PathBuf; - use rumors::before::Clock; + use ciborium::ser::into_writer as into_cbor; use slog::Discard; use tempfile::TempDir; @@ -368,21 +391,16 @@ mod test { serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() } - /// A deterministic nonempty version: the seed party with two - /// events. - fn executed() -> Version { - let mut clock = Clock::seed(); - clock.tick(); - clock.tick().clone() + fn seq(millis: u64) -> SessionSeq { + SessionSeq::from_unix_millis(millis).unwrap() } fn boundary() -> Boundary { Boundary { network: network(1), - burned: Bloom::new(), - executed: executed(), job: Some(Committed { session: SessionId::random(), + seq: seq(1_760_000_000_000), job: JobId::random(), successor: JobId::random(), outcome: JobOutcome::Committed, @@ -394,29 +412,39 @@ mod test { boundary.job.as_ref().expect("a committed job").job } - /// The record's bytes are on-disk format, frozen at version 0. If + /// The frozen v0 snapshot decodes and converts: the commitment + /// survives with a sequence number of zero. Nothing can regenerate + /// the snapshot; the code that wrote it is gone, and git keeps the + /// bytes. + #[test] + fn v0_record_converts() { + let bytes = std::fs::read("tests/output/boundary-record-v0.bin").expect("missing snapshot"); + let decoded: Boundary = format::decode(&bytes).unwrap(); + assert_eq!(decoded.network, network(1)); + let job = decoded.job.unwrap(); + assert_eq!(job.session, "abandon-ability".parse().unwrap()); + assert_eq!(job.successor, "able-about".parse().unwrap()); + assert_eq!(job.seq, seq(0)); + } + + /// The record's bytes are on-disk format, frozen at version 1. If /// this fails, STOP: do not re-pin. Copy the old shape into a /// frozen module and add a new version instead; see /// [`crate::format`]. #[test] - fn pin_record_format_v0() { + fn pin_record_format_v1() { let record = Boundary { network: network(1), - burned: { - let mut burned = Bloom::new(); - burned.insert(&network_key(network(2))); - burned - }, - executed: executed(), job: Some(Committed { session: "abandon-ability".parse().unwrap(), + seq: seq(1_760_000_000_000), job: "zoo-zero".parse().unwrap(), successor: "able-about".parse().unwrap(), outcome: JobOutcome::Committed, }), }; let bytes = format::encode(&record); - let path = "tests/output/boundary-record-v0.bin"; + let path = "tests/output/boundary-record-v1.bin"; if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { std::fs::write(path, &bytes).unwrap(); } else { @@ -425,12 +453,10 @@ mod test { } let decoded: Boundary = format::decode(&bytes).unwrap(); assert_eq!(decoded.network, record.network); - assert!(decoded.is_burned(network(2))); - assert_eq!(decoded.executed, record.executed); - assert_eq!( - decoded.job.unwrap().successor, - record.job.unwrap().successor - ); + let (decoded, expected) = (decoded.job.unwrap(), record.job.unwrap()); + assert_eq!(decoded.session, expected.session); + assert_eq!(decoded.seq, expected.seq); + assert_eq!(decoded.successor, expected.successor); } /// A record from a newer software version loads as untrusted, and @@ -442,7 +468,7 @@ mod test { #[derive(Serialize)] struct Envelope(u16, #[serde(with = "crate::format::cbor_bytes")] Vec); let mut bytes = Vec::new(); - into_cbor(&Envelope(1, b"from the future".to_vec()), &mut bytes).unwrap(); + into_cbor(&Envelope(2, b"from the future".to_vec()), &mut bytes).unwrap(); let scratch = Locker::new(&test_log(), slots.clone()).unwrap(); scratch.tenant(BOUNDARY).store(&bytes).await.unwrap(); drop(scratch); @@ -451,18 +477,6 @@ mod test { assert!(store.untrusted()); } - /// The burned set's keys are on-disk format: a change to the - /// network's serde shape would silently forget every burn, and a - /// forgotten burn admits a flip-flop instead of refusing it. If - /// this fails, STOP, and see the warning on [`crate::bloom`]. - #[test] - fn pin_network_keys() { - assert_eq!( - network_key(network(1)), - [0x50, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - #[tokio::test] async fn commitments_survive_restarts() { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); @@ -472,7 +486,6 @@ mod test { let (a, mut b) = (boundary(), boundary()); b.network = network(2); - b.burned = a.burned_for(b.network); first.advance(&a).await.unwrap(); first.advance(&b).await.unwrap(); drop(first); @@ -482,11 +495,9 @@ mod test { let recorded = next.boundary().unwrap(); assert_eq!(recorded.network, b.network); assert_eq!(job_of(&recorded), job_of(&b)); - assert!(recorded.is_burned(network(1))); - assert!(!recorded.is_burned(network(3))); - assert_eq!(recorded.executed, b.executed); let (recorded, expected) = (recorded.job.unwrap(), b.job.unwrap()); assert_eq!(recorded.session, expected.session); + assert_eq!(recorded.seq, expected.seq); assert_eq!(recorded.successor, expected.successor); assert!(matches!(recorded.outcome, JobOutcome::Committed)); } diff --git a/server/src/lib.rs b/server/src/lib.rs index 10588c3e..acdf3ab1 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -10,7 +10,6 @@ extern crate function_name; #[cfg(all(feature = "embedded", feature = "test-support"))] compile_error!("`test-support` must not be enabled for an embedded server"); -pub mod bloom; pub mod bookmark; pub mod boundary; pub mod certs; diff --git a/server/src/manager.rs b/server/src/manager.rs index 283818a0..2cbfd495 100644 --- a/server/src/manager.rs +++ b/server/src/manager.rs @@ -31,9 +31,10 @@ use sush_common::authn::{ Nonce, RequestVerifier, SeqWindow, }; use sush_common::jobs::{ - Access, JobId, JobStatusMap, Session, SessionId, SessionSignerNonce, SignedJob, + Access, JobId, JobOutputStream, JobStatusMap, Session, SessionClientNonce, SessionId, + SessionPreimage, SessionSeq, SessionServerNonce, SessionSignerNonce, SessionSushNonce, + SignedJob, }; -use sush_common::jobs::{JobOutputStream, SessionSushNonce}; use sush_common::keys::{KeyError, KeyId, SshPublicKey}; use sush_common::targets::{Cubbies, SledHealth, SledVersion}; use sush_common::version::LONG_VERSION; @@ -87,7 +88,7 @@ pub struct JobManager { log: Logger, nonces: Arc>>, identities: Arc>>, - session_sush_nonce: Arc>, + session_server_nonce: Arc>, output_dir: JobOutputDir, own_baseboard: BaseboardId, linked: LinkedBaseboards, @@ -144,7 +145,7 @@ impl JobManager { info!(log, "starting sush"; "version" => LONG_VERSION); let (tx_req, rx_req) = mpsc::channel(16); let requests = ReceiverStream::new(rx_req); - let session_sush_nonce = Arc::new(SyncMutex::new(SessionSushNonce::random())); + let session_server_nonce = Arc::new(SyncMutex::new(SessionServerNonce::random())); let boundary = Arc::new(BoundaryStore::new(&log, locker)); boundary.load().await; let (rx_state, join_state) = StateManager::run( @@ -156,7 +157,6 @@ impl JobManager { cubbies, universe, anchors, - session_sush_nonce.clone(), boundary, shutdown, )?; @@ -164,7 +164,7 @@ impl JobManager { log: log.new(o!("component" => "job manager")), nonces: Arc::new(Mutex::new(LruCache::new(MAX_OUTSTANDING_NONCES))), identities: Arc::new(Mutex::new(LruCache::new(MAX_CACHED_IDENTITIES))), - session_sush_nonce, + session_server_nonce, own_baseboard, linked, output_dir, @@ -457,28 +457,41 @@ impl JobManager { Ok(session.into_session_id()) } - pub fn session_sush_nonce(&self) -> SessionSushNonce { - *self.session_sush_nonce.lock().unwrap() + /// Replace the current server nonce with a fresh one and return + /// it. A fetch is the only rotation, so a client must finish its + /// ceremony before the next fetch. + pub fn fresh_server_nonce(&self) -> SessionServerNonce { + let nonce = SessionServerNonce::random(); + *self.session_server_nonce.lock().unwrap() = nonce; + nonce } pub async fn session_start( &self, authn: &Identity, session_id: SessionId, + client_nonce: SessionClientNonce, signer_nonce: SessionSignerNonce, + seq: SessionSeq, wait: bool, ) -> Result<(), JobError> { - // Verify that the session ID provided by the client is meant for - // this instance of Sush, by checking that the signer nonce was - // hashed with our baseboard and nonce. The state machine rotates - // the nonce when a session activates. - let sush_nonce = *self.session_sush_nonce.lock().unwrap(); - let computed = SessionId::compute(self.own_baseboard(), sush_nonce, signer_nonce); - if computed != session_id { + // Verify that the session ID provided by the client is meant + // for this instance of Sush: it must hash over our baseboard + // and current server nonce. The preimage then travels instead + // of the id, so every peer derives for itself both the id and + // the sequence number the id commits to. + let server_nonce = *self.session_server_nonce.lock().unwrap(); + let preimage = SessionPreimage { + baseboard: self.own_baseboard().clone(), + sush_nonce: SessionSushNonce::compute(client_nonce, server_nonce), + signer_nonce, + seq, + }; + if preimage.session_id() != session_id { return Err(JobError::InvalidSessionId); } - self.session_request(authn, SessionRequest::Start(session_id)) + self.session_request(authn, SessionRequest::Start(preimage)) .await?; if wait { self.wait_for(self.wait_for_session(session_id)).await?; diff --git a/server/src/messages.rs b/server/src/messages.rs index 8a8a29f7..8dad368e 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -6,15 +6,15 @@ use chrono::{DateTime, Utc}; use ciborium::{Value, de::from_reader as from_cbor, ser::into_writer as into_cbor}; -use rumors::Version; use serde::{Deserialize, Serialize}; use sled_hardware_types::BaseboardId; -use thiserror::Error; use sush_api::JobStartParams; use sush_common::authn::SignedLogin; use sush_common::jobs::JobOutputState; -use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob, SkipReason}; +use sush_common::jobs::{ + Access, JobId, ProcessError, SessionId, SessionPreimage, SignedJob, SkipReason, +}; use sush_common::keys::KeyId; use sush_common::version::VersionInfo; @@ -194,7 +194,7 @@ pub mod v0 { pub fn kind(&self) -> &'static str { match self { Self::Session(r) => match r.request { - SessionRequest::Start(_) => "session start", + SessionRequest::Start(..) => "session start", SessionRequest::Stop(_) => "session stop", SessionRequest::Skip(..) => "session skip", SessionRequest::AllowAttach(..) => "session allow", @@ -213,7 +213,7 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum SessionRequest { - Start(SessionId), + Start(SessionPreimage), Stop(SessionId), Skip(SessionId, JobId), AllowAttach(SessionId, KeyId, Access), @@ -240,7 +240,6 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum Event { Job(JobEvent), - Error(Error), Version(VersionInfo), } @@ -267,25 +266,6 @@ pub mod v0 { } } } - - #[derive(Clone, Debug, Deserialize, Eq, Error, PartialEq, Serialize)] - pub enum Error { - #[error( - "Concurrent sessions detected: \ - ours is {own_session}@{own_version}, \ - incoming is {incoming_session}@{incoming_version}" - )] - ConcurrentSessions { - own_session: SessionId, - own_version: Box, - incoming_session: SessionId, - incoming_version: Box, - }, - #[error("Sled re-entered a burned universe")] - UniverseFlipFlop, - #[error("Sled hopped into a session its record cannot order")] - SessionHop, - } } #[cfg(test)] @@ -297,7 +277,9 @@ mod wire_format { use sush_api::JobWait; use sush_common::authn::{ChallengeResponse, Nonce, RequestVerifier}; use sush_common::hash::hash; - use sush_common::jobs::{JobMode, JobStartRequest}; + use sush_common::jobs::{ + JobMode, JobStartRequest, SessionSeq, SessionSignerNonce, SessionSushNonce, + }; use sush_common::keys::{EncodedSignature, Signed}; use super::v0::*; @@ -344,7 +326,7 @@ mod wire_format { fn snapshot_name(message: &Message) -> &'static str { match message { Message::Request(Request::Session(r)) => match r.request { - SessionRequest::Start(_) => "session-start-request", + SessionRequest::Start(..) => "session-start-request", SessionRequest::Stop(_) => "session-stop-request", SessionRequest::Skip(..) => "session-skip-request", SessionRequest::AllowAttach(..) => "session-allow-attach-request", @@ -369,13 +351,9 @@ mod wire_format { } Event::Job(JobEvent::Error(..)) => "job-error-event", Event::Job(JobEvent::Skipped(_, _, reason)) => match reason { - SkipReason::BelowFloor => "job-skipped-event", - SkipReason::AlreadyHandled => "already-handled-skip-event", + SkipReason::SessionSuperseded => "job-skipped-event", SkipReason::SessionEnded => "session-ended-skip-event", }, - Event::Error(Error::ConcurrentSessions { .. }) => "concurrent-sessions-error", - Event::Error(Error::SessionHop) => "session-hop-error", - Event::Error(Error::UniverseFlipFlop) => "universe-flip-flop-error", }, } } @@ -455,7 +433,15 @@ mod wire_format { }; vec![ - Message::Request(Request::session(actor(), SessionRequest::Start(session))), + Message::Request(Request::session( + actor(), + SessionRequest::Start(SessionPreimage { + baseboard: baseboard.clone(), + sush_nonce: SessionSushNonce::from_be_bytes([0x11; 32]), + signer_nonce: SessionSignerNonce::from_be_bytes([0x22; 32]), + seq: SessionSeq::from_unix_millis(1_760_000_000_000).unwrap(), + }), + )), Message::Request(Request::session(actor(), SessionRequest::Stop(session))), Message::Request(Request::session( actor(), @@ -514,36 +500,17 @@ mod wire_format { Event::Job(JobEvent::Skipped( skip_job, skip_when, - SkipReason::BelowFloor, - )), - ), - Message::Event( - baseboard.clone(), - Event::Job(JobEvent::Skipped( - skip_job, - skip_when, - SkipReason::AlreadyHandled, + SkipReason::SessionSuperseded, )), ), Message::Event( - baseboard.clone(), + baseboard, Event::Job(JobEvent::Skipped( skip_job, skip_when, SkipReason::SessionEnded, )), ), - Message::Event( - baseboard.clone(), - Event::Error(Error::ConcurrentSessions { - own_session: long_session, - own_version: Box::new(Version::new()), - incoming_session: sid("zoo-zoo-zoo-zoo-zoo-zoo-zoo-zebra"), - incoming_version: Box::new(Version::new()), - }), - ), - Message::Event(baseboard.clone(), Event::Error(Error::SessionHop)), - Message::Event(baseboard, Event::Error(Error::UniverseFlipFlop)), ] } @@ -552,9 +519,12 @@ mod wire_format { /// variant cannot leave a stale snapshot behind. #[test] fn wire_formats() { - // The boundary module pins its record format in the same + // The boundary module pins its record formats in the same // directory. - let mut expected = BTreeSet::from(["boundary-record-v0.bin".to_string()]); + let mut expected = BTreeSet::from([ + "boundary-record-v0.bin".to_string(), + "boundary-record-v1.bin".to_string(), + ]); for message in samples() { let name = snapshot_name(&message); assert!( diff --git a/server/src/server.rs b/server/src/server.rs index f1a6d665..565526d0 100644 --- a/server/src/server.rs +++ b/server/src/server.rs @@ -24,11 +24,12 @@ use tokio_tungstenite::tungstenite::protocol::{Role, WebSocketConfig}; use sush_api::{ AccessParam, Authorization, AuthorizedRangeRequest, JobHistoryParams, JobIdParam, JobOutputParams, JobStartParams, JobStopParams, JobTargetParams, KeyIdParam, RoutingParam, - SessionAndJobIds, SessionAndKeyIds, SessionIdParam, SessionStartBody, SessionStartNonce, - SushApi, WaitParam, + SessionAndJobIds, SessionAndKeyIds, SessionIdParam, SessionStartBody, SushApi, WaitParam, }; use sush_common::authn::Identity; -use sush_common::jobs::{JsonJobStatusMap, Session, SignedJob, job_status_to_json_map}; +use sush_common::jobs::{ + JsonJobStatusMap, Session, SessionServerNonce, SignedJob, job_status_to_json_map, +}; use sush_common::keys::{SshPublicKey, pem_cert_chain}; use sush_common::targets::SledVersion; use sush_common::version::VersionInfo; @@ -118,18 +119,16 @@ impl SushApi for ApiServer { } } - async fn session_start_nonce( + async fn server_nonce( ctx: RequestContext, headers: Header, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let mgr = ctx.context(); let Authorization { authorization } = headers.into_inner(); let _authn = mgr .iam(authorization, None, request_line(&ctx.request)) .await?; - Ok(HttpResponseOk(SessionStartNonce { - nonce: mgr.session_sush_nonce(), - })) + Ok(HttpResponseOk(mgr.fresh_server_nonce())) } async fn session_start( @@ -146,8 +145,12 @@ impl SushApi for ApiServer { .await?; let WaitParam { wait } = query.into_inner(); let SessionIdParam { session_id } = params.into_inner(); - let SessionStartBody { signer_nonce } = body.into_inner(); - mgr.session_start(&authn, session_id, signer_nonce, wait) + let SessionStartBody { + client_nonce, + signer_nonce, + seq, + } = body.into_inner(); + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, wait) .await?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/server/src/state.rs b/server/src/state.rs index 5bc68c64..d7bad03a 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -8,7 +8,7 @@ //! messages via the gossip protocol. use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use chrono::{DateTime, Utc}; use futures::{FutureExt as _, Stream, StreamExt}; @@ -23,14 +23,13 @@ use x509_cert::Certificate; use sush_api::JobStartParams; use sush_common::jobs::{ - Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, - SessionSushNonce, SignedJob, SkipReason, VerifiedJob, + Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, SessionSeq, + SignedJob, SkipReason, VerifiedJob, }; use sush_common::keys::{KeyError, KeyId}; use sush_common::targets::Cubbies; use sush_common::version::VersionInfo; -use crate::bloom::Bloom; use crate::bookmark::SushBookmark; use crate::boundary::{Boundary, BoundaryStore, Committed, JobOutcome}; use crate::certs::Certificates; @@ -40,7 +39,7 @@ use crate::history::JobHistory; use crate::identities::Identities; use crate::job::SocketSender; use crate::locker::Locker; -use crate::messages::v0::{Error, Event, JobEvent, JobRequest, Message, Request, SessionRequest}; +use crate::messages::v0::{Event, JobEvent, JobRequest, Message, Request, SessionRequest}; use crate::messages::{VersionedMessage, VersionedMessage::*}; use crate::output::JobOutputDir; use crate::rack::Rack; @@ -66,18 +65,21 @@ pub struct QueuedJob { pub replayed: bool, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub enum SessionState { - Inactive { - /// Join of all observed session start requests. - starts: Version, - }, + #[default] + Inactive, Active { - /// Join of all observed session start requests. - starts: Version, + /// Where the active session falls in the signer-assigned + /// order. A start wins the slot only by exceeding this + /// sequence number, with ties broken by session id. + seq: SessionSeq, - /// When the active session started. - started: Version, + /// Whether this session's start arrived as live traffic + /// rather than replayed history. A session replayed onto a + /// sled with no boundary record may hold jobs a previous + /// life ran, so [`Past::screen`] refuses it. + live: bool, /// The active session. session: Box, @@ -91,18 +93,10 @@ pub enum SessionState { } impl SessionState { - fn starts(&self) -> Version { - use SessionState::*; - match self { - Inactive { starts } => starts.clone(), - Active { starts, .. } => starts.clone(), - } - } - pub fn session(&self) -> Option<&Session> { use SessionState::*; match self { - Inactive { .. } => None, + Inactive => None, Active { session, .. } => Some(session), } } @@ -128,111 +122,48 @@ impl SessionState { fn queued_jobs(&self) -> Option<&QueuedJobs> { use SessionState::*; match self { - Inactive { .. } => None, + Inactive => None, Active { queued_jobs, .. } => Some(queued_jobs), } } } -impl Default for SessionState { - fn default() -> Self { - Self::Inactive { - starts: Version::new(), - } - } -} - -/// Whether we have a job boundary we can rely on, and if so what the -/// execution floor is. -/// -/// The floor lives in memory only, since a floor written to disk would -/// reach the next life, which might join a different universe; and because -/// versions don't compare across universes, every session would look -/// concurrent with the floor, leading to a permanent lock-out. Instead, -/// we raise a fresh floor at each universe entry from messages already -/// there, so future sessions come to dominate it. +/// Whether we have a job boundary we can rely on. #[derive(Debug)] enum Commitment { - /// Tabula rasa. The floor is raised when we join a universe with - /// history. - Absent { floor: Option }, - - /// We have a valid boundary record. The floor is raised on re-entering - /// a universe, or by a session hop. - Trusted { - boundary: Box, - floor: Option, - }, + /// Tabula rasa: no life has committed to a job here. + Absent, + + /// We have a valid boundary record. + Trusted { boundary: Box }, /// The boundary record is corrupt; no jobs may run here. Untrusted, } impl Commitment { - /// Judge the boundary record at entry into a universe. On a flip-flop - /// (re-entering a burned universe), the error to report is also returned. - fn at_entry( - log: &Logger, - store: &BoundaryStore, - rumors: &GossipNetwork, - ) -> (Self, Option) { + /// Judge the boundary record at startup. + fn at_entry(store: &BoundaryStore) -> Self { if store.untrusted() { - return (Self::Untrusted, None); + return Self::Untrusted; } - - let network = rumors.network(); - let entry = rumors.snapshot().latest().clone(); match store.boundary() { - Some(boundary) if boundary.network != network && boundary.is_burned(network) => { - warn!(log, "re-entered a burned universe"; "committed" => ?boundary.job); - ( - Self::Trusted { - boundary: Box::new(boundary), - floor: Some(entry), - }, - Some(Error::UniverseFlipFlop), - ) - } - Some(boundary) => ( - Self::Trusted { - boundary: Box::new(boundary), - floor: None, - }, - None, - ), - None if entry != Version::new() => { - warn!(log, "no boundary record, but this universe has history"); - (Self::Absent { floor: Some(entry) }, None) - } - None => (Self::Absent { floor: None }, None), - } - } - - /// Raise the execution floor. An untrusted commitment refuses - /// every job already, so it carries no floor to raise. - fn raise(&mut self, to: Version) { - match self { - Self::Absent { floor } | Self::Trusted { floor, .. } => *floor = Some(to), - Self::Untrusted => {} + Some(boundary) => Self::Trusted { + boundary: Box::new(boundary), + }, + None => Self::Absent, } } } -/// What [`Past::screen`] decides about a live job of the active -/// session. +/// What [`Past::screen`] decides about a job of the active session. enum Admission { /// Execute normally. Admit, - /// The session does not start above the execution floor, - /// so skip the job. + /// Refuse to execute the job because the session has been superseded. Refuse, - /// We have a boundary record, but it's incommensurate with the - /// current session. So "hop" into the new session, and raise the - /// execution floor. - Hop, - /// If we cannot trust our boundary record, we must not admit /// new jobs. Untrusted, @@ -252,14 +183,13 @@ pub struct Past { /// life computes its own. birth: Version, - /// The most recently seen message version. Initialized to `birth`, - /// then joined at every state machine update. - frontier: Version, - /// What job, if any, this sled committed to executing. Durable /// across restarts. committed: Commitment, + /// The gossip universe this life entered. + network: Network, + /// Jobs whose replayed start events say a previous life ran them. /// The end of such a job may replay later and remove it from the /// set, so which jobs truly lack an end is known only when replay @@ -277,60 +207,39 @@ pub struct Past { adjudicated: Option, } -/// Whether `version` strictly dominates `mark`. Equal and concurrent -/// versions do not. -fn dominates(version: &Version, mark: &Version) -> bool { - version > mark -} - impl Past { - fn new(birth: Version, committed: Commitment) -> Self { + fn new(birth: Version, committed: Commitment, network: Network) -> Self { Past { - frontier: birth.clone(), birth, committed, + network, zombies: BTreeSet::new(), reaped: BTreeSet::new(), adjudicated: None, } } - fn observe(&mut self, version: &Version) { - self.frontier |= version; - } - - /// Decide whether the active session, identified by its id and - /// the version of its start message, may run a live job here. - fn screen(&self, network: Network, session: SessionId, started: &Version) -> Admission { + /// Decide whether the active session may run a job here. + /// A committed session may only resume in the network where + /// this sled committed to it, and sessions are superseded + /// by ones with a higher sequence number. + fn screen(&self, session: SessionId, seq: SessionSeq, live: bool) -> Admission { match &self.committed { - // Refuse all jobs if our boundary is untrusted. Commitment::Untrusted => Admission::Untrusted, - - // We must also refuse anything below our floor, - // since we cannot tell replay from re-run there. - Commitment::Absent { floor: Some(floor) } - | Commitment::Trusted { - floor: Some(floor), .. - } if !dominates(started, floor) => Admission::Refuse, - - // Allow jobs in the absence of any previous commitment. - Commitment::Absent { .. } => Admission::Admit, - - // Jobs cannot cross universes, and anything a previous life - // ran in this universe is below the floor checked above. - Commitment::Trusted { boundary, .. } => { - if boundary.network != network { - Admission::Admit - } else if let Some(job) = &boundary.job - && job.session == session + Commitment::Trusted { boundary } => match &boundary.job { + Some(committed) + if (boundary.network == self.network && committed.session == session) + || seq > committed.seq => { Admission::Admit - } else if dominates(started, &boundary.executed) { - Admission::Admit - } else { - Admission::Hop } - } + Some(_) => Admission::Refuse, + None if live => Admission::Admit, + None => Admission::Refuse, + }, + + Commitment::Absent if live => Admission::Admit, + Commitment::Absent => Admission::Refuse, } } @@ -338,17 +247,16 @@ impl Past { fn commit_to( &self, network: Network, - started: &Version, session_id: SessionId, + seq: SessionSeq, job_id: JobId, successor: JobId, ) -> Boundary { Boundary { network, - burned: self.burned_for(network), - executed: self.executed_for(network, started), job: Some(Committed { session: session_id.to_owned(), + seq, job: job_id, successor, outcome: JobOutcome::Committed, @@ -356,25 +264,6 @@ impl Past { } } - /// The burned set for a record committed in `network`; see - /// [`Boundary::burned_for`]. - fn burned_for(&self, network: Network) -> Bloom { - match &self.committed { - Commitment::Trusted { boundary, .. } => boundary.burned_for(network), - _ => Bloom::default(), - } - } - - /// The executed-session join for a record committed in `network` - /// under a session started at `started`; see - /// [`Boundary::executed_for`]. - fn executed_for(&self, network: Network, started: &Version) -> Version { - match &self.committed { - Commitment::Trusted { boundary, .. } => boundary.executed_for(network, started), - _ => started.clone(), - } - } - /// Whether a message at `version` is live traffic rather than /// replayed history; see [`Past::birth`]. Equality admits the /// birth announcement itself. @@ -403,7 +292,6 @@ enum Output { Ran(JobStatus), Stop(JobId), Gossip(Message), - RotateNonce, } impl Output { @@ -540,8 +428,8 @@ impl State { network: Option, ) -> Vec { let mut outputs = self.reap_zombies(log, now); - if let Some(network) = network { - outputs.extend(self.adjudicate_boundary(log, now, network)); + if network.is_some() { + outputs.extend(self.adjudicate_boundary(log, now)); } outputs } @@ -575,12 +463,7 @@ impl State { /// other universe never saw the job and hears nothing, and /// re-entering the job's own universe replays the earlier ruling /// as a status, which stops a second announcement. - fn adjudicate_boundary( - &mut self, - log: &Logger, - now: DateTime, - network: Network, - ) -> Vec { + fn adjudicate_boundary(&mut self, log: &Logger, now: DateTime) -> Vec { // An absent or untrusted record commits to no job, so there // is nothing to rule on. let Commitment::Trusted { boundary, .. } = &self.past.committed else { @@ -589,7 +472,7 @@ impl State { let Some(Committed { job, outcome, .. }) = boundary.job.as_ref() else { return Vec::new(); }; - if boundary.network != network { + if boundary.network != self.past.network { return Vec::new(); } let (job_id, outcome) = (*job, outcome.clone()); @@ -659,7 +542,6 @@ impl State { incoming_version: &Version, message: &Arc, ) -> Vec { - self.past.observe(incoming_version); match message.as_ref() { V0(Message::Request(request)) => { let actor = request.actor(); @@ -713,87 +595,60 @@ impl State { let mut outputs = Vec::new(); match request { - SessionRequest::Start(session_id) => match &mut self.session { - // Re-announcement of active session; absorb and ignore. - Active { - starts, session, .. - } if session.session_id() == *session_id => { - *starts |= incoming_version.clone(); - info!(log, "duplicate session start"; "session_id" => %session_id); - } + SessionRequest::Start(preimage) => { + let session_id = preimage.session_id(); + match &self.session { + // Re-announcement of the active session; ignore. + Active { session, .. } if session.session_id() == session_id => { + info!(log, "duplicate session start"; "session_id" => %session_id); + } - // If the incoming new session is in the *strict - // causal future* of everything we have observed, - // adopt it regardless of our current state. Every - // other peer will make the same decision, so it - // will come to dominate the network entirely. - Inactive { starts } | Active { starts, .. } if *incoming_version > *starts => { - info!(log, "session started"; "session_id" => %session_id, "actor" => %actor); - let mut session = Session::started(*session_id, actor.clone()); - if let Commitment::Trusted { boundary, .. } = &self.past.committed - && let Some(committed) = boundary - .job - .as_ref() - .filter(|committed| committed.session == *session_id) + // A start wins the slot only by exceeding the + // active session in the signer-assigned order, + // with ties broken by id. This one does not; + // ignore it. + Active { seq, session, .. } + if (preimage.seq, session_id) <= (*seq, session.session_id()) => { info!( - log, "resuming committed session at its successor"; + log, "stale session start"; "session_id" => %session_id, - "successor" => %committed.successor, + "seq" => %preimage.seq, + "active_seq" => %seq, ); - session.resume_at(committed.successor); } - outputs.extend(self.skip_queued_jobs(now)); - outputs.push(Output::RotateNonce); - self.session = Active { - starts: self.session.starts() | incoming_version.clone(), - started: incoming_version.clone(), - session: Box::new(session), - queued_jobs: QueuedJobs::new(), - attach_grants: BTreeMap::new(), - }; - } - - // When the incoming session is concurrent, we can't - // establish which is older, so we have to kill both - // sessions. Every peer will make this same decision, - // so as a partition resolves, both sessions will be - // killed everywhere. - Active { - starts, - started, - session, - .. - } if incoming_version.partial_cmp(starts).is_none() => { - let starts = starts.to_owned(); - let error = Error::ConcurrentSessions { - own_session: session.session_id(), - own_version: Box::new(started.clone()), - incoming_session: *session_id, - incoming_version: Box::new(incoming_version.clone()), - }; - outputs.extend(self.skip_queued_jobs(now)); - outputs.push(Output::Gossip(Message::Event( - self.rack.own_baseboard().to_owned(), - Event::Error(error), - ))); - self.session = Inactive { - starts: starts | incoming_version.clone(), - }; - } - // Stale or concurrent session while inactive; OBE. - // Absorb and ignore. - Inactive { starts, .. } | Active { starts, .. } => { - info!( - log, "stale session start"; - "session_id" => %session_id, - "starts" => %starts, - "incoming_version" => %incoming_version, - ); - *starts |= incoming_version.clone(); + // This one does. Every peer applies the same + // total order, so every peer converges on the + // same session. + Inactive | Active { .. } => { + info!(log, "session started"; "session_id" => %session_id, "actor" => %actor); + let mut session = Session::started(session_id, actor.clone()); + if let Commitment::Trusted { boundary } = &self.past.committed + && boundary.network == self.past.network + && let Some(committed) = boundary + .job + .as_ref() + .filter(|committed| committed.session == session_id) + { + info!( + log, "resuming committed session at its successor"; + "session_id" => %session_id, + "successor" => %committed.successor, + ); + session.resume_at(committed.successor); + } + outputs.extend(self.skip_queued_jobs(now)); + self.session = Active { + seq: preimage.seq, + live: self.past.is_live(incoming_version), + session: Box::new(session), + queued_jobs: QueuedJobs::new(), + attach_grants: BTreeMap::new(), + }; + } } - }, + } SessionRequest::Stop(session_id) => { // Any session stop request for a session that isn't // ours is silently ignored. Even in the case of @@ -805,9 +660,7 @@ impl State { // ordering above when handling session-start. Together, // this means that we can't fail to react to a stop for // the active session. - if let Active { - starts, session, .. - } = &self.session + if let Active { session, .. } = &self.session && session.session_id() == *session_id { if session.started_by() != Some(actor) { @@ -821,9 +674,7 @@ impl State { "session_id" => %session_id, "actor" => %actor, ); outputs.extend(self.skip_queued_jobs(now)); - self.session = Inactive { - starts: starts.clone(), - }; + self.session = Inactive; } } } @@ -1127,9 +978,6 @@ impl State { outputs.push(Output::Ran(status)); } } - Event::Error(error) => { - error!(log, "session error"; "error" => %error); - } Event::Version(info) => { if self.rack.known_version(baseboard_id.clone(), info.clone()) { info!(log, "sled version"; "version" => %info); @@ -1166,7 +1014,7 @@ impl State { && queued_jobs.len() >= MAX_QUEUED_JOBS && !queued_jobs.contains_key(&job_id) { - // We have no choice; drop the job on the floor. + // We have no choice; discard the job. warn!(log, "too many jobs queued"; "job_id" => %job_id, "max" => MAX_QUEUED_JOBS); } else { // Insert the job into our queue. Every job joins the @@ -1236,7 +1084,8 @@ impl State { let mut outputs = Vec::new(); let SessionState::Active { session, - started, + seq, + live, queued_jobs, .. } = &mut self.session @@ -1263,20 +1112,7 @@ impl State { break; } - let admission = self.past.screen(network, session.session_id(), started); - if matches!(admission, Admission::Hop) { - warn!( - log, "hopping sessions, raising the execution floor"; - "session_id" => %session.session_id(), "started" => ?started, - ); - self.past.committed.raise(self.past.frontier.clone()); - outputs.push(Output::Gossip(Message::Event( - self.rack.own_baseboard().to_owned(), - Event::Error(Error::SessionHop), - ))); - continue; - } - + let admission = self.past.screen(session.session_id(), *seq, *live); let QueuedJob { job: request, params, @@ -1304,15 +1140,11 @@ impl State { ), )); } else if matches!(admission, Admission::Refuse) { - // Below the floor the queue skips forward in chain - // order. A live job gets the terminal skip status; - // a replayed one is history, and re-reporting it - // on every rejoin would grow the message set. if !replayed { - warn!(log, "skipping job below the execution floor"; "job_id" => %job_id); + warn!(log, "skipping a superseded session's job"; "job_id" => %job_id); outputs.push(Output::job_event( self.rack.own_baseboard(), - JobEvent::Skipped(job_id, now, SkipReason::BelowFloor), + JobEvent::Skipped(job_id, now, SkipReason::SessionSuperseded), )); } } else if self @@ -1345,8 +1177,8 @@ impl State { attachment, boundary: self.past.commit_to( network, - started, session.session_id(), + *seq, job_id, session .session_id() @@ -1376,7 +1208,6 @@ fn apply( executor: &mut Executor, rumors: Option<&GossipNetwork>, store: &Arc, - nonce: &Arc>, input: Input, ) { let now = Utc::now(); @@ -1385,14 +1216,13 @@ fn apply( tx_state.send_modify(|state| { outputs = state.step(log, now, network, input); }); - apply_outputs(executor, rumors, store, nonce, outputs); + apply_outputs(executor, rumors, store, outputs); } fn apply_outputs( executor: &mut Executor, rumors: Option<&GossipNetwork>, store: &Arc, - nonce: &Arc>, outputs: Vec, ) { for output in outputs { @@ -1417,9 +1247,6 @@ fn apply_outputs( gossip(rumors, message.into()); } } - Output::RotateNonce => { - *nonce.lock().unwrap() = SessionSushNonce::random(); - } } } } @@ -1433,7 +1260,6 @@ fn drain_ready( executor: &mut Executor, rumors: Option<&GossipNetwork>, store: &Arc, - nonce: &Arc>, ) { while let Some(Some((version, message))) = causal_messages.next().now_or_never() { apply( @@ -1442,25 +1268,21 @@ fn drain_ready( executor, rumors, store, - nonce, Input::Gossip(version, message), ); } } /// Enter a universe: judge the boundary record, announce this build, -/// and mark the birth. The announce is this life's first send, and a -/// flip-flop report follows it. The birth is the version of whichever -/// went out last. +/// and mark the birth at the announce's version. fn enter_universe( - log: &Logger, own_baseboard: BaseboardId, anchors: &[Certificate], store: &BoundaryStore, rumors: &GossipNetwork, ) -> Result { - let (committed, report) = Commitment::at_entry(log, store, rumors); - let mut birth = gossip( + let committed = Commitment::at_entry(store); + let birth = gossip( rumors, Message::Event( own_baseboard.clone(), @@ -1468,13 +1290,11 @@ fn enter_universe( ) .into(), ); - if let Some(report) = report { - birth = gossip( - rumors, - Message::Event(own_baseboard.clone(), Event::Error(report)).into(), - ); - } - State::new(own_baseboard, anchors, Past::new(birth, committed)) + State::new( + own_baseboard, + anchors, + Past::new(birth, committed, rumors.network()), + ) } /// Send a message into the gossip set, returning the version it was @@ -1525,7 +1345,6 @@ impl StateManager { mut cubbies: watch::Receiver, mut universe: watch::Receiver, anchors: &[Certificate], - session_sush_nonce: Arc>, store: Arc, shutdown: CancellationToken, ) -> Result<(watch::Receiver, JoinHandle<()>), KeyError> @@ -1555,7 +1374,7 @@ impl StateManager { let mut initial_state = State::new( own_baseboard.clone(), anchors, - Past::new(Version::new(), Commitment::Absent { floor: None }), + Past::new(Version::new(), Commitment::Absent, initial.network()), )?; initial_state .rack @@ -1587,14 +1406,9 @@ impl StateManager { // processed, so the birth mark splits this life's // traffic from replayed history. if let Some((rumors, _)) = &network { - let entered = enter_universe( - &log, - own_baseboard.clone(), - &anchors, - &boundary_store, - rumors, - ) - .expect("trust anchors validated at startup"); + let entered = + enter_universe(own_baseboard.clone(), &anchors, &boundary_store, rumors) + .expect("trust anchors validated at startup"); tx_state.send_modify(|state| { *state = entered; state.rack.set_cubbies(cubbies.borrow_and_update().clone()); @@ -1694,7 +1508,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, Input::Gossip(version, message), ); // Settle only after replayed traffic: @@ -1711,7 +1524,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, ); apply( &log, @@ -1719,7 +1531,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, Input::Drained, ); } @@ -1734,7 +1545,6 @@ impl StateManager { &mut executor, network.as_ref().map(|(rumors, _)| rumors), &boundary_store, - &session_sush_nonce, Input::Cubbies(cubbies.borrow_and_update().clone()), ); }, @@ -1771,7 +1581,6 @@ impl StateManager { doomed.insert(job_id); } let entered = enter_universe( - &log, own_baseboard.clone(), &anchors, &boundary_store, @@ -1791,7 +1600,6 @@ impl StateManager { &mut executor, Some(&*rumors), &boundary_store, - &session_sush_nonce, ); apply( &log, @@ -1799,7 +1607,6 @@ impl StateManager { &mut executor, Some(&*rumors), &boundary_store, - &session_sush_nonce, Input::Drained, ); } @@ -1813,10 +1620,12 @@ impl StateManager { mod test { use std::time::Duration; - use rumors::before::Clock; use x509_cert::time::Validity; - use sush_common::jobs::{JobMode, JobOutputState, JobStartRequest}; + use sush_common::jobs::{ + JobMode, JobOutputState, JobStartRequest, SessionPreimage, SessionSignerNonce, + SessionSushNonce, + }; use sush_common::keys::{EphemeralKey, KeyType, Signer as _}; use sush_common::targets::Target; @@ -1885,10 +1694,9 @@ mod test { fn committed_boundary(job: JobId) -> Boundary { Boundary { network: network(1), - burned: Bloom::new(), - executed: Version::new(), job: Some(Committed { session: SessionId::random(), + seq: seq(10), job, successor: JobId::random(), outcome: JobOutcome::Committed, @@ -1906,186 +1714,91 @@ mod test { ) } - fn past(committed: Option, floor: Option) -> Past { + fn past(committed: Option) -> Past { let committed = match committed { Some(boundary) => Commitment::Trusted { boundary: Box::new(boundary), - floor, }, - None => Commitment::Absent { floor }, + None => Commitment::Absent, }; - Past::new(Version::new(), committed) - } - - /// Versions with known causal relations: `older < started < - /// newer`, with `concurrent` unordered against all three. - struct Versions { - older: Version, - started: Version, - newer: Version, - concurrent: Version, - } - - fn versions() -> Versions { - let mut a = Clock::seed(); - let mut b = a.fork(); - Versions { - older: a.tick().clone(), - started: a.tick().clone(), - newer: a.tick().clone(), - concurrent: b.tick().clone(), + Past::new(Version::new(), committed, network(1)) + } + + fn seq(millis: u64) -> SessionSeq { + SessionSeq::from_unix_millis(millis).unwrap() + } + + /// A preimage for a fresh session authorized at `millis`; tests + /// reference the session by the derived id. + fn preimage(millis: u64) -> SessionPreimage { + SessionPreimage { + baseboard: baseboard(), + sush_nonce: SessionSushNonce::random(), + signer_nonce: SessionSignerNonce::random(), + seq: seq(millis), } } - /// The admission rules: a sled with no record admits, a record - /// from a foreign universe admits, the committed session admits - /// outright, a session started strictly above the executed join - /// admits, and everything else hops. + /// The admission rules: the committed session resumes even from + /// replay, a session above the committed sequence number admits, + /// and everything at or below it refuses, in any universe. #[test] fn admission_rules() { - let Versions { - older, - started, - newer, - concurrent, - } = versions(); let session = SessionId::random(); - let job = JobId::random(); let committed = Boundary { network: network(1), - burned: Bloom::new(), - executed: started.clone(), job: Some(Committed { session, - job, + seq: seq(10), + job: JobId::random(), successor: JobId::random(), outcome: JobOutcome::Committed, }), }; + let past = past(Some(committed)); - let recordless = past(None, None); - assert!(matches!( - recordless.screen(network(1), session, &started), - Admission::Admit - )); - - let past = past(Some(committed), None); - assert!(matches!( - past.screen(network(2), session, &started), - Admission::Admit - )); - // The committed session admits outright: it resumes at the - // stored successor when it activates, and chain position - // keeps every earlier job from popping. assert!(matches!( - past.screen(network(1), session, &started), + past.screen(session, seq(10), false), Admission::Admit )); assert!(matches!( - past.screen(network(1), SessionId::random(), &newer), + past.screen(SessionId::random(), seq(11), false), Admission::Admit )); - for unordered in [&older, &started, &concurrent] { + for at_or_below in [seq(10), seq(9)] { assert!(matches!( - past.screen(network(1), SessionId::random(), unordered), - Admission::Hop + past.screen(SessionId::random(), at_or_below, true), + Admission::Refuse )); } } - /// A floor refuses every session not started strictly above it, - /// in every universe and under either commitment that carries one: - /// the floor belongs to this life, not to any record. + /// With no record, a live session admits and a replayed one + /// refuses: a sled that has never written a boundary cannot tell + /// replay from re-run. #[test] - fn floors_refuse_below() { - let Versions { - older: below, - started: floor, - newer: above, - concurrent, - } = versions(); - let at: Version = floor.clone(); - let committed = Boundary { - network: network(1), - burned: Bloom::new(), - executed: floor.clone(), - job: None, - }; - - for past in [ - past(None, Some(floor.clone())), - past(Some(committed), Some(floor.clone())), - ] { - for started in [&at, &below, &concurrent] { - for net in [network(1), network(2)] { - assert!(matches!( - past.screen(net, SessionId::random(), started), - Admission::Refuse - )); - } - } - assert!(matches!( - past.screen(network(1), SessionId::random(), &above), - Admission::Admit - )); - } - } - - /// A hop raises the floor to the frontier, so the hopped session - /// screens as a refusal from then on: one hop report per session. - /// A session started strictly above the frontier admits again. - #[test] - fn raised_floor_refuses_hopped_session() { - let mut a = Clock::seed(); - let mut b = a.fork(); - let started = a.tick().clone(); - let executed = b.tick().clone(); - let above = started.clone() | executed.clone() | a.tick().clone(); - let session = SessionId::random(); - let committed = Boundary { - network: network(1), - burned: Bloom::new(), - executed: executed.clone(), - job: None, - }; - - let mut past = past(Some(committed), None); - past.observe(&started); - past.observe(&executed); - assert_eq!(past.frontier, &started | executed.clone()); + fn recordless_admission_is_live_only() { + let past = past(None); assert!(matches!( - past.screen(network(1), session, &started), - Admission::Hop + past.screen(SessionId::random(), seq(1), true), + Admission::Admit )); - - past.committed.raise(past.frontier.clone()); assert!(matches!( - past.screen(network(1), session, &started), + past.screen(SessionId::random(), seq(1), false), Admission::Refuse )); - assert!(matches!( - past.screen(network(1), session, &above), - Admission::Admit - )); } - /// An untrusted record admits nothing in any universe, and a - /// raise leaves it untrusted. + /// An untrusted record admits nothing, live or replayed. #[test] fn untrusted_admits_nothing() { - let Versions { started, .. } = versions(); - let mut past = Past::new(Version::new(), Commitment::Untrusted); - for net in [network(1), network(2)] { + let past = Past::new(Version::new(), Commitment::Untrusted, network(1)); + for live in [true, false] { assert!(matches!( - past.screen(net, SessionId::random(), &started), + past.screen(SessionId::random(), seq(1), live), Admission::Untrusted )); } - past.committed.raise(started.clone()); - assert!(matches!( - past.screen(network(1), SessionId::random(), &started), - Admission::Untrusted - )); } /// Settle declares a zombie interrupted exactly once. @@ -2094,7 +1807,7 @@ mod test { let log = discard(); let now = Utc::now(); let zombie = JobId::random(); - let mut state = state(past(None, None)); + let mut state = state(past(None)); state.past.zombies.insert(zombie); start_job(&mut state, zombie, now); @@ -2113,9 +1826,12 @@ mod test { let log = discard(); let now = Utc::now(); let job = JobId::random(); - let mut state = state(past(Some(committed_boundary(job)), None)); + // A sled that entered another universe never saw the job. + let mut elsewhere = state(past(Some(committed_boundary(job)))); + elsewhere.past.network = network(2); + assert!(elsewhere.settle(&log, now, Some(network(2))).is_empty()); - assert!(state.settle(&log, now, Some(network(2))).is_empty()); + let mut state = state(past(Some(committed_boundary(job)))); let outputs = state.settle(&log, now, Some(network(1))); assert_eq!(outputs.len(), 1); assert!(interrupted(&outputs[0], job)); @@ -2130,7 +1846,7 @@ mod test { let now = Utc::now(); let job = JobId::random(); - let mut witnessed = state(past(Some(committed_boundary(job)), None)); + let mut witnessed = state(past(Some(committed_boundary(job)))); start_job(&mut witnessed, job, now); assert!(witnessed.settle(&log, now, Some(network(1))).is_empty()); assert_eq!(witnessed.past.adjudicated, Some(job)); @@ -2160,7 +1876,7 @@ mod test { } let store = BoundaryStore::new(&log, &Locker::null()); store.load().await; - let state = enter_universe(&log, baseboard(), anchors, &store, &rumors).unwrap(); + let state = enter_universe(baseboard(), anchors, &store, &rumors).unwrap(); let messages = rumors.causal_messages(); Sim { log, @@ -2178,8 +1894,7 @@ mod test { let rumors: GossipNetwork = Seed::grow(&self.log, &Locker::null()).await.into_rumors(); let store = BoundaryStore::new(&self.log, &Locker::null()); store.load().await; - self.state = - enter_universe(&self.log, baseboard(), &self.anchors, &store, &rumors).unwrap(); + self.state = enter_universe(baseboard(), &self.anchors, &store, &rumors).unwrap(); self.messages = rumors.causal_messages(); self.rumors = rumors; } @@ -2238,10 +1953,11 @@ mod test { let root = root(); let anchors = vec![root.cert().clone()]; let mut sim = Sim::new(&anchors, Vec::new()).await; - let session_id = SessionId::random(); + let preimage = preimage(1); + let session_id = preimage.session_id(); sim.request(Request::session( root.key_id().clone(), - SessionRequest::Start(session_id), + SessionRequest::Start(preimage), )); (root, sim, session_id, Session::new(session_id)) } @@ -2256,13 +1972,13 @@ mod test { let actor = root.key_id().clone(); let mut sim = Sim::new(&anchors, Vec::new()).await; - let session_id = SessionId::random(); + let preimage = preimage(1); + let session_id = preimage.session_id(); let session = Session::new(session_id); - let outputs = sim.request(Request::session( + sim.request(Request::session( actor.clone(), - SessionRequest::Start(session_id), + SessionRequest::Start(preimage), )); - assert!(outputs.iter().any(|o| matches!(o, Output::RotateNonce))); assert!(sim.state.session.session().is_some()); let job_id = session.next_job_id(); @@ -2306,46 +2022,117 @@ mod test { assert!(sim.state.session.session().is_none()); } - /// A session the boundary record cannot order hops: the floor - /// rises, one report goes out, and the session's jobs are skipped - /// below the floor rather than run. A later job in the same - /// session reports no second hop. + /// Sessions must not be resumed across networks. #[tokio::test] - async fn hop_through_step() { + async fn committed_session_does_not_resume_across_networks() { + let mut root = root(); + let anchors = vec![root.cert().clone()]; + let actor = root.key_id().clone(); + let mut sim = Sim::new(&anchors, Vec::new()).await; + + // A previous life, in another network, committed to job A + // of the session and recorded the chain position after it. + let committed_seq = 5; + let committed_preimage = preimage(committed_seq); + let session_id = committed_preimage.session_id(); + let mut signer = Session::new(session_id); + let job_a_id = signer.next_job_id(); + let job_a = signed_job(&mut root, job_a_id, session_id, "true").await; + signer.job_started(job_a.clone()); + let successor_id = signer.next_job_id(); + let other = network(7); + assert_ne!(sim.rumors.network(), other); + sim.state.past.committed = Commitment::Trusted { + boundary: Box::new(Boundary { + network: other, + job: Some(Committed { + session: session_id, + seq: committed_preimage.seq, + job: job_a_id, + successor: successor_id, + outcome: JobOutcome::Committed, + }), + }), + }; + + // The session's start arrives in this network. + sim.request(Request::session( + actor.clone(), + SessionRequest::Start(committed_preimage), + )); + assert!( + sim.state + .session + .session() + .is_some_and(|s| s.session_id() == session_id) + ); + + // If the committed job arrives again, it skips as superseded + // instead of running. + let outputs = sim.request(Request::job( + actor.clone(), + JobRequest::Start(job_a, JobStartParams::default()), + )); + assert!(!outputs.iter().any(|o| matches!(o, Output::Run(_)))); + assert!(outputs.iter().any(|o| matches!( + o, + Output::Gossip(Message::Event( + _, + Event::Job(JobEvent::Skipped(id, _, SkipReason::SessionSuperseded)), + )) if *id == job_a_id + ))); + + // That job's successor also does not run. + let job_b = signed_job(&mut root, successor_id, session_id, "true").await; + let outputs = sim.request(Request::job( + actor.clone(), + JobRequest::Start(job_b, JobStartParams::default()), + )); + assert!(!outputs.iter().any(|o| matches!(o, Output::Run(_)))); + + // A newer session outranks the committed one and runs. + let next = preimage(committed_seq + 1); + let next_id = next.session_id(); + sim.request(Request::session(actor.clone(), SessionRequest::Start(next))); + let job_c_id = Session::new(next_id).next_job_id(); + let job_c = signed_job(&mut root, job_c_id, next_id, "true").await; + let outputs = sim.request(Request::job( + actor, + JobRequest::Start(job_c, JobStartParams::default()), + )); + assert!(outputs.iter().any(|o| matches!(o, Output::Run(_)))); + } + + /// A session at or below the committed sequence number is + /// refused: its jobs skip as superseded rather than run, and the + /// chain still advances so the session is not wedged. + #[tokio::test] + async fn superseded_session_skips_through_step() { let (mut root, mut sim, session_id, mut session) = sim_with_session().await; let actor = root.key_id().clone(); - // A boundary record whose executed join the session start - // does not dominate: the current frontier, which includes - // the start itself. + // A committed boundary the active session, at sequence + // number 1, does not exceed. sim.state.past.committed = Commitment::Trusted { boundary: Box::new(Boundary { network: sim.rumors.network(), - burned: Bloom::new(), - executed: sim.rumors.snapshot().latest().clone(), - job: None, + job: Some(Committed { + session: SessionId::random(), + seq: seq(10), + job: JobId::random(), + successor: JobId::random(), + outcome: JobOutcome::Committed, + }), }), - floor: None, }; - let hops = |outputs: &[Output]| { - outputs - .iter() - .filter(|o| { - matches!( - o, - Output::Gossip(Message::Event(_, Event::Error(Error::SessionHop))) - ) - }) - .count() - }; let skipped = |outputs: &[Output], job_id: JobId| { outputs.iter().any(|o| { matches!( o, Output::Gossip(Message::Event( _, - Event::Job(JobEvent::Skipped(id, _, SkipReason::BelowFloor)), + Event::Job(JobEvent::Skipped(id, _, SkipReason::SessionSuperseded)), )) if *id == job_id ) }) @@ -2357,7 +2144,6 @@ mod test { actor.clone(), JobRequest::Start(signed.clone(), JobStartParams::default()), )); - assert_eq!(hops(&outputs), 1); assert!(!outputs.iter().any(|o| matches!(o, Output::Run(_)))); assert!(skipped(&outputs, job_id)); @@ -2368,7 +2154,6 @@ mod test { actor, JobRequest::Start(next, JobStartParams::default()), )); - assert_eq!(hops(&outputs), 0); assert!(skipped(&outputs, next_id)); } @@ -2429,14 +2214,15 @@ mod test { let mut root = root(); let anchors = vec![root.cert().clone()]; let actor = root.key_id().clone(); - let session_id = SessionId::random(); + let preimage = preimage(1); + let session_id = preimage.session_id(); let session = Session::new(session_id); let job_id = session.next_job_id(); let signed = signed_job(&mut root, job_id, session_id, "true").await; let history = vec![ Message::Request(Request::session( actor.clone(), - SessionRequest::Start(session_id), + SessionRequest::Start(preimage), )), Message::Request(Request::job( actor, @@ -2470,7 +2256,6 @@ mod test { boundary.network = home; sim.state.past.committed = Commitment::Trusted { boundary: Box::new(boundary), - floor: None, }; }; diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index fd91afcb..fbb4d5ce 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -24,8 +24,9 @@ use tokio_util::sync::CancellationToken; use chrono::Utc; use sush_api::{JobStartParams, JobWait}; use sush_common::jobs::{ - JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, SessionId, - SessionSignerNonce, SignedJob, SkipReason, + JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, + SessionClientNonce, SessionId, SessionPreimage, SessionSeq, SessionSignerNonce, + SessionSushNonce, SignedJob, SkipReason, }; use sush_common::keys::{EphemeralKey, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, SledHealth, SledId, Target}; @@ -160,15 +161,15 @@ async fn jobs_gossip_between_sleds() { // A session started on sled A becomes B's active session too. let authn_a = fake_identity(&mut root).await; let authn_b = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("session gossips to B", 60, async || { @@ -209,14 +210,14 @@ async fn jobs_gossip_between_sleds() { .await; // A session started on B supersedes A's everywhere. - let successor_nonce = SessionSignerNonce::random(); - let successor = SessionId::compute( - b.mgr.own_baseboard(), - b.mgr.session_sush_nonce(), - successor_nonce, - ); + let Ceremony { + session_id: successor, + client_nonce, + signer_nonce, + seq, + } = ceremony(&b.mgr, 2); b.mgr - .session_start(&authn_b, successor, successor_nonce, true) + .session_start(&authn_b, successor, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("supersession gossips to A", 60, async || { @@ -231,7 +232,7 @@ async fn jobs_gossip_between_sleds() { #[named] #[tokio::test] -async fn concurrent_sessions_annihilate() { +async fn partitioned_sessions_converge_on_higher_seq() { let (_tmp, dir) = pki("sush-concurrent-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); @@ -257,26 +258,41 @@ async fn concurrent_sessions_annihilate() { sees(&b, &a).await; // Partitioned, each sled starts its own session. Neither start - // can hear the other, so the two versions are concurrent. + // can hear the other, but the signer-assigned order still ranks + // them. a.peers.send(BTreeSet::new()).unwrap(); b.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; - let nonce_a = SessionSignerNonce::random(); - let session_a = SessionId::compute(a.mgr.own_baseboard(), a.mgr.session_sush_nonce(), nonce_a); + let a_ceremony = ceremony(&a.mgr, 1); + let session_a = a_ceremony.session_id; let mut session = Session::new(session_a); a.mgr - .session_start(&authn_a, session_a, nonce_a, true) + .session_start( + &authn_a, + session_a, + a_ceremony.client_nonce, + a_ceremony.signer_nonce, + a_ceremony.seq, + true, + ) .await .unwrap(); - let nonce_b = SessionSignerNonce::random(); - let session_b = SessionId::compute(b.mgr.own_baseboard(), b.mgr.session_sush_nonce(), nonce_b); + let b_ceremony = ceremony(&b.mgr, 2); + let session_b = b_ceremony.session_id; b.mgr - .session_start(&authn_b, session_b, nonce_b, true) + .session_start( + &authn_b, + session_b, + b_ceremony.client_nonce, + b_ceremony.signer_nonce, + b_ceremony.seq, + true, + ) .await .unwrap(); // A job on A sits queued behind a hole in the chain, so it can - // never run before the sessions annihilate. + // never run before B's session supersedes A's. let hole_id = session.next_job_id(); let hole = sign_job_for(&mut root, hole_id, session_a, "true", &a.baseboard).await; session.job_started(hole); @@ -294,14 +310,23 @@ async fn concurrent_sessions_annihilate() { }) .await; - // The partition heals. Each sled hears a session start concurrent - // with its own: both sessions are killed everywhere, and the - // queued job is skipped rather than left queued forever. + // The partition heals. B's session carries the higher sequence + // number, so every sled converges on it, and A's queued job is + // skipped rather than left queued forever. a.peers.send(BTreeSet::from([b.addr])).unwrap(); b.peers.send(BTreeSet::from([a.addr])).unwrap(); - eventually("the sessions annihilate", 120, async || { - a.mgr.session(&authn_a).is_none() && b.mgr.session(&authn_b).is_none() - }) + eventually( + "the higher sequence number wins everywhere", + 120, + async || { + a.mgr + .session(&authn_a) + .is_some_and(|s| s.session_id() == session_b) + && b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session_b) + }, + ) .await; eventually("the queued job is skipped", 60, async || { a.mgr.job_status(&authn_a, &job_id).await.is_ok_and(|map| { @@ -339,15 +364,15 @@ async fn rejoining_replays_without_reexecuting() { // Sled A runs a whole job before B exists. let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -389,15 +414,22 @@ async fn rejoining_replays_without_reexecuting() { // Live traffic still executes everywhere: a fresh session's job runs // on both sleds. sees(&a, &b).await; - let successor_nonce = SessionSignerNonce::random(); - let successor = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), - successor_nonce, - ); + let Ceremony { + session_id: successor, + client_nonce, + signer_nonce: successor_nonce, + seq, + } = ceremony(&a.mgr, 2); let fresh = Session::new(successor); a.mgr - .session_start(&authn_a, successor, successor_nonce, true) + .session_start( + &authn_a, + successor, + client_nonce, + successor_nonce, + seq, + true, + ) .await .unwrap(); let live_job = fresh.next_job_id(); @@ -588,15 +620,15 @@ async fn stragglers_do_not_interrupt_live_jobs() { .await; sees(&a, &b).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -643,7 +675,35 @@ async fn stragglers_do_not_interrupt_live_jobs() { /// Wait until `anchor` has applied `joiner`'s build announcement. A /// session started on `anchor` afterward causally follows everything -/// `joiner` held at entry, so it clears the joiner's entry floor. +/// `joiner` held at entry, so the joiner serves it. +/// Everything one sled hears at session start: a client nonce, a +/// signer nonce, the signer's clock reading, and the session id over +/// them all. +struct Ceremony { + session_id: SessionId, + client_nonce: SessionClientNonce, + signer_nonce: SessionSignerNonce, + seq: SessionSeq, +} + +/// Build a [`Ceremony`] against one sled. Fetches a fresh server +/// nonce, so earlier ceremonies against that sled go stale. +fn ceremony(mgr: &JobManager, seq: u64) -> Ceremony { + let client_nonce = SessionClientNonce::random(); + let preimage = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce: SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()), + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_millis(seq).unwrap(), + }; + Ceremony { + session_id: preimage.session_id(), + client_nonce, + signer_nonce: preimage.signer_nonce, + seq: preimage.seq, + } +} + async fn sees(anchor: &Sled, joiner: &Sled) { eventually("the anchor sees the joiner", 60, async || { anchor @@ -694,8 +754,8 @@ async fn lost_suffix_never_reruns() { let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; // Sled B keeps its boundary in a locker. It joins before the - // session starts: a record-less sled raises its floor at entry, - // and serves only sessions started after it arrived. + // session starts: a record-less sled serves only sessions + // that started after it arrived. let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); let b_shutdown = CancellationToken::new(); @@ -709,15 +769,15 @@ async fn lost_suffix_never_reruns() { }) .await; sees(&a, &b).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("the session gossips to B", 120, async || { @@ -816,15 +876,15 @@ async fn lost_suffix_never_reruns() { assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); // A new session serves immediately. - let signer_nonce = SessionSignerNonce::random(); - let session2_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id: session2_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 2); let session2 = Session::new(session2_id); a.mgr - .session_start(&authn_a, session2_id, signer_nonce, true) + .session_start(&authn_a, session2_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("the new session gossips to B", 120, async || { @@ -895,15 +955,15 @@ async fn session_resumes_at_stored_successor() { .await; sees(&a, &b).await; sees(&a, &c).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("the session gossips to B and C", 120, async || { @@ -1016,7 +1076,7 @@ async fn session_resumes_at_stored_successor() { #[named] #[tokio::test] -async fn universe_flip_flop_raises_floor() { +async fn supersession_crosses_universes() { let (_tmp, dir) = pki("sush-flipflop-", 3); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); @@ -1034,15 +1094,15 @@ async fn universe_flip_flop_raises_floor() { let authn_a = fake_identity(&mut root).await; let d = Sled::start(&log, &dir, 3, &root_pem, &shutdown).await; let authn_d = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session2_id = SessionId::compute( - d.mgr.own_baseboard(), - d.mgr.session_sush_nonce(), + let Ceremony { + session_id: session2_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&d.mgr, 2); let mut session2 = Session::new(session2_id); d.mgr - .session_start(&authn_d, session2_id, signer_nonce, true) + .session_start(&authn_d, session2_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1060,15 +1120,15 @@ async fn universe_flip_flop_raises_floor() { }) .await; sees(&a, &x).await; - let signer_nonce = SessionSignerNonce::random(); - let session1_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id: session1_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session1 = Session::new(session1_id); a.mgr - .session_start(&authn_a, session1_id, signer_nonce, true) + .session_start(&authn_a, session1_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("session one gossips to X", 120, async || { @@ -1143,9 +1203,10 @@ async fn universe_flip_flop_raises_floor() { }) .await; - // X dies again and flip-flops back to A's universe. Its record - // burned that universe, so X raises its floor: session one's next - // job is refused, and nothing re-runs. + // X dies again and returns to A's universe. Its record holds + // session two's sequence number, which session one does not + // exceed, even from another universe: session one's next job is + // refused, and nothing re-runs. x_shutdown.cancel(); drop(x); d.peers.send(BTreeSet::new()).unwrap(); @@ -1168,13 +1229,13 @@ async fn universe_flip_flop_raises_floor() { .job_start(&authn_x, j3, JobStartParams::default()) .await .unwrap(); - eventually("the floor skips the old session", 120, async || { + eventually("the old session is superseded", 120, async || { a.mgr.job_status(&authn_a, &j3_id).await.is_ok_and(|map| { map.get(&x.baseboard).is_some_and(|s| { matches!( s, JobStatus::Skipped { - reason: SkipReason::BelowFloor, + reason: SkipReason::SessionSuperseded, .. } ) @@ -1183,17 +1244,17 @@ async fn universe_flip_flop_raises_floor() { }) .await; - // A witnessed the refusal, so a session started now begins above - // X's floor, and serves X again. - let signer_nonce = SessionSignerNonce::random(); - let session3_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + // A session with a sequence number above the committed one serves + // X again. + let Ceremony { + session_id: session3_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 3); let session3 = Session::new(session3_id); a.mgr - .session_start(&authn_a, session3_id, signer_nonce, true) + .session_start(&authn_a, session3_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); eventually("session three gossips to X", 120, async || { @@ -1220,7 +1281,7 @@ async fn universe_flip_flop_raises_floor() { map.get(&x.baseboard) .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) }), - "a session started above the floor must serve" + "a session above the committed one must serve" ); // X flops back to D's universe a second time. The job X served in @@ -1247,13 +1308,13 @@ async fn universe_flip_flop_raises_floor() { .job_start(&authn_x, j5, JobStartParams::default()) .await .unwrap(); - eventually("the floor skips session two as well", 120, async || { + eventually("session two is superseded as well", 120, async || { x.mgr.job_status(&authn_x, &j5_id).await.is_ok_and(|map| { map.get(&x.baseboard).is_some_and(|s| { matches!( s, JobStatus::Skipped { - reason: SkipReason::BelowFloor, + reason: SkipReason::SessionSuperseded, .. } ) @@ -1295,15 +1356,15 @@ async fn witnessed_session_survives_restart() { }) .await; sees(&a, &b).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let mut session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1374,14 +1435,14 @@ async fn bookmarks_survive_restart() { // Sled A holds session history, so it wins every dominance contest. let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1478,15 +1539,15 @@ async fn gossip_survives_bookmark_failure() { }) .await; sees(&a, &b).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), + let Ceremony { + session_id, + client_nonce, signer_nonce, - ); + seq, + } = ceremony(&a.mgr, 1); let session = Session::new(session_id); a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) + .session_start(&authn_a, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); diff --git a/server/tests/output/already-handled-skip-event.bin b/server/tests/output/already-handled-skip-event.bin deleted file mode 100644 index 7eed01d4..00000000 Binary files a/server/tests/output/already-handled-skip-event.bin and /dev/null differ diff --git a/server/tests/output/boundary-record-v1.bin b/server/tests/output/boundary-record-v1.bin new file mode 100644 index 00000000..582b74df Binary files /dev/null and b/server/tests/output/boundary-record-v1.bin differ diff --git a/server/tests/output/concurrent-sessions-error.bin b/server/tests/output/concurrent-sessions-error.bin deleted file mode 100644 index f26b5582..00000000 Binary files a/server/tests/output/concurrent-sessions-error.bin and /dev/null differ diff --git a/server/tests/output/job-skipped-event.bin b/server/tests/output/job-skipped-event.bin index 074aa944..1b887084 100644 Binary files a/server/tests/output/job-skipped-event.bin and b/server/tests/output/job-skipped-event.bin differ diff --git a/server/tests/output/session-hop-error.bin b/server/tests/output/session-hop-error.bin deleted file mode 100644 index e872b2aa..00000000 --- a/server/tests/output/session-hop-error.bin +++ /dev/null @@ -1 +0,0 @@ -bV0eEventkpart_numberk913-0000019mserial_numberkBRM42220030eErrorjSessionHop \ No newline at end of file diff --git a/server/tests/output/session-start-request.bin b/server/tests/output/session-start-request.bin index 36e0f284..17a42a11 100644 Binary files a/server/tests/output/session-start-request.bin and b/server/tests/output/session-start-request.bin differ diff --git a/server/tests/output/universe-flip-flop-error.bin b/server/tests/output/universe-flip-flop-error.bin deleted file mode 100644 index 2b752144..00000000 --- a/server/tests/output/universe-flip-flop-error.bin +++ /dev/null @@ -1 +0,0 @@ -bV0eEventkpart_numberk913-0000019mserial_numberkBRM42220030eErrorpUniverseFlipFlop \ No newline at end of file diff --git a/sush.json b/sush.json index 2e2af6e5..1736962b 100644 --- a/sush.json +++ b/sush.json @@ -558,11 +558,11 @@ } } }, - "/session-nonce": { + "/server-nonce": { "get": { - "summary": "Get the sush server nonce needed to start a new session.", - "description": "The nonce will need to be sent to the signer server, which will give back its own nonce. Both nonces combined (along with the baseboard ID) will form the session ID.", - "operationId": "session_start_nonce", + "summary": "Get a fresh server nonce for the session ceremony.", + "description": "The client mixes its own nonce with this one to form the sush nonce it submits to the signer. Fetching replaces the previous server nonce, so only the latest ceremony can complete.", + "operationId": "server_nonce", "parameters": [ { "in": "header", @@ -579,7 +579,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SessionStartNonce" + "$ref": "#/components/schemas/SessionServerNonce" } } } @@ -1558,40 +1558,45 @@ "session_id" ] }, + "SessionClientNonce": { + "description": "The session nonce component generated by the sush client (on the operator's machine).", + "type": "string" + }, "SessionId": { - "description": "A globally unique identifier for a session.", + "description": "A globally unique identifier for a session. Constructed only from a `SessionPreimage`.", + "type": "string" + }, + "SessionSeq": { + "description": "The session's place in the signer-assigned order of sessions. Taken as whole milliseconds since the Unix epoch according to the *signer's* wall clock; the server's clock can't be trusted, so the server never interprets a sequence number as a time, only as an order. Sessions authorized in the same millisecond share a sequence number and tie-break by ID. Encoded as four codephrase words: 2048⁴ = 2⁴⁴ values, good through the year 2527.", + "type": "string" + }, + "SessionServerNonce": { + "description": "The session nonce component generated by the sush server (on the rack).", "type": "string" }, "SessionSignerNonce": { - "description": "The component of the hashed Session ID generated by the signer server.", + "description": "The component of the hashed Session ID generated by the signing server.", "type": "string" }, "SessionStartBody": { "type": "object", "properties": { + "client_nonce": { + "$ref": "#/components/schemas/SessionClientNonce" + }, + "seq": { + "$ref": "#/components/schemas/SessionSeq" + }, "signer_nonce": { "$ref": "#/components/schemas/SessionSignerNonce" } }, "required": [ + "client_nonce", + "seq", "signer_nonce" ] }, - "SessionStartNonce": { - "type": "object", - "properties": { - "nonce": { - "$ref": "#/components/schemas/SessionSushNonce" - } - }, - "required": [ - "nonce" - ] - }, - "SessionSushNonce": { - "description": "The component of the hashed Session ID generated by the Sush server.", - "type": "string" - }, "Signed_for_JobStartRequest": { "description": "A signed envelope around some data.", "type": "object", @@ -1614,27 +1619,10 @@ }, "SkipReason": { "description": "Why a sled will never run a job.", - "oneOf": [ - { - "type": "string", - "enum": [ - "session-ended" - ] - }, - { - "description": "The job's session sits at or below the sled's execution floor, where the sled cannot tell replay from re-run.", - "type": "string", - "enum": [ - "below-floor" - ] - }, - { - "description": "The job's chain position precedes the sled's recorded commitment: a previous life already handled it.", - "type": "string", - "enum": [ - "already-handled" - ] - } + "type": "string", + "enum": [ + "session-superseded", + "session-ended" ] }, "SledHealth": { diff --git a/tests/src/integration_tests.rs b/tests/src/integration_tests.rs index 1929ba8a..6f404f8c 100644 --- a/tests/src/integration_tests.rs +++ b/tests/src/integration_tests.rs @@ -38,18 +38,17 @@ use sush_client::{AuthzSigner, Client, Error as ClientError}; use sush_common::hash::hash; use sush_common::interactive::{InteractiveJobControl, InteractiveJobMessage}; use sush_common::jobs::{ - Access, JobLimits, JobOutputState, JobOutputStream, JobStatus, Session, SessionId, - SessionSignerNonce, job_status_try_from_json_map, + Access, JobLimits, JobOutputState, JobOutputStream, JobStatus, Session, + job_status_try_from_json_map, }; use sush_common::targets::Cubbies; use sush_server::proxy::{Targets, platform_tls}; use sush_server::{ApiServer, ProxyServer}; use crate::test_utils::{ - SignJobRequest as _, authz, ephemeral_test_root, manager_and_test_root, test_baseboard_id, - test_logger, test_pki, vouched_proxy_pems, + SignJobRequest as _, authz, ephemeral_test_root, http_session_ceremony, manager_and_test_root, + test_baseboard_id, test_logger, test_pki, vouched_proxy_pems, }; -use sush_client::types::SessionStartBody; const TIMEOUT: Duration = Duration::from_secs(10); @@ -93,23 +92,12 @@ async fn client_server() { assert_eq!(iam, identity, "who am I?"); // Start a session and run a job. - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - &test_baseboard_id(), - client - .session_start_nonce() - .send() - .await - .unwrap() - .into_inner() - .nonce, - signer_nonce, - ); + let (session_id, body) = http_session_ceremony(&client, 1).await; let session = Session::new(session_id); client .session_start() .session_id(session.session_id()) - .body(SessionStartBody { signer_nonce }) + .body(body) .send() .await .expect("can't start session"); @@ -210,23 +198,12 @@ async fn client_proxy_server() { // Attach to an interactive job through the proxy, routed by the // target path segment, and echo bytes over the bridged upgrade. - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - &test_baseboard_id(), - client - .session_start_nonce() - .send() - .await - .unwrap() - .into_inner() - .nonce, - signer_nonce, - ); + let (session_id, body) = http_session_ceremony(&client, 1).await; let session = Session::new(session_id); client .session_start() .session_id(session.session_id()) - .body(SessionStartBody { signer_nonce }) + .body(body) .send() .await .expect("can't start session"); @@ -389,23 +366,12 @@ async fn interactive_job() { assert_eq!(iam, identity, "who am I?"); // Start a session and run an interactive job. - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - &test_baseboard_id(), - client - .session_start_nonce() - .send() - .await - .unwrap() - .into_inner() - .nonce, - signer_nonce, - ); + let (session_id, body) = http_session_ceremony(&client, 1).await; let session = Session::new(session_id); client .session_start() .session_id(session.session_id()) - .body(SessionStartBody { signer_nonce }) + .body(body) .send() .await .expect("can't start session"); @@ -693,23 +659,12 @@ async fn streaming_job() { signer.set(Some(credentials)); // Start a streaming job too big for its pipe, so it blocks until we attach. - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - &test_baseboard_id(), - client - .session_start_nonce() - .send() - .await - .unwrap() - .into_inner() - .nonce, - signer_nonce, - ); + let (session_id, body) = http_session_ceremony(&client, 1).await; let session = Session::new(session_id); client .session_start() .session_id(session.session_id()) - .body(SessionStartBody { signer_nonce }) + .body(body) .send() .await .expect("can't start session"); @@ -848,23 +803,12 @@ async fn streaming_job_linger() { signer.set(Some(credentials)); // Run a streaming job small enough to finish before anyone attaches. - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - &test_baseboard_id(), - client - .session_start_nonce() - .send() - .await - .unwrap() - .into_inner() - .nonce, - signer_nonce, - ); + let (session_id, body) = http_session_ceremony(&client, 1).await; let session = Session::new(session_id); client .session_start() .session_id(session.session_id()) - .body(SessionStartBody { signer_nonce }) + .body(body) .send() .await .expect("can't start session"); diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index c0c5dcf6..a8243aad 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -27,7 +27,8 @@ use sush_client::context::Authz; use sush_common::authn::{Challenge, ChallengeResponse, Credentials, Identity, Nonce, RequestKey}; use sush_common::jobs::{ Access, JobId, JobLimits, JobMode, JobOutputState, JobOutputStream::*, JobStartRequest, - JobStatus, ProcessError, Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, + JobStatus, ProcessError, Session, SessionId, SessionPreimage, SessionSeq, SessionSignerNonce, + SignedJob, SkipReason, }; use sush_common::keys::{EphemeralKey, KeyError, KeyType, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; @@ -39,9 +40,9 @@ use sush_server::output::{JobOutputDir, OutputDirs}; use sush_server::{JobError, JobManager}; use crate::test_utils::{ - IntoBytes as _, SignJobRequest as _, ephemeral_test_root, ephemeral_test_subject, - fake_identity, manager_and_test_root, manager_test_root_and_peer, no_cubbies, null_gossip, - test_baseboard_id, test_logger, + IntoBytes as _, SessionCeremony, SignJobRequest as _, ephemeral_test_root, + ephemeral_test_subject, fake_identity, manager_and_test_root, manager_test_root_and_peer, + no_cubbies, null_gossip, session_ceremony, test_baseboard_id, test_logger, }; use sush_server::executor::PathIsolation; @@ -101,11 +102,15 @@ async fn jobs() { let (mgr, mut root, dir, _shutdown) = manager_and_test_root(log).await; let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -286,10 +291,14 @@ async fn job_session_enforced() { )); // With some other session active, likewise. - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); - mgr.session_start(&authn, session_id, signer_nonce, true) + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); assert!(matches!( @@ -299,33 +308,80 @@ async fn job_session_enforced() { )); } -/// A session nonce is consumed when a session activates, but not by a -/// failed start. +/// Only the next fetch consumes the server nonce: a failed start +/// leaves it alone, and so does activation. #[named] #[tokio::test] -async fn session_nonce_rotates() { +async fn fetch_rotates_server_nonce() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let nonce = mgr.session_sush_nonce(); + let ceremony = session_ceremony(&mgr, 1); let bogus = SessionId::random(); - let signer_nonce = SessionSignerNonce::random(); assert!(matches!( - mgr.session_start(&authn, bogus, signer_nonce, true).await, + mgr.session_start( + &authn, + bogus, + ceremony.client_nonce, + ceremony.signer_nonce, + ceremony.seq, + true, + ) + .await, Err(JobError::InvalidSessionId) )); - let session_id = SessionId::compute(mgr.own_baseboard(), nonce, signer_nonce); - mgr.session_start(&authn, session_id, signer_nonce, true) - .await - .unwrap(); + // The failed start left the server nonce alone, so the ceremony + // still verifies. + mgr.session_start( + &authn, + ceremony.session_id, + ceremony.client_nonce, + ceremony.signer_nonce, + ceremony.seq, + true, + ) + .await + .unwrap(); + + // Activation leaves it alone too: a later grant over the same + // sush nonce verifies and supersedes. + let second = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce: ceremony.sush_nonce, + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_millis(2).unwrap(), + }; + mgr.session_start( + &authn, + second.session_id(), + ceremony.client_nonce, + second.signer_nonce, + second.seq, + true, + ) + .await + .unwrap(); - let stale_signer_nonce = SessionSignerNonce::random(); - let stale = SessionId::compute(mgr.own_baseboard(), nonce, stale_signer_nonce); + // A fetch rotates: everything built over the old nonce is stale. + mgr.fresh_server_nonce(); + let stale = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce: ceremony.sush_nonce, + signer_nonce: SessionSignerNonce::random(), + seq: ceremony.seq, + }; assert!(matches!( - mgr.session_start(&authn, stale, stale_signer_nonce, true) - .await, + mgr.session_start( + &authn, + stale.session_id(), + ceremony.client_nonce, + stale.signer_nonce, + stale.seq, + true, + ) + .await, Err(JobError::InvalidSessionId) )); } @@ -337,11 +393,15 @@ async fn job_stop() { let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -454,11 +514,15 @@ async fn cancel_queued_job() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -542,11 +606,15 @@ async fn session_stop_skips_queued_jobs() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -596,25 +664,37 @@ async fn superseding_session_skips_queued_jobs() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; - let new_signer_nonce = SessionSignerNonce::random(); - let new_session_id = SessionId::compute( - mgr.own_baseboard(), - mgr.session_sush_nonce(), + let SessionCeremony { + session_id: new_session_id, + client_nonce: new_client_nonce, + signer_nonce: new_signer_nonce, + seq: new_seq, + .. + } = session_ceremony(&mgr, 2); + mgr.session_start( + &authn, + new_session_id, + new_client_nonce, new_signer_nonce, - ); - mgr.session_start(&authn, new_session_id, new_signer_nonce, true) - .await - .expect("should be able to start a superseding session"); + new_seq, + true, + ) + .await + .expect("should be able to start a superseding session"); wait_for_session_ended_skip(&mgr, &authn, &job_id).await; } @@ -626,11 +706,15 @@ async fn job_output_perms() { let dir_perms = metadata(&dir).await.unwrap().permissions(); let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -694,11 +778,15 @@ async fn cubby_targets() { .await .unwrap(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let target: Target = "14".parse().unwrap(); @@ -803,11 +891,15 @@ async fn root_certs_from_files() { // A job signed by the configured root runs. let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -886,11 +978,15 @@ async fn job_output_dir_moves() { .unwrap(); let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -988,12 +1084,21 @@ async fn universe_swap() { .unwrap(); let authn = fake_identity(&mut root).await; - async fn run_job(mgr: &JobManager, root: &mut EphemeralKey, authn: &Identity) -> JobId { - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + async fn run_job( + mgr: &JobManager, + root: &mut EphemeralKey, + authn: &Identity, + seq: u64, + ) -> JobId { + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(mgr, seq); let session = Session::new(session_id); - mgr.session_start(authn, session_id, signer_nonce, true) + mgr.session_start(authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -1015,7 +1120,7 @@ async fn universe_swap() { // Run a job to completion in the first universe, and leave a // second one running at the swap. - let job_id = run_job(&mgr, &mut root, &authn).await; + let job_id = run_job(&mgr, &mut root, &authn, 1).await; assert!(mgr.job_status(&authn, &job_id).await.is_ok()); let session = mgr.session(&authn).expect("live session"); let lingering = session.next_job_id(); @@ -1047,7 +1152,7 @@ async fn universe_swap() { // The manager serves the new universe. Running a fresh job also // orders this check after the swap has fully settled. - run_job(&mgr, &mut root, &authn).await; + run_job(&mgr, &mut root, &authn, 2).await; // Nothing about the old universe's jobs crossed over: not the // completed job's history, and not the killed job's ending. @@ -1065,11 +1170,15 @@ async fn shutdown() { let log = test_logger(function_name!()); let (mut mgr, mut root, _dir, shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1138,11 +1247,15 @@ async fn attribution() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); assert_eq!( @@ -1229,11 +1342,15 @@ async fn job_targets() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1362,11 +1479,15 @@ async fn attach_grants() { EphemeralKey::new_root(KeyType::P256, ephemeral_test_subject(), validity).unwrap(); let guest = fake_identity(&mut guest_key).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&owner, session_id, signer_nonce, true) + mgr.session_start(&owner, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -1488,11 +1609,15 @@ async fn skip_and_stop_are_starter_only() { EphemeralKey::new_root(KeyType::P256, ephemeral_test_subject(), validity).unwrap(); let guest = fake_identity(&mut guest_key).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let job_id = Session::new(session_id).next_job_id(); - mgr.session_start(&owner, session_id, signer_nonce, true) + mgr.session_start(&owner, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1595,12 +1720,16 @@ async fn skip_racing_start_converges() { let (mgr, mut root, peer, _dir, _shutdown) = manager_test_root_and_peer(log).await; let owner = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut mirror = Session::new(session_id); let job_id = mirror.next_job_id(); - mgr.session_start(&owner, session_id, signer_nonce, true) + mgr.session_start(&owner, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1654,11 +1783,15 @@ async fn too_much_cpu() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -1708,11 +1841,15 @@ async fn too_much_output() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -1806,11 +1943,15 @@ async fn output_ranges() { let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); let job_id = session.next_job_id(); @@ -2106,11 +2247,15 @@ async fn job_signal_dispositions() { let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let baseboard_id = mgr.own_baseboard(); let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -2151,11 +2296,15 @@ async fn interactive_target_required() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -2177,11 +2326,15 @@ async fn job_json() { let log = test_logger(function_name!()); let (mgr, mut root, dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); @@ -2216,11 +2369,15 @@ async fn streaming_validation() { let log = test_logger(function_name!()); let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; let authn = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = - SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let SessionCeremony { + session_id, + client_nonce, + signer_nonce, + seq, + .. + } = session_ceremony(&mgr, 1); let mut session = Session::new(session_id); - mgr.session_start(&authn, session_id, signer_nonce, true) + mgr.session_start(&authn, session_id, client_nonce, signer_nonce, seq, true) .await .unwrap(); diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 8733f17c..1979a657 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -32,7 +32,10 @@ use sush_client::context::Authz; use sush_client::{Client, ResponseValue}; use sush_common::authn::{Challenge, ChallengeResponse, Credentials, Identity, Nonce, RequestKey}; use sush_common::codephrases::Codephrase; -use sush_common::jobs::{JobId, JobMode, JobStartRequest, SessionId, VerifiedJob}; +use sush_common::jobs::{ + JobId, JobMode, JobStartRequest, SessionClientNonce, SessionId, SessionPreimage, SessionSeq, + SessionSignerNonce, SessionSushNonce, VerifiedJob, +}; use sush_common::keys::{EphemeralKey, KeyType, Signer, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; use sush_server::executor::PathIsolation; @@ -239,6 +242,63 @@ pub async fn manager_and_test_root( (mgr, root, dir, shutdown) } +/// Everything the rack hears at session start: a client nonce, a +/// signer nonce, the signer's clock reading, and the session id over +/// them all. +pub struct SessionCeremony { + pub session_id: SessionId, + pub client_nonce: SessionClientNonce, + pub sush_nonce: SessionSushNonce, + pub signer_nonce: SessionSignerNonce, + pub seq: SessionSeq, +} + +/// Build a [`SessionCeremony`] against `mgr`. Fetches a fresh server +/// nonce, as `GET /session-nonce` would, so earlier ceremonies +/// against `mgr` go stale. +pub fn session_ceremony(mgr: &JobManager, seq: u64) -> SessionCeremony { + let client_nonce = SessionClientNonce::random(); + let sush_nonce = SessionSushNonce::compute(client_nonce, mgr.fresh_server_nonce()); + let preimage = SessionPreimage { + baseboard: mgr.own_baseboard().clone(), + sush_nonce, + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_millis(seq).unwrap(), + }; + SessionCeremony { + session_id: preimage.session_id(), + client_nonce, + sush_nonce, + signer_nonce: preimage.signer_nonce, + seq: preimage.seq, + } +} + +/// Build a session id and the start body carrying its parts: like +/// [`session_ceremony`], but the server nonce comes from `client`'s +/// rack over HTTP. +pub async fn http_session_ceremony( + client: &Client, + seq: u64, +) -> (SessionId, sush_client::types::SessionStartBody) { + let client_nonce = SessionClientNonce::random(); + let server_nonce = client.server_nonce().send().await.unwrap().into_inner(); + let preimage = SessionPreimage { + baseboard: test_baseboard_id(), + sush_nonce: SessionSushNonce::compute(client_nonce, server_nonce), + signer_nonce: SessionSignerNonce::random(), + seq: SessionSeq::from_unix_millis(seq).unwrap(), + }; + ( + preimage.session_id(), + sush_client::types::SessionStartBody { + client_nonce, + signer_nonce: preimage.signer_nonce, + seq: preimage.seq, + }, + ) +} + /// Like [`manager_and_test_root`], but also hands back a clone of the /// manager's gossip network, for tests that play a peer. pub async fn manager_test_root_and_peer(