From 82134823a07b4f45096a287f790c169511859617 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 26 Aug 2026 15:27:18 +0200 Subject: [PATCH 01/91] MAINTAINERS: add Alice Ryhl as co-maintainer for Rust [ALLOC] Add Alice as co-maintainer for the Rust alloc module to share maintenance responsibility. This is also a natural fit, as some of the alloc code overlaps with the mm subsystem. Acked-by: Gary Guo Acked-by: Alice Ryhl Acked-by: Liam R. Howlett (Oracle) Acked-by: Vlastimil Babka (SUSE) Acked-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20260826132729.3281308-1-dakr@kernel.org Signed-off-by: Danilo Krummrich --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 3a19da74d00c..1945b1001849 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23973,6 +23973,7 @@ K: \b(?i:rust)\b RUST [ALLOC] M: Danilo Krummrich +M: Alice Ryhl R: Lorenzo Stoakes R: Vlastimil Babka R: Liam R. Howlett From c2bc9bd605add0b9f1f3b7e93b649bd814de226a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 4 Sep 2026 15:12:29 +0100 Subject: [PATCH 02/91] rust: pin-init: internal: extract utility code to new module Create a new `util.rs` to host utility code that are generic and can be shared by multiple macros. Link: https://patch.msgid.link/20260904-tuple-struct-v1-1-72c50bd037fd@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/lib.rs | 1 + rust/pin-init/internal/src/pin_data.rs | 17 +++++----------- rust/pin-init/internal/src/util.rs | 27 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 rust/pin-init/internal/src/util.rs diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index 60d5093f3128..4d8ff86484b6 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -18,6 +18,7 @@ mod init; mod pin_data; mod pinned_drop; +mod util; mod zeroable; #[proc_macro_attribute] diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index ff194d27565e..074bc6b3091a 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -10,7 +10,10 @@ Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, }; -use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; +use crate::{ + diagnostics::{DiagCtxt, ErrorGuaranteed}, + util::*, +}; pub(crate) mod kw { syn::custom_keyword!(PinnedDrop); @@ -81,21 +84,11 @@ pub(crate) fn pin_data( // // We need to perform this after parsing so we can reliably detect field cfgs. for (field_idx, field) in struct_.fields.iter_mut().enumerate() { - let cfg: Vec<_> = field - .attrs - .iter() - .filter(|a| a.path().is_ident("cfg")) - .map(|a| { - a.parse_args::() - .expect("parse as token stream cannot fail") - }) - .collect(); - + let cfg = field.attrs.extract_cfg_attrs(); if cfg.is_empty() { continue; } - field.attrs.retain(|a| !a.path().is_ident("cfg")); let cfg_true_struct = quote!(#struct_); let punctuated = match &mut struct_.fields { diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs new file mode 100644 index 000000000000..ed18ab7d45e6 --- /dev/null +++ b/rust/pin-init/internal/src/util.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use syn::Attribute; + +pub(crate) trait AttrListExt { + fn extract_cfg_attrs(&mut self) -> Vec; +} + +impl AttrListExt for Vec { + fn extract_cfg_attrs(&mut self) -> Vec { + let cfg: Vec<_> = self + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if !cfg.is_empty() { + self.retain(|a| !a.path().is_ident("cfg")); + } + + cfg + } +} From 1ee699c4516607c0b594813c9c81da4285ed7d56 Mon Sep 17 00:00:00 2001 From: Mohamad Alsadhan Date: Fri, 4 Sep 2026 15:12:30 +0100 Subject: [PATCH 03/91] rust: pin-init: internal: pin_data: support tuple struct projections `#[pin_data]` rejects tuple structs because it assumes every field has a name, which it uses for the projection field, the `__Unpin` field and the pin-data accessor. Identify fields by `syn::Member` instead, so that tuple fields are referred to by their index in generated field accesses. The names that generated items still need are derived from the index as `_0`, `_1`, etc. The projection of a tuple struct is a tuple struct itself, so projected fields are accessed with the same `.0`, `.1` syntax as on the input type rather than through synthesised names. Signed-off-by: Mohamad Alsadhan [ Moved utility code to util.rs as extension trait - Gary ] Link: https://patch.msgid.link/20260904-tuple-struct-v1-2-72c50bd037fd@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 112 +++++++++++++++++-------- rust/pin-init/internal/src/util.rs | 31 ++++++- rust/pin-init/src/lib.rs | 23 +++++ 3 files changed, 130 insertions(+), 36 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 074bc6b3091a..8cd9bf139567 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -7,7 +7,8 @@ parse_quote, parse_quote_spanned, spanned::Spanned, visit_mut::VisitMut, - Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, + Field, Fields, Generics, Ident, Index, Item, Member, PathSegment, Type, TypePath, Visibility, + WhereClause, }; use crate::{ @@ -49,6 +50,7 @@ fn to_tokens(&self, tokens: &mut TokenStream) { struct FieldInfo<'a> { field: &'a Field, + member: Member, pinned: bool, } @@ -129,10 +131,12 @@ pub(crate) fn pin_data( replacer.visit_generics_mut(&mut struct_.generics); replacer.visit_fields_mut(&mut struct_.fields); + let is_tuple_struct = matches!(struct_.fields, Fields::Unnamed(_)); let fields: Vec> = struct_ .fields .iter_mut() - .map(|field| { + .enumerate() + .map(|(index, field)| { let len = field.attrs.len(); field.attrs.retain(|a| !a.path().is_ident("pin")); let pinned_count = len - field.attrs.len(); @@ -144,23 +148,30 @@ pub(crate) fn pin_data( !field.attrs.iter().any(|a| a.path().is_ident("cfg")), "cfgs should be all resolved at this point" ); + let member = match &field.ident { + Some(ident) => Member::Named(ident.clone()), + None => Member::Unnamed(Index { + index: index as u32, + span: field.span(), + }), + }; FieldInfo { field: &*field, + member, pinned: pinned_count != 0, } }) .collect(); for field in &fields { - let ident = field.field.ident.as_ref().unwrap(); - if !field.pinned && is_phantom_pinned(&field.field.ty) { dcx.warn( field.field, format!( - "The field `{ident}` of type `PhantomPinned` only has an effect \ + "The field {} of type `PhantomPinned` only has an effect \ if it has the `#[pin]` attribute", + field.member.display_name(), ), ); } @@ -168,8 +179,13 @@ pub(crate) fn pin_data( let unpin_impl = generate_unpin_impl(&struct_.ident, &struct_.generics, &fields); let drop_impl = generate_drop_impl(&struct_.ident, &struct_.generics, args); - let projections = - generate_projections(&struct_.vis, &struct_.ident, &struct_.generics, &fields); + let projections = generate_projections( + &struct_.vis, + &struct_.ident, + &struct_.generics, + is_tuple_struct, + &fields, + ); let the_pin_data = generate_the_pin_data(&struct_.vis, &struct_.ident, &struct_.generics, &fields); @@ -231,7 +247,7 @@ fn generate_unpin_impl( unreachable!() }; let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { - let ident = f.field.ident.as_ref().unwrap(); + let ident = f.member.as_ident(); let ty = &f.field.ty; quote!( #ident: #ty @@ -313,6 +329,7 @@ fn generate_projections( vis: &Visibility, ident: &Ident, generics: &Generics, + is_tuple_struct: bool, fields: &[FieldInfo<'_>], ) -> TokenStream { let (impl_generics, ty_generics, _) = generics.split_for_impl(); @@ -325,28 +342,32 @@ fn generate_projections( let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields .iter() .map(|field| { - let Field { vis, ident, ty, .. } = &field.field; + let Field { vis, ty, .. } = &field.field; + let member = &field.member; + // The projection of a tuple struct is a tuple struct itself, so its fields are + // positional and must not be named. + let name = (!is_tuple_struct).then(|| { + let ident = field.member.as_ident(); + quote!(#ident:) + }); - let ident = ident - .as_ref() - .expect("only structs with named fields are supported"); if field.pinned { ( quote!( - #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, + #vis #name ::core::pin::Pin<&'__pin mut #ty>, ), quote!( // SAFETY: this field is structurally pinned. - #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, + #name unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#member) }, ), ) } else { ( quote!( - #vis #ident: &'__pin mut #ty, + #vis #name &'__pin mut #ty, ), quote!( - #ident: &mut #this.#ident, + #name &mut #this.#member, ), ) } @@ -355,24 +376,52 @@ fn generate_projections( let structurally_pinned_fields_docs = fields .iter() .filter(|f| f.pinned) - .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); + .map(|f| format!(" - {}", f.member.display_name())); let not_structurally_pinned_fields_docs = fields .iter() .filter(|f| !f.pinned) - .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); + .map(|f| format!(" - {}", f.member.display_name())); let docs = format!(" Pin-projections of [`{ident}`]"); + let (projection_def, projection_init) = if is_tuple_struct { + ( + quote! { + #vis struct #projection #generics_with_pin_lt ( + #(#fields_decl)* + ::core::marker::PhantomData<&'__pin mut ()>, + ) #whr; + }, + quote! { + #projection( + #(#fields_proj)* + ::core::marker::PhantomData, + ) + }, + ) + } else { + ( + quote! { + #vis struct #projection #generics_with_pin_lt + #whr + { + #(#fields_decl)* + ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, + } + }, + quote! { + #projection { + #(#fields_proj)* + ___pin_phantom_data: ::core::marker::PhantomData, + } + }, + ) + }; quote! { #[doc = #docs] // Allow `non_snake_case` since the same warning will be emitted on // the struct definition. #[allow(dead_code, non_snake_case)] #[doc(hidden)] - #vis struct #projection #generics_with_pin_lt - #whr - { - #(#fields_decl)* - ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, - } + #projection_def impl #impl_generics #ident #ty_generics #whr @@ -390,10 +439,7 @@ impl #impl_generics #ident #ty_generics ) -> #projection #ty_generics_with_pin_lt { // SAFETY: we only give access to `&mut` for fields not structurally pinned. let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; - #projection { - #(#fields_proj)* - ___pin_phantom_data: ::core::marker::PhantomData, - } + #projection_init } } } @@ -414,11 +460,9 @@ fn generate_the_pin_data( let field_accessors = fields .iter() .map(|f| { - let Field { vis, ident, ty, .. } = f.field; - - let field_name = ident - .as_ref() - .expect("only structs with named fields are supported"); + let Field { vis, ty, .. } = f.field; + let field_name = f.member.as_ident(); + let member = &f.member; let pin_marker = if f.pinned { quote!(Pinned) } else { @@ -443,7 +487,7 @@ fn generate_the_pin_data( // - If `#pin_marker` is `Pinned`, the corresponding field is structurally // pinned. // - Other safety requirements follows the safety requirement. - unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) } + unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#member) } } } }) diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index ed18ab7d45e6..ed2c78f0658f 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -use proc_macro2::TokenStream; -use syn::Attribute; +use proc_macro2::{Ident, TokenStream}; +use quote::format_ident; +use syn::{Attribute, Index, Member}; pub(crate) trait AttrListExt { fn extract_cfg_attrs(&mut self) -> Vec; @@ -25,3 +26,29 @@ fn extract_cfg_attrs(&mut self) -> Vec { cfg } } + +pub(crate) trait MemberExt { + /// Returns an identifier for the member. + /// + /// Tuple fields have no name of their own, so they are named `_0`, `_1`, ... instead. + fn as_ident(&self) -> Ident; + + /// Obtain a display name for the member in diagnostics. + fn display_name(&self) -> String; +} + +impl MemberExt for Member { + fn as_ident(&self) -> Ident { + match self { + Member::Named(ident) => ident.clone(), + Member::Unnamed(Index { index, .. }) => format_ident!("_{index}"), + } + } + + fn display_name(&self) -> String { + match self { + Member::Named(ident) => format!("`{ident}`"), + Member::Unnamed(Index { index, .. }) => format!("index `{index}`"), + } + } +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index f1463be9479d..6bf42c9e9097 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -304,6 +304,9 @@ /// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, /// then `#[pin]` directs the type of initializer that is required. /// +/// Tuple structs are supported as well. Their fields have no names, so the generated projection +/// is a tuple struct too and its fields are accessed by index. +/// /// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this /// macro, and change your `Drop` implementation to `PinnedDrop` annotated with /// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. @@ -327,6 +330,26 @@ /// } /// ``` /// +/// The same as a tuple struct, projected by index: +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// use core::pin::Pin; +/// use pin_init::pin_data; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data] +/// struct DriverData(#[pin] CMutex>, Box<[u8; 1024 * 1024]>); +/// +/// fn queue(data: Pin<&mut DriverData>) -> Pin<&mut CMutex>> { +/// data.project().0 +/// } +/// ``` +/// /// ``` /// # #![feature(allocator_api)] /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; From d48aa955cb91745ef2e5089f9277a7a95060cce3 Mon Sep 17 00:00:00 2001 From: Mohamad Alsadhan Date: Fri, 4 Sep 2026 15:12:31 +0100 Subject: [PATCH 04/91] rust: pin-init: internal: init: support tuple structs in `[pin_]init!` Extend the initializer syntax so that a field can be named by an index, addressing tuple struct fields the same way a struct expression does: pin_init!(Foo { 0: value, 1 <- initializer }) Tuple fields are not exposed by a `let` binding to the fields after them, since they have no name to bind; `_0` would shadow a user variable. Signed-off-by: Mohamad Alsadhan [ Fixed incorrect index calculation and cleaned up the code - Gary ] Link: https://patch.msgid.link/20260904-tuple-struct-v1-3-72c50bd037fd@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 115 +++++++++++++++++------------ rust/pin-init/src/lib.rs | 32 +++++++- 2 files changed, 95 insertions(+), 52 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index fd0b5ea4a0a3..5920bb281a50 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -8,10 +8,13 @@ parse_quote, punctuated::Punctuated, spanned::Spanned, - token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Path, Token, Type, + token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, LitInt, Member, Path, Token, Type, }; -use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; +use crate::{ + diagnostics::{DiagCtxt, ErrorGuaranteed}, + util::*, +}; pub(crate) struct Initializer { attrs: Vec, @@ -36,11 +39,11 @@ struct InitializerField { enum InitializerKind { Value { - ident: Ident, + member: Member, value: Option<(Token![:], Expr)>, }, Init { - ident: Ident, + member: Member, _left_arrow_token: Token![<-], value: Expr, }, @@ -52,9 +55,9 @@ enum InitializerKind { } impl InitializerKind { - fn ident(&self) -> Option<&Ident> { + fn member(&self) -> Option<&Member> { match self { - Self::Value { ident, .. } | Self::Init { ident, .. } => Some(ident), + Self::Value { member, .. } | Self::Init { member, .. } => Some(member), Self::Code { .. } => None, } } @@ -229,9 +232,9 @@ fn init_fields( cfgs }; - let ident = match kind { - InitializerKind::Value { ident, .. } => ident, - InitializerKind::Init { ident, .. } => ident, + let member = match kind { + InitializerKind::Value { member, .. } => member, + InitializerKind::Init { member, .. } => member, InitializerKind::Code { block, .. } => { let stmt = &block.stmts; res.extend(quote! { @@ -243,27 +246,28 @@ fn init_fields( continue; } }; + let ident = member.as_ident(); let slot = if pinned { quote! { // SAFETY: // - `slot` is valid and properly aligned. - // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. - // - `make_field_check` prevents `#ident` from being used twice, therefore - // `(*slot).#ident` is exclusively accessed and has not been initialized. + // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned. + // - `make_field_check` prevents `#member` from being used twice, therefore + // `(*slot).#member` is exclusively accessed and has not been initialized. (unsafe { #data.#ident(#slot) }) } } else { quote! { // For `init!()` macro, everything is unpinned. // SAFETY: - // - `&raw mut (*slot).#ident` is valid. - // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. - // - `make_field_check` prevents `#ident` from being used twice, therefore - // `(*slot).#ident` is exclusively accessed and has not been initialized. + // - `&raw mut (*slot).#member` is valid. + // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned. + // - `make_field_check` prevents `#member` from being used twice, therefore + // `(*slot).#member` is exclusively accessed and has not been initialized. (unsafe { ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new( - &raw mut (*#slot).#ident + &raw mut (*#slot).#member ) }) } @@ -273,11 +277,11 @@ fn init_fields( let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); let init = match kind { - InitializerKind::Value { ident, value } => { + InitializerKind::Value { value, .. } => { let value = value .as_ref() .map(|(_, value)| quote!(#value)) - .unwrap_or_else(|| quote!(#ident)); + .unwrap_or_else(|| quote!(#member)); quote! { #(#attrs)* @@ -294,14 +298,23 @@ fn init_fields( InitializerKind::Code { .. } => unreachable!(), }; + // A tuple field has no name that could be bound here (the `_0` identifiers are considered + // implementation detail and not user-facing). + let binding = match member { + Member::Named(ident) => quote! { + #(#cfgs)* + // Allow `non_snake_case` since the same warning is going to be reported for the + // struct field. + #[allow(unused_variables, non_snake_case)] + let #ident = #guard.let_binding(); + }, + Member::Unnamed(_) => quote!(), + }; + res.extend(quote! { #init - #(#cfgs)* - // Allow `non_snake_case` since the same warning is going to be reported for the struct - // field. - #[allow(unused_variables, non_snake_case)] - let #ident = #guard.let_binding(); + #binding }); guards.push(guard); @@ -326,9 +339,9 @@ fn make_field_check( ) -> TokenStream { let field_attrs: Vec<_> = fields .iter() - .filter_map(|f| f.kind.ident().map(|_| &f.attrs)) + .filter_map(|f| f.kind.member().map(|_| &f.attrs)) .collect(); - let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect(); + let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.member()).collect(); let zeroing_trailer = match init_kind { InitKind::Normal => None, InitKind::Zeroing => Some(quote! { @@ -376,7 +389,8 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { let lh = content.lookahead1(); if lh.peek(End) || lh.peek(Token![..]) { break; - } else if lh.peek(Ident) || lh.peek(Token![_]) || lh.peek(Token![#]) { + } else if lh.peek(Ident) || lh.peek(LitInt) || lh.peek(Token![_]) || lh.peek(Token![#]) + { fields.push_value(content.parse()?); let lh = content.lookahead1(); if lh.peek(End) { @@ -450,31 +464,36 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { impl Parse for InitializerKind { fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { let lh = input.lookahead1(); - if lh.peek(Token![_]) { - Ok(Self::Code { + let member = if lh.peek(Token![_]) { + return Ok(Self::Code { _underscore_token: input.parse()?, _colon_token: input.parse()?, block: input.parse()?, + }); + } else if lh.peek(Ident) || lh.peek(LitInt) { + input.parse::()? + } else { + return Err(lh.error()); + }; + + let lh = input.lookahead1(); + if lh.peek(Token![<-]) { + Ok(Self::Init { + member, + _left_arrow_token: input.parse()?, + value: input.parse()?, + }) + } else if lh.peek(Token![:]) { + Ok(Self::Value { + member, + value: Some((input.parse()?, input.parse()?)), + }) + } else if matches!(member, Member::Named(_)) && (lh.peek(Token![,]) || lh.peek(End)) { + // Short-hand syntax, available for named fields only. + Ok(Self::Value { + member, + value: None, }) - } else if lh.peek(Ident) { - let ident = input.parse()?; - let lh = input.lookahead1(); - if lh.peek(Token![<-]) { - Ok(Self::Init { - ident, - _left_arrow_token: input.parse()?, - value: input.parse()?, - }) - } else if lh.peek(Token![:]) { - Ok(Self::Value { - ident, - value: Some((input.parse()?, input.parse()?)), - }) - } else if lh.peek(Token![,]) || lh.peek(End) { - Ok(Self::Value { ident, value: None }) - } else { - Err(lh.error()) - } } else { Err(lh.error()) } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 6bf42c9e9097..fc34cfd7372d 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -592,7 +592,7 @@ macro_rules! stack_try_pin_init { }; } -/// Construct an in-place, fallible pinned initializer for `struct`s. +/// Construct an in-place, fallible pinned initializer for structs, including tuple structs. /// /// The error type defaults to [`Infallible`]; if you need a different one, write `? Error` at the /// end, after the struct initializer. @@ -626,6 +626,28 @@ macro_rules! stack_try_pin_init { /// # Box::pin_init(demo()).unwrap(); /// ``` /// +/// The fields of a tuple struct are addressed by their index: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Pair(usize, Bar); +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// # fn demo() -> impl PinInit { +/// let initializer = pin_init!(Pair { +/// 0: 42, +/// 1 <- Bar { x: 64 }, +/// }); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// /// Arbitrary Rust expressions can be used to set the value of a variable. /// /// The fields are initialized in the order that they appear in the initializer. So it is possible @@ -744,9 +766,11 @@ macro_rules! stack_try_pin_init { /// /// # Syntax /// -/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with -/// the following modifications is expected: +/// As already mentioned in the examples above, inside of `pin_init!` a struct initializer with the +/// following modifications is expected: /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. +/// - Tuple struct fields are named by their index, as in `0: value` or `0 <- initializer`. They +/// are not exposed by a `let` binding, since they have no name to bind. /// - You can use `_: { /* run any user-code here */ },` anywhere where you can place fields in /// order to run arbitrary code. /// - In front of the initializer you can write `&this in` to have access to a [`NonNull`] @@ -785,7 +809,7 @@ macro_rules! stack_try_pin_init { /// [`NonNull`]: core::ptr::NonNull pub use pin_init_internal::pin_init; -/// Construct an in-place, fallible initializer for `struct`s. +/// Construct an in-place, fallible initializer for structs, including tuple structs. /// /// This macro defaults the error to [`Infallible`]; if you need a different one, write `? Error` /// at the end, after the struct initializer. From 3b3ff9034a497fb3b8429f6fe8ba5697bc7dcebd Mon Sep 17 00:00:00 2001 From: Mohamad Alsadhan Date: Fri, 4 Sep 2026 15:12:32 +0100 Subject: [PATCH 05/91] rust: pin-init: internal: init: support tuple struct constructor syntax A tuple struct whose fields are all set to a value reads better written like a call to its constructor than with the indices spelled out: pin_init!(Foo(value, value)) Parse the two forms into separate types and rewrite the constructor arguments into the indexed fields they are shorthand for, so that only the parser has to know about the second form. The arguments have no names, so they cannot use `<-`. Parse it anyway and reject it afterwards, which reports the position of every offending `<-` rather than stopping at the first one. `cfg` needs different treatment for tuple constructor syntax. As non-derive proc macros are invoked before cfg is resolved, the macro cannot know whether a field survives, and dropping a tuple field renumbers every field after it. That cannot be expressed by attaching a `cfg` attribute to the initializer of a single field. Thus, resolve tuple field cfgs up front instead, by generating two cfg-gated invocations of the macro with one field resolved in each. This is the approach of commit 5bbf2b2deb94 ("rust: pin-init: internal: rework how `#[pin_data]` handles cfg"), and it is linear time because only one of the two branches is ever expanded. Struct expression syntax do not renumber, so using tuple structs with struct syntax can keep using the existing attribute-based handling. Suggested-by: Gary Guo Link: https://github.com/Rust-for-Linux/pin-init/pull/165 Signed-off-by: Mohamad Alsadhan [ Pre-expand cfgs for tuple init syntax. Use generics instead of separate types for normalization - Gary ] Co-developed-by: Gary Guo Link: https://patch.msgid.link/20260904-tuple-struct-v1-4-72c50bd037fd@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 381 +++++++++++++++++++++++++++-- rust/pin-init/internal/src/lib.rs | 11 +- rust/pin-init/src/lib.rs | 16 ++ 3 files changed, 383 insertions(+), 25 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 5920bb281a50..dc5dfb2ca114 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -1,14 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, ToTokens, TokenStreamExt}; use syn::{ - braced, + braced, parenthesized, parse::{End, Parse}, parse_quote, - punctuated::Punctuated, + punctuated::{Pair, Punctuated}, spanned::Spanned, - token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, LitInt, Member, Path, Token, Type, + token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Index, LitInt, Member, Path, Token, + Type, }; use crate::{ @@ -16,14 +17,87 @@ util::*, }; -pub(crate) struct Initializer { +pub(crate) struct Initializer { attrs: Vec, this: Option, + kind: Kind, + error: Option<(Token![?], Type)>, +} + +pub(crate) struct InitExprStruct { path: Path, brace_token: token::Brace, fields: Punctuated, rest: Option<(Token![..], Expr)>, - error: Option<(Token![?], Type)>, +} + +pub(crate) struct InitExprTuple { + path: Path, + paren_token: token::Paren, + fields: Punctuated, +} + +pub(crate) enum InitExprKind { + Struct(InitExprStruct), + Tuple(InitExprTuple), +} + +struct InitTupleField { + attrs: Vec, + /// `<-` is not valid in constructor syntax; it is parsed anyway so that it can be rejected + /// with a proper diagnostic instead of a parse error. + left_arrow_token: Option, + value: Expr, +} + +impl InitExprTuple { + fn normalize(self) -> InitExprStruct { + let InitExprTuple { + path, + paren_token, + fields, + } = self; + InitExprStruct { + path, + brace_token: token::Brace { + span: paren_token.span, + }, + fields: fields + .into_pairs() + .enumerate() + .map(|(index, pair)| { + let (field, comma) = pair.into_tuple(); + let span = field.value.span(); + let field = InitializerField { + attrs: field.attrs, + kind: InitializerKind::Value { + member: Member::Unnamed(Index { + index: index.try_into().unwrap(), + span, + }), + value: Some((Token![:](span), field.value)), + }, + }; + Pair::new(field, comma) + }) + .collect(), + rest: None, + } + } + + fn validate(&self, dcx: &mut DiagCtxt) -> Result<(), ErrorGuaranteed> { + let mut result = Ok(()); + for field in &self.fields { + if let Some(left_arrow_token) = &field.left_arrow_token { + result = Err(dcx.error( + left_arrow_token, + "`<-` is not supported in tuple constructor syntax; name the fields by index \ + instead, e.g. `Type { 0 <- initializer, 1: value }`", + )); + } + } + result + } } struct This { @@ -71,16 +145,103 @@ struct DefaultErrorAttribute { ty: Box, } -pub(crate) fn expand( +pub(crate) fn expand_with_cfg( + initializer: Initializer, + default_error: Option<&'static str>, + pinned: bool, + dcx: &mut DiagCtxt, +) -> Result { + let initializer = match initializer.kind { + InitExprKind::Tuple(expr) => { + expr.validate(dcx)?; + + let mut initializer = Initializer { + attrs: initializer.attrs, + this: initializer.this, + kind: expr, + error: initializer.error, + }; + + // Removing a tuple field renumbers every field after it, which cannot be expressed with + // a `cfg` attribute on the initializer of a single field. Therefore, resolve tuple + // field cfgs before continuing. Struct expression syntax uses explicit numbers, so + // there is no need to pre-expand them and we only need to emit their cfgs on generated + // code. + for (field_idx, field) in initializer.kind.fields.iter_mut().enumerate() { + let cfg = field.attrs.extract_cfg_attrs(); + + if cfg.is_empty() { + continue; + } + + let true_initializer = initializer.to_token_stream(); + initializer.kind.fields = initializer + .kind + .fields + .into_pairs() + .enumerate() + .filter(|&(index, _)| index != field_idx) + .map(|(_, pair)| pair) + .collect(); + + let false_initializer = &initializer; + + let macro_name = if pinned { + quote!(::pin_init::pin_init) + } else { + quote!(::pin_init::init) + }; + + // Resolve one field at a time until we've got no more tuple field cfgs. + // + // This is linear time because macro invocations with false cfg will not be + // expanded. + return Ok(quote! { + { + // Use `{}` delimiter here so semicolon is not required, otherwise the + // expression becomes unit type. + #[cfg(all(#(#cfg,)*))] + #macro_name! { #true_initializer } + + #[cfg(not(all(#(#cfg,)*)))] + #macro_name! { #false_initializer } + } + }); + } + + // No cfgs left, we can normalize the initializer to the struct kind. + Initializer { + attrs: initializer.attrs, + this: initializer.this, + kind: initializer.kind.normalize(), + error: initializer.error, + } + } + + InitExprKind::Struct(expr) => Initializer { + attrs: initializer.attrs, + this: initializer.this, + kind: expr, + error: initializer.error, + }, + }; + + expand(initializer, default_error, pinned, dcx) +} + +fn expand( Initializer { attrs, this, - path, - brace_token, - fields, - rest, + kind: + InitExprStruct { + path, + brace_token, + fields, + rest, + }, error, - }: Initializer, + }: Initializer, default_error: Option<&'static str>, pinned: bool, dcx: &mut DiagCtxt, @@ -99,7 +260,10 @@ pub(crate) fn expand( } else if let Some(default_error) = default_error { syn::parse_str(default_error).unwrap() } else { - dcx.error(brace_token.span.close(), "expected `? ` after `}`"); + dcx.error( + brace_token.span.close(), + "expected `? ` after initializer", + ); parse_quote!(::core::convert::Infallible) } }, @@ -377,11 +541,8 @@ fn make_field_check( } } -impl Parse for Initializer { - fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { - let attrs = input.call(Attribute::parse_outer)?; - let this = input.peek(Token![&]).then(|| input.parse()).transpose()?; - let path = input.parse()?; +impl InitExprStruct { + fn parse_with_path(path: Path, input: syn::parse::ParseStream<'_>) -> syn::Result { let content; let brace_token = braced!(content in input); let mut fields = Punctuated::new(); @@ -408,6 +569,51 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { .peek(Token![..]) .then(|| Ok::<_, syn::Error>((content.parse()?, content.parse()?))) .transpose()?; + Ok(Self { + path, + brace_token, + fields, + rest, + }) + } +} + +impl InitExprTuple { + fn parse_with_path(path: Path, input: syn::parse::ParseStream<'_>) -> syn::Result { + let content; + let paren_token = parenthesized!(content in input); + let mut fields = Punctuated::new(); + while !content.is_empty() { + fields.push_value(InitTupleField { + attrs: content.call(Attribute::parse_outer)?, + left_arrow_token: content.parse()?, + value: content.parse()?, + }); + if content.is_empty() { + break; + } + fields.push_punct(content.parse()?); + } + Ok(InitExprTuple { + path, + paren_token, + fields, + }) + } +} + +impl Parse for Initializer { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let attrs = input.call(Attribute::parse_outer)?; + let this = input.peek(Token![&]).then(|| input.parse()).transpose()?; + let path = input.parse()?; + let kind = if input.peek(token::Brace) { + InitExprKind::Struct(InitExprStruct::parse_with_path(path, input)?) + } else if input.peek(token::Paren) { + InitExprKind::Tuple(InitExprTuple::parse_with_path(path, input)?) + } else { + return Err(input.error("expected curly braces or parentheses")); + }; let error = input .peek(Token![?]) .then(|| Ok::<_, syn::Error>((input.parse()?, input.parse()?))) @@ -426,10 +632,7 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { Ok(Self { attrs, this, - path, - brace_token, - fields, - rest, + kind, error, }) } @@ -499,3 +702,137 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { } } } + +impl ToTokens for Initializer { + fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { + attrs, + this, + kind, + error, + } = self; + tokens.append_all(attrs); + this.to_tokens(tokens); + kind.to_tokens(tokens); + if let Some((question, ty)) = error { + question.to_tokens(tokens); + ty.to_tokens(tokens); + } + } +} + +impl ToTokens for InitExprKind { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Struct(init) => init.to_tokens(tokens), + Self::Tuple(init) => init.to_tokens(tokens), + } + } +} + +impl ToTokens for InitExprStruct { + fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { + path, + brace_token, + fields, + rest, + } = self; + path.to_tokens(tokens); + brace_token.surround(tokens, |tokens| { + fields.to_tokens(tokens); + if let Some((dotdot, expr)) = rest { + dotdot.to_tokens(tokens); + expr.to_tokens(tokens); + } + }); + } +} + +impl ToTokens for InitExprTuple { + fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { + path, + paren_token, + fields, + } = self; + path.to_tokens(tokens); + paren_token.surround(tokens, |tokens| fields.to_tokens(tokens)); + } +} + +impl ToTokens for InitTupleField { + fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { + attrs, + left_arrow_token, + value, + } = self; + tokens.append_all(attrs); + left_arrow_token.to_tokens(tokens); + value.to_tokens(tokens); + } +} + +impl ToTokens for InitializerAttribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::DefaultError(DefaultErrorAttribute { ty }) => { + quote!(#[default_error(#ty)]).to_tokens(tokens); + } + } + } +} + +impl ToTokens for This { + fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { + _and_token, + ident, + _in_token, + } = self; + _and_token.to_tokens(tokens); + ident.to_tokens(tokens); + _in_token.to_tokens(tokens); + } +} + +impl ToTokens for InitializerField { + fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { attrs, kind } = self; + tokens.append_all(attrs); + kind.to_tokens(tokens); + } +} + +impl ToTokens for InitializerKind { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Value { member, value } => { + member.to_tokens(tokens); + if let Some((colon, expr)) = value { + colon.to_tokens(tokens); + expr.to_tokens(tokens); + } + } + Self::Init { + member, + _left_arrow_token, + value, + } => { + member.to_tokens(tokens); + _left_arrow_token.to_tokens(tokens); + value.to_tokens(tokens); + } + Self::Code { + _underscore_token, + _colon_token, + block, + } => { + _underscore_token.to_tokens(tokens); + _colon_token.to_tokens(tokens); + block.to_tokens(tokens); + } + } + } +} diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index 4d8ff86484b6..c488019d6250 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -49,12 +49,17 @@ pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream { #[proc_macro] pub fn init(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), false, dcx)) - .into() + DiagCtxt::with(|dcx| { + init::expand_with_cfg(input, Some("::core::convert::Infallible"), false, dcx) + }) + .into() } #[proc_macro] pub fn pin_init(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), true, dcx)).into() + DiagCtxt::with(|dcx| { + init::expand_with_cfg(input, Some("::core::convert::Infallible"), true, dcx) + }) + .into() } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fc34cfd7372d..4110590348c6 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -648,6 +648,20 @@ macro_rules! stack_try_pin_init { /// # Box::pin_init(demo()).unwrap(); /// ``` /// +/// A tuple struct whose fields are all set to a value can also be written like a call to its +/// constructor: +/// +/// ```rust +/// # use pin_init::*; +/// #[pin_data] +/// struct Pair(usize, usize); +/// +/// # fn demo() -> impl PinInit { +/// let initializer = pin_init!(Pair(42, 64)); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// /// Arbitrary Rust expressions can be used to set the value of a variable. /// /// The fields are initialized in the order that they appear in the initializer. So it is possible @@ -771,6 +785,8 @@ macro_rules! stack_try_pin_init { /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. /// - Tuple struct fields are named by their index, as in `0: value` or `0 <- initializer`. They /// are not exposed by a `let` binding, since they have no name to bind. +/// - A tuple struct can also be initialized with constructor syntax, as in `Type(value, value)`. +/// Since its arguments are not named, they cannot use `<-`; write them out by index instead. /// - You can use `_: { /* run any user-code here */ },` anywhere where you can place fields in /// order to run arbitrary code. /// - In front of the initializer you can write `&this in` to have access to a [`NonNull`] From e30decc3f9a0ded621d5b0f19a5253b84d21be0c Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 16 Sep 2026 13:11:32 +0100 Subject: [PATCH 06/91] rust: pin-init: internal: init: emit `slot` using mixed site hygiene Syn/quote's default is the call site hygiene. If user provides a field named `slot`, it will conflict with pin-init generated `slot` identifier. Change it to use mixed site hygiene instead. Fixes: 4883830e9784 ("rust: pin-init: rewrite the initializer macros using `syn`") Link: https://patch.msgid.link/20260916-dev-hygiene-v1-1-dfb6ae2588ba@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index dc5dfb2ca114..6ade192d27e6 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, ToTokens, TokenStreamExt}; +use quote::{format_ident, quote, quote_spanned, ToTokens, TokenStreamExt}; use syn::{ braced, parenthesized, parse::{End, Parse}, @@ -269,7 +269,7 @@ fn expand( }, |(_, err)| Box::new(err), ); - let slot = format_ident!("slot"); + let slot = Ident::new("slot", Span::mixed_site()); let (has_data_trait, get_data, init_from_closure) = if pinned { ( format_ident!("HasPinData"), @@ -302,7 +302,7 @@ fn assert_zeroable(_: *mut T) }; let this = match this { None => quote!(), - Some(This { ident, .. }) => quote! { + Some(This { ident, .. }) => quote_spanned! { Span::mixed_site() => // Create the `this` so it can be referenced by the user inside of the // expressions creating the individual fields. let #ident = unsafe { ::core::ptr::NonNull::new_unchecked(slot) }; @@ -312,7 +312,7 @@ fn assert_zeroable(_: *mut T) let data = Ident::new("__data", Span::mixed_site()); let init_fields = init_fields(&fields, pinned, &data, &slot); let field_check = make_field_check(&fields, init_kind, &path); - Ok(quote! {{ + Ok(quote_spanned! { Span::mixed_site() => { // Get the data about fields from the supplied type. // SAFETY: TODO let #data = unsafe { @@ -512,7 +512,7 @@ fn make_field_check( ..::core::mem::zeroed() }), }; - quote! { + quote_spanned! { Span::mixed_site() => #[allow(unreachable_code)] // We use unreachable code to perform field checks. They're still checked by the compiler. // SAFETY: this code is never executed. From 7f849c19e22f105c756b32a6a00dfee9cc1a2c2c Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 16 Sep 2026 13:11:33 +0100 Subject: [PATCH 07/91] rust: pin-init: internal: init: use `slot` identifier directly with mixed site Use `quote_spanned!(Span::mixed_site() => ...)` directly instead of creating `slot` identifier and interpolate it later. Do the same for `__data`, too. This is needed so that we can customize the location of the span without having the `slot` identifier getting in the way. For example: quote_spanned!(Span::mixed_site().located_at(loc) => ...) with `slot` mentioned directly will have the diagnostic pointing to the desired location, while `#slot` won't. Link: https://patch.msgid.link/20260916-dev-hygiene-v1-2-dfb6ae2588ba@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 31 ++++++++++++------------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 6ade192d27e6..a77ffef9926b 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -269,7 +269,6 @@ fn expand( }, |(_, err)| Box::new(err), ); - let slot = Ident::new("slot", Span::mixed_site()); let (has_data_trait, get_data, init_from_closure) = if pinned { ( format_ident!("HasPinData"), @@ -286,7 +285,7 @@ fn expand( let init_kind = get_init_kind(rest, dcx); let zeroable_check = match init_kind { InitKind::Normal => quote!(), - InitKind::Zeroing => quote! { + InitKind::Zeroing => quote_spanned! { Span::mixed_site() => // The user specified `..Zeroable::zeroed()` at the end of the list of fields. // Therefore we check if the struct implements `Zeroable` and then zero the memory. // This allows us to also remove the check that all fields are present (since we @@ -295,9 +294,9 @@ fn assert_zeroable(_: *mut T) where T: ::pin_init::Zeroable {} // Ensure that the struct is indeed `Zeroable`. - assert_zeroable(#slot); + assert_zeroable(slot); // SAFETY: The type implements `Zeroable` by the check above. - unsafe { ::core::ptr::write_bytes(#slot, 0, 1) }; + unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; }, }; let this = match this { @@ -309,20 +308,19 @@ fn assert_zeroable(_: *mut T) }, }; // `mixed_site` ensures that the data is not accessible to the user-controlled code. - let data = Ident::new("__data", Span::mixed_site()); - let init_fields = init_fields(&fields, pinned, &data, &slot); + let init_fields = init_fields(&fields, pinned); let field_check = make_field_check(&fields, init_kind, &path); Ok(quote_spanned! { Span::mixed_site() => { // Get the data about fields from the supplied type. // SAFETY: TODO - let #data = unsafe { + let data = unsafe { use ::pin_init::__internal::#has_data_trait; // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit // generics (which need to be present with that syntax). #path::#get_data() }; - // Ensure that `#data` really is of type `#data` and help with type inference: - let init = #data.__make_closure::<_, #error>( + // Ensure that `data` really is of type `data` and help with type inference: + let init = data.__make_closure::<_, #error>( move |slot| { #zeroable_check #this @@ -380,12 +378,7 @@ fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKi } /// Generate the code that initializes the fields of the struct using the initializers in `field`. -fn init_fields( - fields: &Punctuated, - pinned: bool, - data: &Ident, - slot: &Ident, -) -> TokenStream { +fn init_fields(fields: &Punctuated, pinned: bool) -> TokenStream { let mut guards = vec![]; let mut guard_attrs = vec![]; let mut res = TokenStream::new(); @@ -413,16 +406,16 @@ fn init_fields( let ident = member.as_ident(); let slot = if pinned { - quote! { + quote_spanned! { Span::mixed_site() => // SAFETY: // - `slot` is valid and properly aligned. // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned. // - `make_field_check` prevents `#member` from being used twice, therefore // `(*slot).#member` is exclusively accessed and has not been initialized. - (unsafe { #data.#ident(#slot) }) + (unsafe { data.#ident(slot) }) } } else { - quote! { + quote_spanned! { Span::mixed_site() => // For `init!()` macro, everything is unpinned. // SAFETY: // - `&raw mut (*slot).#member` is valid. @@ -431,7 +424,7 @@ fn init_fields( // `(*slot).#member` is exclusively accessed and has not been initialized. (unsafe { ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new( - &raw mut (*#slot).#member + &raw mut (*slot).#member ) }) } From fb3d5c6b4cca465c68472a918b861b2296768dad Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 16 Sep 2026 13:11:34 +0100 Subject: [PATCH 08/91] rust: pin-init: internal: init: provide span for slot projection When initializing a field, pin-init first projects the full slot to a slot of the field, then initialize it by invoking a method. Currently the span location is not explicitly set, so the error message points to the full macro invocation. Improve it by provide a span, so Rust will point to the specific fields causing the error when type check fails. Old error message: error[E0308]: mismatched types --> tests/ui/compile-fail/init/field_value_wrong_type.rs:8:28 | 8 | let _ = init!(Foo { a: () }); | ---------------^^--- | | | | | expected `usize`, found `()` | arguments to this method are incorrect New error message: error[E0308]: mismatched types --> tests/ui/compile-fail/init/field_value_wrong_type.rs:8:28 | 8 | let _ = init!(Foo { a: () }); | ---^^ | | | | | expected `usize`, found `()` | arguments to this method are incorrect Link: https://patch.msgid.link/20260916-dev-hygiene-v1-3-dfb6ae2588ba@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index a77ffef9926b..1d2db93dd33d 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -404,9 +404,10 @@ fn init_fields(fields: &Punctuated, pinned: bool) - } }; let ident = member.as_ident(); + let span = Span::mixed_site().located_at(ident.span()); let slot = if pinned { - quote_spanned! { Span::mixed_site() => + quote_spanned! { span => // SAFETY: // - `slot` is valid and properly aligned. // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned. @@ -415,7 +416,7 @@ fn init_fields(fields: &Punctuated, pinned: bool) - (unsafe { data.#ident(slot) }) } } else { - quote_spanned! { Span::mixed_site() => + quote_spanned! { span => // For `init!()` macro, everything is unpinned. // SAFETY: // - `&raw mut (*slot).#member` is valid. @@ -432,6 +433,7 @@ fn init_fields(fields: &Punctuated, pinned: bool) - // `mixed_site` ensures that the guard is not accessible to the user-controlled code. let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + let full_span = kind.span(); let init = match kind { InitializerKind::Value { value, .. } => { @@ -440,14 +442,13 @@ fn init_fields(fields: &Punctuated, pinned: bool) - .map(|(_, value)| quote!(#value)) .unwrap_or_else(|| quote!(#member)); - quote! { + quote_spanned! { full_span => #(#attrs)* let mut #guard = #slot.write(#value); - } } InitializerKind::Init { value, .. } => { - quote! { + quote_spanned! { full_span => #(#attrs)* let mut #guard = #slot.init(#value)?; } @@ -458,7 +459,7 @@ fn init_fields(fields: &Punctuated, pinned: bool) - // A tuple field has no name that could be bound here (the `_0` identifiers are considered // implementation detail and not user-facing). let binding = match member { - Member::Named(ident) => quote! { + Member::Named(ident) => quote_spanned! { span => #(#cfgs)* // Allow `non_snake_case` since the same warning is going to be reported for the // struct field. From dfb6a037fd586f1ffcba3b143dab58f5c88294d1 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 16 Sep 2026 19:41:30 +0100 Subject: [PATCH 09/91] rust: pin-init: internal: use span of `Index` for generated identifiers Use the span of `Index` for generated identifiers, so diagnostics can point to the span of the index. Link: https://patch.msgid.link/20260916184131.435059-1-gary@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index ed2c78f0658f..0b5c28b21091 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -41,7 +41,7 @@ impl MemberExt for Member { fn as_ident(&self) -> Ident { match self { Member::Named(ident) => ident.clone(), - Member::Unnamed(Index { index, .. }) => format_ident!("_{index}"), + Member::Unnamed(Index { index, span }) => format_ident!("_{index}", span = *span), } } From 125a931f9d7f0fd93037f1c8c8154ab5b19678c8 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Sun, 21 Jun 2026 19:59:07 +0900 Subject: [PATCH 10/91] rust: time: warn and assert out-of-range input in fsleep() and udelay() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fsleep() documents out-of-range input as a bug but does not check it, unlike udelay(). Add `debug_assert!` calls to catch it in debug builds. Also add `pr_warn_once!` calls in both fsleep() and udelay() so the clamped path is observable at runtime. Reviewed-by: Onur Özkan Reviewed-by: Andreas Hindborg Signed-off-by: Sang-Heon Jeon Link: https://msgid.link/20260621105909.4055210-1-ekffu200098@gmail.com Signed-off-by: Andreas Hindborg --- rust/kernel/time/delay.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs index b5b1b42797a0..4debe23731bf 100644 --- a/rust/kernel/time/delay.rs +++ b/rust/kernel/time/delay.rs @@ -8,7 +8,10 @@ //! C header: [`include/linux/delay.h`](srctree/include/linux/delay.h). use super::Delta; -use crate::prelude::*; +use crate::{ + pr_warn_once, + prelude::*, // +}; /// Sleeps for a given duration at least. /// @@ -18,8 +21,7 @@ /// `delta` must be within `[0, i32::MAX]` microseconds; /// otherwise, it is erroneous behavior. That is, it is considered a bug /// to call this function with an out-of-range value, in which case the function -/// will sleep for at least the maximum value in the range and may warn -/// in the future. +/// will sleep for at least the maximum value in the range and warns once. /// /// The behavior above differs from the C side [`fsleep()`] for which out-of-range /// values mean "infinite timeout" instead. @@ -32,10 +34,13 @@ pub fn fsleep(delta: Delta) { // overflow inside fsleep, which could lead to unintentional infinite sleep. const MAX_DELTA: Delta = Delta::from_micros(i32::MAX as i64); + debug_assert!(delta.as_nanos() >= 0); + debug_assert!(delta <= MAX_DELTA); + let delta = if (Delta::ZERO..=MAX_DELTA).contains(&delta) { delta } else { - // TODO: Add WARN_ONCE() when it's supported. + pr_warn_once!("attempted to fsleep() with out of range delta\n"); MAX_DELTA }; @@ -54,7 +59,8 @@ pub fn fsleep(delta: Delta) { /// /// `delta` must be within `[0, MAX_UDELAY_MS]` in milliseconds; /// otherwise, it is erroneous behavior. That is, it is considered a bug to -/// call this function with an out-of-range value. +/// call this function with an out-of-range value, in which case the function +/// will delay for at least the maximum value in the range and warns once. /// /// The behavior above differs from the C side [`udelay()`] for which out-of-range /// values could lead to an overflow and unexpected behavior. @@ -69,6 +75,7 @@ pub fn udelay(delta: Delta) { let delta = if (Delta::ZERO..=MAX_UDELAY_DELTA).contains(&delta) { delta } else { + pr_warn_once!("attempted to udelay() with out of range delta\n"); MAX_UDELAY_DELTA }; From 6a8e2c63f2fece3828ac09e4e434666de842acfe Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Wed, 22 Jul 2026 09:10:50 +0900 Subject: [PATCH 11/91] rust: time: rename ClockSource trait to ClockId The `ClockSource` trait has nothing to do with the C `struct clocksource` in `include/linux/clocksource.h`, which abstracts the hardware counter used as a source of time for the majority of the clockids. The trait instead carries a `clockid_t` `ID`, i.e. one of the IDs of "the various system clocks (for POSIX.1b interval timers)" as described in include/uapi/linux/time.h (CLOCK_MONOTONIC, CLOCK_REALTIME, ...). It thus plays the role of a `clockid_t`, and the `ClockSource` name overlaps confusingly with the C `clocksource` concept when reading across C and Rust code. Rename the trait to `ClockId` to reflect that it represents a `clockid_t`. This is a pure rename; there is no functional change. Suggested-by: John Stultz Signed-off-by: FUJITA Tomonori Link: https://lore.kernel.org/rust-for-linux/CANDhNCrKMdHCmL76LWCROVF2Ly-9NxmfmQ2T+P=iv34aVkO2uQ@mail.gmail.com/ Reviewed-by: Andreas Hindborg Link: https://msgid.link/20260722001050.4115708-1-tomo@flapping.org Signed-off-by: Andreas Hindborg --- rust/kernel/time.rs | 34 +++++++-------- rust/kernel/time/hrtimer.rs | 82 ++++++++++++++++++------------------- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 6c0a5e8090d0..60717f17e9ba 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -53,9 +53,9 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { unsafe { bindings::__msecs_to_jiffies(msecs) } } -/// Trait for clock sources. +/// Trait for kernel clock identifiers. /// -/// Selection of the clock source depends on the use case. In some cases the usage of a +/// Selection of the clock depends on the use case. In some cases the usage of a /// particular clock is mandatory, e.g. in network protocols, filesystems. In other /// cases the user of the clock has to decide which clock is best suited for the /// purpose. In most scenarios clock [`Monotonic`] is the best choice as it @@ -66,13 +66,13 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { /// Implementers must ensure that `ktime_get()` returns a value in the inclusive range /// `0..=KTIME_MAX` (i.e., greater than or equal to 0 and less than or equal to /// `KTIME_MAX`, where `KTIME_MAX` equals `i64::MAX`). -pub unsafe trait ClockSource { - /// The kernel clock ID associated with this clock source. +pub unsafe trait ClockId { + /// The kernel clock ID associated with this clock. /// /// This constant corresponds to the C side `clockid_t` value. const ID: bindings::clockid_t; - /// Get the current time from the clock source. + /// Get the current time from the clock. /// /// The function must return a value in the range `0..=KTIME_MAX`. fn ktime_get() -> bindings::ktime_t; @@ -93,7 +93,7 @@ pub unsafe trait ClockSource { // SAFETY: The kernel's `ktime_get()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for Monotonic { +unsafe impl ClockId for Monotonic { const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -120,7 +120,7 @@ fn ktime_get() -> bindings::ktime_t { // SAFETY: The kernel's `ktime_get_real()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for RealTime { +unsafe impl ClockId for RealTime { const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -140,7 +140,7 @@ fn ktime_get() -> bindings::ktime_t { // SAFETY: The kernel's `ktime_get_boottime()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for BootTime { +unsafe impl ClockId for BootTime { const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -164,7 +164,7 @@ fn ktime_get() -> bindings::ktime_t { // SAFETY: The kernel's `ktime_get_clocktai()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for Tai { +unsafe impl ClockId for Tai { const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -180,24 +180,24 @@ fn ktime_get() -> bindings::ktime_t { /// The `inner` value is in the range from 0 to `KTIME_MAX`. #[repr(transparent)] #[derive(PartialEq, PartialOrd, Eq, Ord)] -pub struct Instant { +pub struct Instant { inner: bindings::ktime_t, _c: PhantomData, } -impl Clone for Instant { +impl Clone for Instant { fn clone(&self) -> Self { *self } } -impl Copy for Instant {} +impl Copy for Instant {} -impl Instant { +impl Instant { /// Get the current time from the clock source. #[inline] pub fn now() -> Self { - // INVARIANT: The `ClockSource::ktime_get()` function returns a value in the range + // INVARIANT: The `ClockId::ktime_get()` function returns a value in the range // from 0 to `KTIME_MAX`. Self { inner: C::ktime_get(), @@ -239,7 +239,7 @@ pub(crate) unsafe fn from_ktime(ktime: bindings::ktime_t) -> Self { } } -impl ops::Sub for Instant { +impl ops::Sub for Instant { type Output = Delta; // By the type invariant, it never overflows. @@ -251,7 +251,7 @@ fn sub(self, other: Instant) -> Delta { } } -impl ops::Add for Instant { +impl ops::Add for Instant { type Output = Self; #[inline] @@ -271,7 +271,7 @@ fn add(self, rhs: Delta) -> Self::Output { } } -impl ops::Sub for Instant { +impl ops::Sub for Instant { type Output = Self; #[inline] diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs index 2d7f1131a813..87aafbee0ce9 100644 --- a/rust/kernel/time/hrtimer.rs +++ b/rust/kernel/time/hrtimer.rs @@ -403,14 +403,14 @@ //! //! [`Arc`]: kernel::sync::Arc -use super::{ClockSource, Delta, Instant}; +use super::{ClockId, Delta, Instant}; use crate::{prelude::*, types::Opaque}; use core::{marker::PhantomData, ptr::NonNull}; use pin_init::PinInit; /// A type-alias to refer to the [`Instant`] for a given `T` from [`HrTimer`]. /// -/// Where `C` is the [`ClockSource`] of the [`HrTimer`]. +/// Where `C` is the [`ClockId`] of the [`HrTimer`]. pub type HrTimerInstant = Instant<<>::TimerMode as HrTimerMode>::Clock>; /// A timer backed by a C `struct hrtimer`. @@ -855,7 +855,7 @@ pub trait HrTimerExpires { fn as_nanos(&self) -> i64; } -impl HrTimerExpires for Instant { +impl HrTimerExpires for Instant { #[inline] fn as_nanos(&self) -> i64 { Instant::::as_nanos(self) @@ -870,22 +870,22 @@ fn as_nanos(&self) -> i64 { } mod private { - use crate::time::ClockSource; + use crate::time::ClockId; pub trait Sealed {} - impl Sealed for super::AbsoluteMode {} - impl Sealed for super::RelativeMode {} - impl Sealed for super::AbsolutePinnedMode {} - impl Sealed for super::RelativePinnedMode {} - impl Sealed for super::AbsoluteSoftMode {} - impl Sealed for super::RelativeSoftMode {} - impl Sealed for super::AbsolutePinnedSoftMode {} - impl Sealed for super::RelativePinnedSoftMode {} - impl Sealed for super::AbsoluteHardMode {} - impl Sealed for super::RelativeHardMode {} - impl Sealed for super::AbsolutePinnedHardMode {} - impl Sealed for super::RelativePinnedHardMode {} + impl Sealed for super::AbsoluteMode {} + impl Sealed for super::RelativeMode {} + impl Sealed for super::AbsolutePinnedMode {} + impl Sealed for super::RelativePinnedMode {} + impl Sealed for super::AbsoluteSoftMode {} + impl Sealed for super::RelativeSoftMode {} + impl Sealed for super::AbsolutePinnedSoftMode {} + impl Sealed for super::RelativePinnedSoftMode {} + impl Sealed for super::AbsoluteHardMode {} + impl Sealed for super::RelativeHardMode {} + impl Sealed for super::AbsolutePinnedHardMode {} + impl Sealed for super::RelativePinnedHardMode {} } /// Operational mode of [`HrTimer`]. @@ -894,16 +894,16 @@ pub trait HrTimerMode: private::Sealed { const C_MODE: bindings::hrtimer_mode; /// Type representing the clock source. - type Clock: ClockSource; + type Clock: ClockId; /// Type representing the expiration specification (absolute or relative time). type Expires: HrTimerExpires; } /// Timer that expires at a fixed point in time. -pub struct AbsoluteMode(PhantomData); +pub struct AbsoluteMode(PhantomData); -impl HrTimerMode for AbsoluteMode { +impl HrTimerMode for AbsoluteMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS; type Clock = C; @@ -911,9 +911,9 @@ impl HrTimerMode for AbsoluteMode { } /// Timer that expires after a delay from now. -pub struct RelativeMode(PhantomData); +pub struct RelativeMode(PhantomData); -impl HrTimerMode for RelativeMode { +impl HrTimerMode for RelativeMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL; type Clock = C; @@ -921,8 +921,8 @@ impl HrTimerMode for RelativeMode { } /// Timer with absolute expiration time, pinned to its current CPU. -pub struct AbsolutePinnedMode(PhantomData); -impl HrTimerMode for AbsolutePinnedMode { +pub struct AbsolutePinnedMode(PhantomData); +impl HrTimerMode for AbsolutePinnedMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED; type Clock = C; @@ -930,8 +930,8 @@ impl HrTimerMode for AbsolutePinnedMode { } /// Timer with relative expiration time, pinned to its current CPU. -pub struct RelativePinnedMode(PhantomData); -impl HrTimerMode for RelativePinnedMode { +pub struct RelativePinnedMode(PhantomData); +impl HrTimerMode for RelativePinnedMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED; type Clock = C; @@ -939,8 +939,8 @@ impl HrTimerMode for RelativePinnedMode { } /// Timer with absolute expiration, handled in soft irq context. -pub struct AbsoluteSoftMode(PhantomData); -impl HrTimerMode for AbsoluteSoftMode { +pub struct AbsoluteSoftMode(PhantomData); +impl HrTimerMode for AbsoluteSoftMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_SOFT; type Clock = C; @@ -948,8 +948,8 @@ impl HrTimerMode for AbsoluteSoftMode { } /// Timer with relative expiration, handled in soft irq context. -pub struct RelativeSoftMode(PhantomData); -impl HrTimerMode for RelativeSoftMode { +pub struct RelativeSoftMode(PhantomData); +impl HrTimerMode for RelativeSoftMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_SOFT; type Clock = C; @@ -957,8 +957,8 @@ impl HrTimerMode for RelativeSoftMode { } /// Timer with absolute expiration, pinned to CPU and handled in soft irq context. -pub struct AbsolutePinnedSoftMode(PhantomData); -impl HrTimerMode for AbsolutePinnedSoftMode { +pub struct AbsolutePinnedSoftMode(PhantomData); +impl HrTimerMode for AbsolutePinnedSoftMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT; type Clock = C; @@ -966,8 +966,8 @@ impl HrTimerMode for AbsolutePinnedSoftMode { } /// Timer with absolute expiration, pinned to CPU and handled in soft irq context. -pub struct RelativePinnedSoftMode(PhantomData); -impl HrTimerMode for RelativePinnedSoftMode { +pub struct RelativePinnedSoftMode(PhantomData); +impl HrTimerMode for RelativePinnedSoftMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT; type Clock = C; @@ -975,8 +975,8 @@ impl HrTimerMode for RelativePinnedSoftMode { } /// Timer with absolute expiration, handled in hard irq context. -pub struct AbsoluteHardMode(PhantomData); -impl HrTimerMode for AbsoluteHardMode { +pub struct AbsoluteHardMode(PhantomData); +impl HrTimerMode for AbsoluteHardMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_HARD; type Clock = C; @@ -984,8 +984,8 @@ impl HrTimerMode for AbsoluteHardMode { } /// Timer with relative expiration, handled in hard irq context. -pub struct RelativeHardMode(PhantomData); -impl HrTimerMode for RelativeHardMode { +pub struct RelativeHardMode(PhantomData); +impl HrTimerMode for RelativeHardMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_HARD; type Clock = C; @@ -993,8 +993,8 @@ impl HrTimerMode for RelativeHardMode { } /// Timer with absolute expiration, pinned to CPU and handled in hard irq context. -pub struct AbsolutePinnedHardMode(PhantomData); -impl HrTimerMode for AbsolutePinnedHardMode { +pub struct AbsolutePinnedHardMode(PhantomData); +impl HrTimerMode for AbsolutePinnedHardMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD; type Clock = C; @@ -1002,8 +1002,8 @@ impl HrTimerMode for AbsolutePinnedHardMode { } /// Timer with relative expiration, pinned to CPU and handled in hard irq context. -pub struct RelativePinnedHardMode(PhantomData); -impl HrTimerMode for RelativePinnedHardMode { +pub struct RelativePinnedHardMode(PhantomData); +impl HrTimerMode for RelativePinnedHardMode { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD; type Clock = C; From f201d785cae6febe2be33235f541ef25b2a13240 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 22 Sep 2026 22:13:19 +0100 Subject: [PATCH 12/91] rust: pin-init: internal: pin_data: use HRTB to work around `trivial_bounds` for `Unpin` `#[pin_data]` generates a `Unpin` implementation for the user, which requires reasoning using field types. This is rejected by rustc as "trivial bounds", which can only be specified using the `#![feature(trivial_bounds)]`. Currently we use an extra lifetime in `__Unpin` to work around this; use HRTB to introduce a dummy lifetime can achieve the same, which the `Zeroable` implementation already uses. Unify the `Unpin` impl to also use the same HRTB trick that `Zeroable` uses. This produces slightly better error message and slightly shorter expansion. Link: https://patch.msgid.link/20260922211320.1390159-1-gary@kernel.org Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 35 +++++++++----------------- rust/pin-init/src/__internal.rs | 16 ------------ 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 8cd9bf139567..55c627499b35 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -5,10 +5,10 @@ use syn::{ parse::{End, Nothing, Parse}, parse_quote, parse_quote_spanned, + punctuated::Punctuated, spanned::Spanned, visit_mut::VisitMut, Field, Fields, Generics, Ident, Index, Item, Member, PathSegment, Type, TypePath, Visibility, - WhereClause, }; use crate::{ @@ -231,21 +231,11 @@ fn generate_unpin_impl( generics: &Generics, fields: &[FieldInfo<'_>], ) -> TokenStream { - let (_, ty_generics, _) = generics.split_for_impl(); - let mut generics_with_pin_lt = generics.clone(); - generics_with_pin_lt.params.insert(0, parse_quote!('__pin)); - generics_with_pin_lt.make_where_clause(); - let ( - impl_generics_with_pin_lt, - ty_generics_with_pin_lt, - Some(WhereClause { - where_token, - predicates, - }), - ) = generics_with_pin_lt.split_for_impl() - else { - unreachable!() - }; + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + let predicates = whr + .map(|x| &x.predicates) + .unwrap_or(const { &Punctuated::new() }); + let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { let ident = f.member.as_ident(); let ty = &f.field.ty; @@ -260,19 +250,18 @@ fn generate_unpin_impl( dead_code, // The fields below are never used. non_snake_case // The warning will be emitted on the struct definition. )] - struct __Unpin #generics_with_pin_lt - #where_token - #predicates + struct __Unpin #generics #whr { - __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>, __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>, #(#pinned_fields),* } #[doc(hidden)] - impl #impl_generics_with_pin_lt ::core::marker::Unpin for #ident #ty_generics - #where_token - __Unpin #ty_generics_with_pin_lt: ::core::marker::Unpin, + impl #impl_generics ::core::marker::Unpin for #ident #ty_generics + where + // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds` + // feature . + for<'__dummy> __Unpin #ty_generics: ::core::marker::Unpin, #predicates {} } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 8e9fd18b993f..67a354fa29c4 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -42,22 +42,6 @@ pub const fn new() -> Self { } } -/// Zero-sized type used to mark a lifetime as invariant. -/// -/// This is a polyfill for the [unstable type] in the standard library of the same name. -/// -/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html -#[repr(transparent)] -#[derive(Clone, Copy, Default)] -pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>); - -impl PhantomInvariantLifetime<'_> { - #[inline(always)] - pub const fn new() -> Self { - Self(PhantomInvariant::new()) - } -} - /// Token type to signify successful initialization. /// /// Can only be constructed via the unsafe [`Self::new`] function. The initializer macros use this From ad5545276a22ccd6061829b9e697edff6c910531 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 19:20:14 +0100 Subject: [PATCH 13/91] rust: pin-init: internal: make `__init_data` and `__pin_data` safe Remove the unsafe markers of `__init_data` and `__pin_data`. These methods are for inference help and does not need to be unsafe. The `HasInitData` cannot be implemented outside pin-init, so the `unsafe trait` marker is not necessary. Link: https://patch.msgid.link/20260923-dev-pin-data-diag-v1-1-e4d7193569e5@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 3 +-- rust/pin-init/internal/src/pin_data.rs | 2 +- rust/pin-init/src/__internal.rs | 11 +++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 1d2db93dd33d..1957be83a196 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -312,8 +312,7 @@ fn assert_zeroable(_: *mut T) let field_check = make_field_check(&fields, init_kind, &path); Ok(quote_spanned! { Span::mixed_site() => { // Get the data about fields from the supplied type. - // SAFETY: TODO - let data = unsafe { + let data = { use ::pin_init::__internal::#has_data_trait; // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit // generics (which need to be present with that syntax). diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 55c627499b35..9fdce1593c7f 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -527,7 +527,7 @@ unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name # type PinData = __ThePinData #ty_generics; #[inline] - unsafe fn __pin_data() -> Self::PinData { + fn __pin_data() -> Self::PinData { __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 67a354fa29c4..532ddcdc1c6a 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -65,12 +65,12 @@ pub unsafe fn new() -> Self { /// /// # Safety /// -/// Only the `init` module is allowed to use this trait. +/// `pin-init` relies on the correctness of the helper functions defined on `PinData`. +/// Thus, only the `#[pin_data]` can implement this trait. pub unsafe trait HasPinData { type PinData; - #[expect(clippy::missing_safety_doc)] - unsafe fn __pin_data() -> Self::PinData; + fn __pin_data() -> Self::PinData; } /// This trait is automatically implemented for every type. It aims to provide the same type @@ -82,8 +82,7 @@ pub unsafe trait HasPinData { pub unsafe trait HasInitData { type InitData; - #[expect(clippy::missing_safety_doc)] - unsafe fn __init_data() -> Self::InitData; + fn __init_data() -> Self::InitData; } pub struct AllData(PhantomInvariant); @@ -113,7 +112,7 @@ unsafe impl HasInitData for T { type InitData = AllData; #[inline] - unsafe fn __init_data() -> Self::InitData { + fn __init_data() -> Self::InitData { AllData(PhantomInvariant::new()) } } From c0243b93d0be35a4e8687b658991e3e6b2f1e9e7 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 19:20:15 +0100 Subject: [PATCH 14/91] rust: pin-init: internal: remove associated type of `HasInitData` There is only a single implementation, the associated type is not necessary. Remove it, and rename `AllData` to `InitData` to be a more meaningful name. The single implementation also means that "unsafe" is redundant, so remove it as well. Link: https://patch.msgid.link/20260923-dev-pin-data-diag-v1-2-e4d7193569e5@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/src/__internal.rs | 37 +++++++++++++-------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 532ddcdc1c6a..e5f75432fe87 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -73,30 +73,31 @@ pub unsafe trait HasPinData { fn __pin_data() -> Self::PinData; } -/// This trait is automatically implemented for every type. It aims to provide the same type -/// inference help as `HasPinData`. +/// This trait is automatically implemented for every type. /// -/// # Safety -/// -/// Only the `init` module is allowed to use this trait. -pub unsafe trait HasInitData { - type InitData; - - fn __init_data() -> Self::InitData; +/// It aims to provide type inference help; `PATH::__init_data()` would be able to retrieve an +/// instance of `InitData>` without having to mention the generics explicitly. +pub trait HasInitData { + #[inline] + fn __init_data() -> InitData { + InitData(PhantomInvariant::new()) + } } -pub struct AllData(PhantomInvariant); +impl HasInitData for T {} -impl Clone for AllData { +pub struct InitData(PhantomInvariant); + +impl Clone for InitData { #[inline] fn clone(&self) -> Self { *self } } -impl Copy for AllData {} +impl Copy for InitData {} -impl AllData { +impl InitData { /// Type inference helper function. #[inline(always)] pub fn __make_closure(self, f: F) -> F @@ -107,16 +108,6 @@ pub fn __make_closure(self, f: F) -> F } } -// SAFETY: TODO. -unsafe impl HasInitData for T { - type InitData = AllData; - - #[inline] - fn __init_data() -> Self::InitData { - AllData(PhantomInvariant::new()) - } -} - /// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. /// /// # Invariants From c5155147cdf0053694f8bcdfcc616d254ae6842e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 19:20:16 +0100 Subject: [PATCH 15/91] rust: pin-init: internal: use `HasInitData` to provide inference help only Currently, type inference of the initialized type is done by use the function calling syntax `PATH::__pin_data` or `PATH::__init_data` to obtain the required init data type, without needing the user to explicitly mention all generics. Switch to always use `HasInitData::__init_data` to provide that inference, which will not produce method-not-found errors as it is always available. Change `__pin_data` to take `InitData` as the receiver, so for types that require `HasPinData`, `HasPinData::__pin_data(data)` can be used to obtain the needed type without having to mention the user-provided path again. The expanded code also uses the span of the type, so diagnostics will hint to user that the type is causing the issue. With this change, the error message when `HasPinData` is not implemented changes from error[E0599]: no associated function or constant named `__pin_data` found for struct `Foo` in the current scope --> tests/ui/compile-fail/init/missing_pin_data.rs:9:9 | 3 | struct Foo { | ---------- associated function or constant `__pin_data` not found for this struct ... 9 | pin_init!(Self { a: 42 }) | ^^^^^^^^^^^^^^^^^^^^^^^^^ associated function or constant not found in `Foo` | = help: items from traits can only be used if the trait is implemented and in scope = note: the following trait defines an item `__pin_data`, perhaps you need to implement it: candidate #1: `pin_init::__internal::HasPinData` to error[E0277]: the trait bound `Foo: pin_init::__internal::HasPinData` is not satisfied --> tests/ui/compile-fail/init/missing_pin_data.rs:9:19 | 9 | pin_init!(Self { a: 42 }) | ^^^^ unsatisfied trait bound | help: the trait `pin_init::__internal::HasPinData` is not implemented for `Foo` --> tests/ui/compile-fail/init/missing_pin_data.rs:3:1 | 3 | struct Foo { | ^^^^^^^^^^ Link: https://patch.msgid.link/20260923-dev-pin-data-diag-v1-3-e4d7193569e5@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 22 ++++++++++++---------- rust/pin-init/internal/src/pin_data.rs | 2 +- rust/pin-init/src/__internal.rs | 2 +- rust/pin-init/src/lib.rs | 3 ++- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 1957be83a196..e98182c65447 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -269,18 +269,17 @@ fn expand( }, |(_, err)| Box::new(err), ); - let (has_data_trait, get_data, init_from_closure) = if pinned { + let (get_pin_data, init_from_closure) = if pinned { ( - format_ident!("HasPinData"), - format_ident!("__pin_data"), + Some( + quote_spanned! { path.span().resolved_at(Span::mixed_site()) => + let data = ::pin_init::__internal::HasPinData::__pin_data(data); + }, + ), format_ident!("pin_init_from_closure"), ) } else { - ( - format_ident!("HasInitData"), - format_ident!("__init_data"), - format_ident!("init_from_closure"), - ) + (None, format_ident!("init_from_closure")) }; let init_kind = get_init_kind(rest, dcx); let zeroable_check = match init_kind { @@ -313,11 +312,14 @@ fn assert_zeroable(_: *mut T) Ok(quote_spanned! { Span::mixed_site() => { // Get the data about fields from the supplied type. let data = { - use ::pin_init::__internal::#has_data_trait; + use ::pin_init::__internal::HasInitData; // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit // generics (which need to be present with that syntax). - #path::#get_data() + #path::__init_data() }; + + #get_pin_data + // Ensure that `data` really is of type `data` and help with type inference: let init = data.__make_closure::<_, #error>( move |slot| { diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 9fdce1593c7f..982a38ceea93 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -527,7 +527,7 @@ unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name # type PinData = __ThePinData #ty_generics; #[inline] - fn __pin_data() -> Self::PinData { + fn __pin_data(_: ::pin_init::__internal::InitData) -> Self::PinData { __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index e5f75432fe87..914c4d1a06eb 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -70,7 +70,7 @@ pub unsafe fn new() -> Self { pub unsafe trait HasPinData { type PinData; - fn __pin_data() -> Self::PinData; + fn __pin_data(_: InitData) -> Self::PinData; } /// This trait is automatically implemented for every type. diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 4110590348c6..319485dc18ce 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -920,7 +920,8 @@ macro_rules! assert_pinned { ($ty:ty, $field:ident, $field_ty:ty, inline) => { // SAFETY: This code is unreachable. let _ = move |ptr: *mut $ty| unsafe { - let data = <$ty as $crate::__internal::HasPinData>::__pin_data(); + let data = <$ty as $crate::__internal::HasInitData>::__init_data(); + let data = $crate::__internal::HasPinData::__pin_data(data); _ = data .$field(ptr) .init($crate::__internal::AlwaysFail::<$field_ty>::new()); From 6f78441c58c1012324da3189ea650cfeeb98966c Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 19:20:17 +0100 Subject: [PATCH 16/91] rust: pin-init: internal: add custom diagnostic when `#[pin_data]` is not implemented Produce a more helpful error message when `#[pin_data]` is omitted. The error message changes from error[E0277]: the trait bound `Foo: pin_init::__internal::HasPinData` is not satisfied --> tests/ui/compile-fail/init/missing_pin_data.rs:9:19 | 9 | pin_init!(Self { a: 42 }) | ^^^^ unsatisfied trait bound | help: the trait `pin_init::__internal::HasPinData` is not implemented for `Foo` --> tests/ui/compile-fail/init/missing_pin_data.rs:3:1 | 3 | struct Foo { | ^^^^^^^^^^ to error[E0277]: `Foo` cannot be used with `pin_init!` macro --> tests/ui/compile-fail/init/missing_pin_data.rs:9:19 | 9 | pin_init!(Self { a: 42 }) | ^^^^ unsatisfied trait bound | help: the trait `pin_init::__internal::HasPinData` is not implemented for `Foo` --> tests/ui/compile-fail/init/missing_pin_data.rs:3:1 | 3 | struct Foo { | ^^^^^^^^^^ = note: did you forget to add `#[pin_data]` attribute to the struct? Link: https://patch.msgid.link/20260923-dev-pin-data-diag-v1-4-e4d7193569e5@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/src/__internal.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 914c4d1a06eb..cba53b8c3c94 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -67,6 +67,10 @@ pub unsafe fn new() -> Self { /// /// `pin-init` relies on the correctness of the helper functions defined on `PinData`. /// Thus, only the `#[pin_data]` can implement this trait. +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be used with `pin_init!` macro", + note = "did you forget to add `#[pin_data]` attribute to the struct?" +)] pub unsafe trait HasPinData { type PinData; From 666b3e7cfa3a5c6afbfb2b5dae017617dfc40863 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 19:20:18 +0100 Subject: [PATCH 17/91] rust: pin-init: add diagnostic attribute for `PinInit` and `Init` Give a slightly more useful error message when these traits are not implemented. Before this change: error[E0277]: the trait bound `impl pin_init::PinInit: Init` is not satisfied --> tests/ui/compile-fail/init/invalid_init.rs:19:16 | 19 | bar <- Bar::new(), | -------^^^^^^^^^^ | | | | | the trait `Init` is not implemented for `impl pin_init::PinInit` | required by a bound introduced by this call After this change: error[E0277]: `impl pin_init::PinInit` cannot be used to movably initialize `Bar` with error `_` --> tests/ui/compile-fail/init/invalid_init.rs:19:16 | 19 | bar <- Bar::new(), | -------^^^^^^^^^^ | | | | | the trait `Init` is not implemented for `impl pin_init::PinInit` | required by a bound introduced by this call | = note: if your type implements `PinInit` but not `Init`, you might be forgetting a `#[pin]` annotation on fields Ideally, we would generate differnet message when `PinInit` is implemented but not `Init`; that is not feasible with today's `diagnostics::on_unimplemented` attribute. However, the added note is worth it because in `init!()` is more rarely used compared to `pin_init!()`; in most cases `Init` being unimplemented is the result of missing `#[pin]`. Link: https://patch.msgid.link/20260923-dev-pin-data-diag-v1-5-e4d7193569e5@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 319485dc18ce..d1ed0561bb27 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -966,6 +966,9 @@ macro_rules! assert_pinned { #[cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] #[cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be used to initialize `{T}` with error `{E}`" +)] pub unsafe trait PinInit: Sized { /// Alias of [`PinInit::__init`]. /// @@ -1100,6 +1103,11 @@ unsafe fn __init(self, slot: *mut T) -> Result<(), E> { #[cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] #[cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be used to movably initialize `{T}` with error `{E}`", + note = "if your type implements `PinInit` but not `Init`, \ + you might be forgetting a `#[pin]` annotation on fields" +)] pub unsafe trait Init: PinInit { /// First initializes the value using `self` then calls the function `f` with the initialized /// value. From 01c3d218e8fa5ac4393e7959701cd6c2e90c4a81 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 15:06:13 +0100 Subject: [PATCH 18/91] rust: pin-init: internal: handle item and expr diagnostics differently Currently `DiagCtxt` is used to allow error recovery in the pin-init macros, by generating both a `compile_error!()` macro item and continue expansion, and then merge token streams together. This works very well for items, however for expressions, this will produce invalid expression and result in a confusing error message. error: macro expansion ignores `::` and any tokens following --> tests/ui/compile-fail/zeroable/invalid_spread.rs:15:9 | 13 | let _ = init!(Foo { | _____________- 14 | | a: 0, 15 | | ..MyZeroable::init_zeroed() | | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 16 | | }); | |______- caused by the macro expansion here | = note: the usage of `init!` is likely invalid in expression context Fix this by wrap the concatenated diagnostics in a block, with the generated expression in its tail position. This results in the desired error message. error: expected nothing or `..Zeroable::init_zeroed()`. --> tests/ui/compile-fail/zeroable/invalid_spread.rs:15:9 | 15 | ..MyZeroable::init_zeroed() | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Reviewed-by: Benno Lossin Link: https://patch.msgid.link/20260923140618.1978181-1-gary@kernel.org Signed-off-by: Gary Guo --- rust/pin-init/internal/src/diagnostics.rs | 55 +++++++++++++++++++---- rust/pin-init/internal/src/lib.rs | 12 ++--- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs index c7d9b3e624fc..c42f1095ab10 100644 --- a/rust/pin-init/internal/src/diagnostics.rs +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -3,7 +3,7 @@ use std::fmt::Display; use proc_macro2::TokenStream; -use quote::quote_spanned; +use quote::{quote, quote_spanned}; use syn::{spanned::Spanned, Error}; pub(crate) struct DiagCtxt(TokenStream); @@ -29,16 +29,55 @@ const fn warn() {} )); } - pub(crate) fn with( - fun: impl FnOnce(&mut DiagCtxt) -> Result, + fn with( + f: impl FnOnce(&mut DiagCtxt) -> Result, + merge_diag: impl FnOnce(TokenStream, TokenStream) -> TokenStream, + convert_diag: impl FnOnce(TokenStream) -> TokenStream, ) -> TokenStream { let mut dcx = Self(TokenStream::new()); - match fun(&mut dcx) { - Ok(mut stream) => { - stream.extend(dcx.0); - stream + match f(&mut dcx) { + Ok(stream) => { + if dcx.0.is_empty() { + stream + } else { + merge_diag(stream, dcx.0) + } } - Err(ErrorGuaranteed(())) => dcx.0, + Err(ErrorGuaranteed(())) => convert_diag(dcx.0), } } + + pub(crate) fn for_item( + f: impl FnOnce(&mut DiagCtxt) -> Result, + ) -> TokenStream { + Self::with( + f, + |mut out, diag| { + out.extend(diag); + out + }, + std::convert::identity, + ) + } + + pub(crate) fn for_expr( + f: impl FnOnce(&mut DiagCtxt) -> Result, + ) -> TokenStream { + Self::with( + f, + |out, diag| { + // Diagnostics that we generate are always items. + // So for expressions create a block to place diagnostics in item position. + quote!({ + #diag + #out + }) + }, + |diag| { + quote!({ + #diag + }) + }, + ) + } } diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index c488019d6250..0410024ba1e6 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -25,31 +25,31 @@ pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { let args = parse_macro_input!(args); let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| pin_data::pin_data(args, input, dcx)).into() + DiagCtxt::for_item(|dcx| pin_data::pin_data(args, input, dcx)).into() } #[proc_macro_attribute] pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { let args = parse_macro_input!(args); let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into() + DiagCtxt::for_item(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into() } #[proc_macro_derive(Zeroable)] pub fn derive_zeroable(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| zeroable::derive(input, dcx)).into() + DiagCtxt::for_item(|dcx| zeroable::derive(input, dcx)).into() } #[proc_macro_derive(MaybeZeroable)] pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| zeroable::maybe_derive(input, dcx)).into() + DiagCtxt::for_item(|dcx| zeroable::maybe_derive(input, dcx)).into() } #[proc_macro] pub fn init(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| { + DiagCtxt::for_expr(|dcx| { init::expand_with_cfg(input, Some("::core::convert::Infallible"), false, dcx) }) .into() @@ -58,7 +58,7 @@ pub fn init(input: TokenStream) -> TokenStream { #[proc_macro] pub fn pin_init(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); - DiagCtxt::with(|dcx| { + DiagCtxt::for_expr(|dcx| { init::expand_with_cfg(input, Some("::core::convert::Infallible"), true, dcx) }) .into() From 73c1f157907cf23ef90e105f48c40fae75ee1008 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 15:06:14 +0100 Subject: [PATCH 19/91] rust: pin-init: internal: make `DiagCtxt` available inside parser Currently, there is a split between `syn::Error` and pin-init's custom `DiagCtxt`. Parsing has no access to `DiagCtxt` and therefore cannot report diagnostics without failing the parse. Bridge the gap by making the `DiagCtxt` available inside parsers using `thread_local!`. `parse_macro_input!` returns error as token stream directly, so it is no longer suitable when the parsing is moved inside `DiagCtxt` closure. Add a `From for ErrorGuaranteed` implementation so `?` can be used to add a `syn::Error` into the current diagnostic context and obtain a `ErrorGuaranteed`. Clean up the handler of using `<-` inside tuple expression as a demonstration of the usefulness of having `DiagCtxt` access inside `Parse`. Reviewed-by: Benno Lossin Link: https://patch.msgid.link/20260923140618.1978181-2-gary@kernel.org Signed-off-by: Gary Guo --- rust/pin-init/internal/src/diagnostics.rs | 84 ++++++++++++++++++----- rust/pin-init/internal/src/init.rs | 41 ++++------- rust/pin-init/internal/src/lib.rs | 32 +++++---- 3 files changed, 96 insertions(+), 61 deletions(-) diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs index c42f1095ab10..efdcf45f40cf 100644 --- a/rust/pin-init/internal/src/diagnostics.rs +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -1,32 +1,68 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +use std::cell::RefCell; use std::fmt::Display; +use std::marker::PhantomData; use proc_macro2::TokenStream; use quote::{quote, quote_spanned}; use syn::{spanned::Spanned, Error}; -pub(crate) struct DiagCtxt(TokenStream); +pub(crate) struct DiagCtxt(PhantomData<*mut ()>); pub(crate) struct ErrorGuaranteed(()); +struct DiagCtxtData { + diag: TokenStream, +} + +thread_local! { + static DIAGNOSTICS: RefCell> = const { RefCell::new(None) }; +} + +// Allows `syn::Error` to be emitted into the current diagnostic context with just `?`. +impl From for ErrorGuaranteed { + fn from(error: syn::Error) -> Self { + DIAGNOSTICS.with_borrow_mut(|data| { + data.as_mut() + .unwrap() + .diag + .extend(error.into_compile_error()); + }); + Self(()) + } +} + impl DiagCtxt { - pub(crate) fn error(&mut self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed { - let error = Error::new(span.span(), msg); - self.0.extend(error.into_compile_error()); - ErrorGuaranteed(()) + pub(crate) fn error(&self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed { + Error::new(span.span(), msg).into() } - pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) { + pub(crate) fn warn(&self, span: impl Spanned, msg: impl Display) { // Have the message start on a new line for visual clarity. let msg = format!("\n{}", msg); - self.0.extend(quote_spanned!(span.span() => - // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable. - const _: () = { - #[deprecated = #msg] - const fn warn() {} - warn(); - }; - )); + DIAGNOSTICS.with_borrow_mut(|data| { + data.as_mut() + .unwrap() + .diag + .extend(quote_spanned!(span.span() => + // Approximate using deprecated warning while `proc_macro_diagnostic` is + // unstable. + const _: () = { + #[deprecated = #msg] + const fn warn() {} + warn(); + }; + )) + }); + } + + /// Execute the provided function with the current diagnostic context. + pub(crate) fn current(f: impl FnOnce(&DiagCtxt) -> R) -> R { + DIAGNOSTICS.with_borrow(|data| { + assert!(data.is_some(), "No active `DiagCtxt`"); + }); + + f(&DiagCtxt(PhantomData)) } fn with( @@ -34,16 +70,26 @@ fn with( merge_diag: impl FnOnce(TokenStream, TokenStream) -> TokenStream, convert_diag: impl FnOnce(TokenStream) -> TokenStream, ) -> TokenStream { - let mut dcx = Self(TokenStream::new()); - match f(&mut dcx) { + DIAGNOSTICS.with_borrow_mut(|data| { + assert!(data.is_none(), "`DiagCtxt` cannot be nested"); + *data = Some(DiagCtxtData { + diag: TokenStream::new(), + }); + }); + + let result = f(&mut DiagCtxt(PhantomData)); + + let data = DIAGNOSTICS.with_borrow_mut(|data| data.take().unwrap()); + + match result { Ok(stream) => { - if dcx.0.is_empty() { + if data.diag.is_empty() { stream } else { - merge_diag(stream, dcx.0) + merge_diag(stream, data.diag) } } - Err(ErrorGuaranteed(())) => convert_diag(dcx.0), + Err(ErrorGuaranteed(())) => convert_diag(data.diag), } } diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index e98182c65447..c5b5a0ce5c4e 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -44,9 +44,6 @@ pub(crate) enum InitExprKind { struct InitTupleField { attrs: Vec, - /// `<-` is not valid in constructor syntax; it is parsed anyway so that it can be rejected - /// with a proper diagnostic instead of a parse error. - left_arrow_token: Option, value: Expr, } @@ -84,20 +81,6 @@ fn normalize(self) -> InitExprStruct { rest: None, } } - - fn validate(&self, dcx: &mut DiagCtxt) -> Result<(), ErrorGuaranteed> { - let mut result = Ok(()); - for field in &self.fields { - if let Some(left_arrow_token) = &field.left_arrow_token { - result = Err(dcx.error( - left_arrow_token, - "`<-` is not supported in tuple constructor syntax; name the fields by index \ - instead, e.g. `Type { 0 <- initializer, 1: value }`", - )); - } - } - result - } } struct This { @@ -153,8 +136,6 @@ pub(crate) fn expand_with_cfg( ) -> Result { let initializer = match initializer.kind { InitExprKind::Tuple(expr) => { - expr.validate(dcx)?; - let mut initializer = Initializer { attrs: initializer.attrs, this: initializer.this, @@ -579,9 +560,20 @@ fn parse_with_path(path: Path, input: syn::parse::ParseStream<'_>) -> syn::Resul let paren_token = parenthesized!(content in input); let mut fields = Punctuated::new(); while !content.is_empty() { + let attrs = content.call(Attribute::parse_outer)?; + + if let Some(left_arrow_token) = content.parse::>()? { + DiagCtxt::current(|dcx| { + dcx.error( + left_arrow_token, + "`<-` is not supported in tuple constructor syntax; name the fields by \ + index instead, e.g. `Type { 0 <- initializer, 1: value }`", + ) + }); + } + fields.push_value(InitTupleField { - attrs: content.call(Attribute::parse_outer)?, - left_arrow_token: content.parse()?, + attrs, value: content.parse()?, }); if content.is_empty() { @@ -758,13 +750,8 @@ fn to_tokens(&self, tokens: &mut TokenStream) { impl ToTokens for InitTupleField { fn to_tokens(&self, tokens: &mut TokenStream) { - let Self { - attrs, - left_arrow_token, - value, - } = self; + let Self { attrs, value } = self; tokens.append_all(attrs); - left_arrow_token.to_tokens(tokens); value.to_tokens(tokens); } } diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index 0410024ba1e6..07b3d33d2282 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -10,7 +10,6 @@ #![allow(missing_docs)] use proc_macro::TokenStream; -use syn::parse_macro_input; use crate::diagnostics::DiagCtxt; @@ -23,43 +22,46 @@ #[proc_macro_attribute] pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { - let args = parse_macro_input!(args); - let input = parse_macro_input!(input); - DiagCtxt::for_item(|dcx| pin_data::pin_data(args, input, dcx)).into() + DiagCtxt::for_item(|dcx| pin_data::pin_data(syn::parse(args)?, syn::parse(input)?, dcx)).into() } #[proc_macro_attribute] pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { - let args = parse_macro_input!(args); - let input = parse_macro_input!(input); - DiagCtxt::for_item(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into() + DiagCtxt::for_item(|dcx| pinned_drop::pinned_drop(syn::parse(args)?, syn::parse(input)?, dcx)) + .into() } #[proc_macro_derive(Zeroable)] pub fn derive_zeroable(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); - DiagCtxt::for_item(|dcx| zeroable::derive(input, dcx)).into() + DiagCtxt::for_item(|dcx| zeroable::derive(syn::parse(input)?, dcx)).into() } #[proc_macro_derive(MaybeZeroable)] pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); - DiagCtxt::for_item(|dcx| zeroable::maybe_derive(input, dcx)).into() + DiagCtxt::for_item(|dcx| zeroable::maybe_derive(syn::parse(input)?, dcx)).into() } #[proc_macro] pub fn init(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); DiagCtxt::for_expr(|dcx| { - init::expand_with_cfg(input, Some("::core::convert::Infallible"), false, dcx) + init::expand_with_cfg( + syn::parse(input)?, + Some("::core::convert::Infallible"), + false, + dcx, + ) }) .into() } #[proc_macro] pub fn pin_init(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); DiagCtxt::for_expr(|dcx| { - init::expand_with_cfg(input, Some("::core::convert::Infallible"), true, dcx) + init::expand_with_cfg( + syn::parse(input)?, + Some("::core::convert::Infallible"), + true, + dcx, + ) }) .into() } From 95593ed3616194c9498cc190cd1a3693ff4bca37 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 23 Sep 2026 15:06:15 +0100 Subject: [PATCH 20/91] rust: pin-init: internal: improve diagnostics robustness against panicking Currently, if proc macro panicked, the diagnostics clean up is not executed, and further invocation will cause the "DiagCtxt cannot be nested" error. While we should aim to have no panics inside proc macros, producing a sensible diagnostics message even when macro panicked is very useful for developing. Thus, catch proc macro panics and convert them to errors, and emit them together with all diagnostics accumulated so far. Ideally we would like panic location w/ line numbers being available as well; however this is not currently implementable without overriding the global panic hook. Reviewed-by: Benno Lossin Link: https://patch.msgid.link/20260923140618.1978181-3-gary@kernel.org Signed-off-by: Gary Guo --- rust/pin-init/internal/src/diagnostics.rs | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs index efdcf45f40cf..e88e520e326b 100644 --- a/rust/pin-init/internal/src/diagnostics.rs +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -4,7 +4,7 @@ use std::fmt::Display; use std::marker::PhantomData; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::{quote, quote_spanned}; use syn::{spanned::Spanned, Error}; @@ -77,7 +77,27 @@ fn with( }); }); - let result = f(&mut DiagCtxt(PhantomData)); + let mut dcx = DiagCtxt(PhantomData); + let result = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(&mut dcx))) { + Ok(result) => result, + Err(payload) => { + // Robustness against panicking in macros. + // + // Ensure that any error messages are still emitted when this happens. + let message = if let Some(&s) = payload.downcast_ref::<&'static str>() { + s + } else if let Some(s) = payload.downcast_ref::() { + s.as_str() + } else { + "Box" + }; + + Err(dcx.error( + Span::mixed_site(), + format!("proc macro panicked: {message}"), + )) + } + }; let data = DIAGNOSTICS.with_borrow_mut(|data| data.take().unwrap()); From 4ebfdb2c8b28eb8ef85b4e4ce65e55bd5a4df8d5 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Fri, 25 Sep 2026 08:25:29 +0900 Subject: [PATCH 21/91] rust: bindings: set HZ to CONFIG_HZ include/uapi/asm-generic/param.h defines HZ as __USER_HZ and include/asm-generic/param.h then sets it to CONFIG_HZ. bindgen keeps the first definition of a macro and ignores a later #undef, so the generated binding is __USER_HZ. Redefine HZ in rust/bindings/lib.rs, the same way compat_ptr_ioctl is redefined there. Nothing in the tree reads bindings::HZ today, so this prevents accidental misuse rather than fixing a bug. Suggested-by: Alice Ryhl Reviewed-by: Gary Guo Signed-off-by: FUJITA Tomonori Link: https://msgid.link/20260924232530.446588-2-tomo@flapping.org Signed-off-by: Andreas Hindborg --- rust/bindings/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 812f8e5a08d5..c17e6c2b775f 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -81,3 +81,8 @@ mod bindings_helper { None } }; + +// `bindgen` keeps the first definition of a macro and ignores a later `#undef`. +// The generated `HZ` is therefore `__USER_HZ` from +// `include/uapi/asm-generic/param.h`, not `CONFIG_HZ`. +pub const HZ: u32 = CONFIG_HZ; From a9aba476d42df2537403f97f185a2b1979d0dd97 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Fri, 25 Sep 2026 08:25:30 +0900 Subject: [PATCH 22/91] rust: time: add Delta::to_jiffies_timeout() for timeout conversion The boundaries of __msecs_to_jiffies() and nsecs_to_jiffies64() depend on which HZ branch is compiled. With CONFIG_HZ_300 nsecs_to_jiffies64() overflows after 64.99 years, which is less than the 292 years a Delta can hold. With HZ=1000 a jiffy is a millisecond, so __msecs_to_jiffies() returns its argument unchanged and never caps it at MAX_JIFFY_OFFSET. Compute the conversion in Rust with mul_u64_add_u64_div_u64() instead. It returns (a * b + c) / d, computing a * b internally in 128 bits, so ceil(nanos * HZ / NSEC_PER_SEC) needs no input clamp and rounds once. The bound then follows from the arithmetic: with HZ <= NSEC_PER_SEC, which a static_assert() checks, the result is at most the nanosecond count. Unless the result saturates, the value is rounded up, so the timeout is never shorter than the requested span. It saturates at zero jiffies for a negative span, i.e. an immediate timeout, and at MAX_JIFFY_OFFSET, the upper bound the kernel uses for a jiffies span. Since MAX_JIFFY_OFFSET is derived from long, only 32 bit can reach it, and a saturated timeout there is finite, so it can be shorter than the requested span. Reviewed-by: Gary Guo Signed-off-by: FUJITA Tomonori Link: https://msgid.link/20260924232530.446588-3-tomo@flapping.org Signed-off-by: Andreas Hindborg --- rust/helpers/helpers.c | 1 + rust/helpers/math.c | 8 +++ rust/kernel/Kconfig.test | 10 ++++ rust/kernel/time.rs | 105 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 rust/helpers/math.c diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 440fb7638e3c..08374b163774 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -73,6 +73,7 @@ #include "kunit.c" #include "list.c" #include "maple_tree.c" +#include "math.c" #include "mm.c" #include "mutex.c" #include "net/genetlink.c" diff --git a/rust/helpers/math.c b/rust/helpers/math.c new file mode 100644 index 000000000000..e2ee29bcce0f --- /dev/null +++ b/rust/helpers/math.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper u64 rust_helper_mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d) +{ + return mul_u64_add_u64_div_u64(a, b, c, d); +} diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test index e6a5c7a795f0..0087749995d2 100644 --- a/rust/kernel/Kconfig.test +++ b/rust/kernel/Kconfig.test @@ -83,4 +83,14 @@ config RUST_BITFIELD_KUNIT_TEST If unsure, say N. +config RUST_TIME_KUNIT_TEST + bool "KUnit tests for the Rust time API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust time API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + endif diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 60717f17e9ba..5ae377e70dd8 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -39,6 +39,10 @@ /// The number of nanoseconds per second. pub const NSEC_PER_SEC: i64 = bindings::NSEC_PER_SEC as i64; +/// The C side `MAX_JIFFY_OFFSET`, i.e. `((LONG_MAX >> 1) - 1)`. It is the upper +/// bound the kernel uses for a jiffies span, not a wait-forever value. +const MAX_JIFFY_OFFSET: isize = (isize::MAX >> 1) - 1; + /// The time unit of Linux kernel. One jiffy equals (1/HZ) second. pub type Jiffies = crate::ffi::c_ulong; @@ -554,6 +558,62 @@ pub fn as_millis_ceil(self) -> i64 { } } + /// Convert this span to a [`Delta`] suitable for use as a timeout. + /// + /// Unless the result saturates, the value is rounded up to the next whole + /// jiffy, so the resulting timeout is never shorter than `self`. + /// + /// A negative span saturates at zero jiffies, i.e. an immediate timeout. + /// + /// A span that does not fit saturates at the kernel's [`MAX_JIFFY_OFFSET`], + /// the upper bound for a jiffies span. That is a finite timeout, so a + /// saturated result can be shorter than the requested span. It is derived + /// from `long`, so only 32 bit can reach it, at about 12 days with `HZ=1000`. + /// + /// # Examples + /// + /// ``` + /// use kernel::time::Delta; + /// + /// // A negative span is an immediate timeout. + /// assert_eq!(Delta::from_millis(-1).to_jiffies_timeout().as_jiffies(), 0); + /// + /// // A span shorter than a jiffy still waits, i.e. the timeout is never + /// // shorter than the span. + /// assert!(Delta::from_nanos(1).to_jiffies_timeout().as_jiffies() >= 1); + /// ``` + /// + /// [`MAX_JIFFY_OFFSET`]: srctree/include/linux/jiffies.h + #[inline] + pub fn to_jiffies_timeout(self) -> Delta { + const HZ: u64 = bindings::HZ as u64; + + // The quotient `(nsecs * HZ + NSEC_PER_SEC - 1) / NSEC_PER_SEC` has to fit in + // `u64`; `nsecs * HZ` does not. With `HZ <= NSEC_PER_SEC` the numerator is at + // most `(nsecs + 1) * NSEC_PER_SEC - 1`, so the quotient is at most `nsecs`. + crate::static_assert!(HZ <= NSEC_PER_SEC as u64); + + // CAST: `i64::max()` makes the value non-negative, so the cast keeps it. + let nsecs = i64::max(self.as_nanos(), 0) as u64; + + // SAFETY: `mul_u64_add_u64_div_u64()` must not be called with a zero divisor, + // and its result must fit in `u64`. `NSEC_PER_SEC` is a non-zero constant, and + // the assertion above bounds the quotient by `nsecs`. + let jiffies = unsafe { + bindings::mul_u64_add_u64_div_u64( + nsecs, + HZ, + (NSEC_PER_SEC - 1) as u64, + NSEC_PER_SEC as u64, + ) + }; + + // CAST: `jiffies` is clamped to `MAX_JIFFY_OFFSET`, which is `<= isize::MAX`. + let jiffies = u64::min(jiffies, MAX_JIFFY_OFFSET as u64) as isize; + + Delta::::from_jiffies(jiffies) + } + /// Return `self % dividend` where `dividend` is in nanoseconds. /// /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is @@ -580,3 +640,48 @@ pub fn rem_nanos(self, dividend: i32) -> Self { } } } + +#[cfg(CONFIG_RUST_TIME_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_time)] +mod tests { + use super::*; + + #[test] + fn to_jiffies_timeout_converts() { + const HZ: isize = bindings::HZ as isize; + + // One second is exactly `HZ` jiffies, and the round-up must not add one. + assert_eq!(Delta::from_secs(1).to_jiffies_timeout().as_jiffies(), HZ); + + // One nanosecond more has to round up to the next whole jiffy. + assert_eq!( + Delta::from_nanos(NSEC_PER_SEC + 1) + .to_jiffies_timeout() + .as_jiffies(), + HZ + 1 + ); + } + + #[test] + fn to_jiffies_timeout_saturates() { + // The result never exceeds `MAX_JIFFY_OFFSET`. On 32 bit with `HZ=1000` this + // span is 2147483647 jiffies, so the clamp is what keeps it in range; on 64 + // bit it fits and the check holds for every possible return value. + let clamped = Delta::from_millis(i64::from(i32::MAX)).to_jiffies_timeout(); + assert!(clamped.as_jiffies() <= MAX_JIFFY_OFFSET); + + // `MAX_JIFFY_OFFSET` is derived from `long`, so only 32 bit can reach it. On + // 64 bit `i64::MAX` nanoseconds is about 292 years, which is 9223372036855 + // jiffies with `HZ=1000`, far below the limit. + #[cfg(not(CONFIG_64BIT))] + { + // An overlong span is clamped to `MAX_JIFFY_OFFSET`. + let overlong = Delta::from_nanos(i64::MAX).to_jiffies_timeout(); + assert_eq!(overlong.as_jiffies(), MAX_JIFFY_OFFSET); + } + + // A negative span is an immediate timeout, however long it is. + let negative = Delta::from_nanos(i64::MIN).to_jiffies_timeout(); + assert_eq!(negative.as_jiffies(), 0); + } +} From 2ea0119f72dba597aa8a98cbdb72c564bfc5cb38 Mon Sep 17 00:00:00 2001 From: Andreas Hindborg Date: Sun, 15 Feb 2026 21:36:04 +0100 Subject: [PATCH 23/91] rust: hrtimer: document handle based design rationale Add implementation notes explaining why the hrtimer abstraction uses a handle based approach. Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Reviewed-by: FUJITA Tomonori Link: https://msgid.link/20260215-hrtimer-docs-v1-1-bff6a6c0d923@kernel.org Signed-off-by: Andreas Hindborg --- rust/kernel/time/hrtimer.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs index 87aafbee0ce9..c0180b8ca5d3 100644 --- a/rust/kernel/time/hrtimer.rs +++ b/rust/kernel/time/hrtimer.rs @@ -402,6 +402,17 @@ //! ``` //! //! [`Arc`]: kernel::sync::Arc +// Implementation details +// +// The reasoning for adopting a handle based approach: +// - If we explicitly drop the target of a timer callback in the timer callback, we +// may get a dangling reference. +// - If the callback owns the last reference to the target, target may be dropped +// in non-sleepable context when the callback is finished. +// - When dropping an object that is the target of an armed timer, we may drop +// fields accessed by the timer callback before we cancel the timer (drop order). +// +// By using a handle, we can make the handle own the callback target and avoid these problems. use super::{ClockId, Delta, Instant}; use crate::{prelude::*, types::Opaque}; From d190aa209416f6d6d7bdaffffaaab2af5d3ccdec Mon Sep 17 00:00:00 2001 From: Andreas Hindborg Date: Fri, 5 Jun 2026 12:56:31 +0200 Subject: [PATCH 24/91] rust: alloc: allow different error types in `KBox::pin_slice` Previously, `KBox::pin_slice` required the initializer error type to match the return error type via `E: From`. This prevented using infallible initializers like `new_mutex!` inside `pin_slice`, because `Infallible` does not implement `From`. Introduce a separate type parameter `E2` for the initializer error type and require `E: From` and `E: From` instead. This allows the initializer to return a different error type that can be converted into the final error type, enabling use of infallible pin initializers in fallible allocation contexts. Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/.E2.9C.94.20Constructing.20Mutex.20from.20PinInit.3CT.2C.20Error.3E Co-developed-by: Benno Lossin Signed-off-by: Benno Lossin Signed-off-by: Andreas Hindborg Link: https://patch.msgid.link/20260605-pin-slice-init-v2-1-b197e84040ba@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kbox.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index c63d6acdbb6f..2c929eb1c3d6 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -345,7 +345,7 @@ pub fn pin(x: T, flags: Flags) -> Result>, AllocError> /// } /// /// // Allocate a boxed slice of 10 `Example`s. - /// let s = KBox::pin_slice( + /// let s = KBox::pin_slice::<_, _, Error, _>( /// | _i | Example::new(), /// 10, /// GFP_KERNEL @@ -355,15 +355,16 @@ pub fn pin(x: T, flags: Flags) -> Result>, AllocError> /// assert_eq!(s[3].d.lock().a, 20); /// # Ok::<(), Error>(()) /// ``` - pub fn pin_slice( + pub fn pin_slice( mut init: Func, len: usize, flags: Flags, ) -> Result>, E> where Func: FnMut(usize) -> Item, - Item: PinInit, + Item: PinInit, E: From, + E: From, { let mut buffer = super::Vec::::with_capacity(len, flags)?; for i in 0..len { From 6e8339118040f24ebcc21db22c9c1b7398b7e39e Mon Sep 17 00:00:00 2001 From: Andreas Hindborg Date: Tue, 29 Sep 2026 17:18:24 +0200 Subject: [PATCH 25/91] rust: alloc: add `NumaNode::id()` accessor `NumaNode` wraps an `i32` but offers no way to read it back. Add a trivial `id()` getter so callers that need to pass the node id to a C binding can recover the underlying `c_int` without reaching into the private field. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Signed-off-by: Andreas Hindborg Link: https://patch.msgid.link/20260929-numa-node-id-v2-1-fdd2c0a49412@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 21067bde6860..68d80f3fa6c8 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -130,6 +130,12 @@ pub fn new(node: i32) -> Result { } Ok(Self(node)) } + + /// Returns the raw NUMA node identifier as an `i32`. + #[inline] + pub fn id(&self) -> i32 { + self.0 + } } /// Specify necessary constant to pass the information to Allocator that the caller doesn't care From 7b5ba47d845f35b04937748a191a1772ffc1f288 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Tue, 13 Jan 2026 19:29:49 +0000 Subject: [PATCH 26/91] rust: bindings: fix VM_MAYSHARE value Commit 2b6a3f061f11 ("mm: declare VMA flags by bit") added a number of overrides to the rust bindings helper as the change performed in the commit caused rust bindgen to not be able to automatically determine values of VMA flags. However a typo was introduced leaving VM_MAYSHARE to be incorrectly interpreted as VM_MAYEXEC in rust code. This patch fixes the typo. Fixes: 2b6a3f061f11 ("mm: declare VMA flags by bit") Reported-by: Chris Mason Closes: https://lore.kernel.org/all/20260113185142.254821-1-clm@meta.com/ Cc: stable@vger.kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Alice Ryhl Reviewed-by: wang lian Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260113192949.689834-1-lorenzo.stoakes@oracle.com [ Reworded to improve title and to fix typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/bindings/bindings_helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 4b31aa7f432f..c706dda5b5ca 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -138,7 +138,7 @@ const vm_flags_t RUST_CONST_HELPER_VM_SHARED = VM_SHARED; const vm_flags_t RUST_CONST_HELPER_VM_MAYREAD = VM_MAYREAD; const vm_flags_t RUST_CONST_HELPER_VM_MAYWRITE = VM_MAYWRITE; const vm_flags_t RUST_CONST_HELPER_VM_MAYEXEC = VM_MAYEXEC; -const vm_flags_t RUST_CONST_HELPER_VM_MAYSHARE = VM_MAYEXEC; +const vm_flags_t RUST_CONST_HELPER_VM_MAYSHARE = VM_MAYSHARE; const vm_flags_t RUST_CONST_HELPER_VM_PFNMAP = VM_PFNMAP; const vm_flags_t RUST_CONST_HELPER_VM_IO = VM_IO; const vm_flags_t RUST_CONST_HELPER_VM_DONTCOPY = VM_DONTCOPY; From f9a6960c7b27c03becd50a2a7620273d80e4040d Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Wed, 30 Sep 2026 14:36:45 +0200 Subject: [PATCH 27/91] rust: bindings: fix `VM_MAYSHARE` blocklist entry Commit 2b6a3f061f11 ("mm: declare VMA flags by bit") added constant helpers for the VMA flags and blocklist entries to suppress their direct bindings. However, it listed `VM_MAYEXEC` twice and omitted `VM_MAYSHARE`. Thus fix it. Fixes: 2b6a3f061f11 ("mm: declare VMA flags by bit") Cc: stable@vger.kernel.org Assisted-by: LLM Reviewed-by: Alice Ryhl Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Vishal Moola (Fractile) Link: https://patch.msgid.link/20260930123645.302318-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/bindgen_parameters | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index 8402b0c93545..93e622aaaf78 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -50,7 +50,7 @@ --blocklist-item VM_MAYREAD --blocklist-item VM_MAYWRITE --blocklist-item VM_MAYEXEC ---blocklist-item VM_MAYEXEC +--blocklist-item VM_MAYSHARE --blocklist-item VM_PFNMAP --blocklist-item VM_IO --blocklist-item VM_DONTCOPY From 158e85db3375c4b0b66223dee7aae16f2357aada Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 28 Sep 2026 18:32:19 +0100 Subject: [PATCH 28/91] rust: pin-init: internal: init: allow reborrow of `Pin<&mut T>` bindings Use `let mut` for created bindings of initialized field, so `Pin<&mut T>` bindings may be reborrowed in subsequent initializers or code blocks. Without `let mut`, attempt to reborrow can produce error (and also a confusing fix): error[E0596]: cannot borrow `gpu` as mutable, as it is not declared as mutable --> drivers/gpu/nova-core/driver.rs:129:17 | 129 | gpu.as_mut().run_selftests(pdev); | ^^^ cannot borrow as mutable | help: consider changing this to be mutable | 119 | mut gpu <- Gpu::new( | +++ error: aborting due to 1 previous error Reported-by: Danilo Krummrich Closes: https://lore.kernel.org/nova-gpu/DLQXWXRP3P1J.28SLFOJJQ2EDK@kernel.org/ Reviewed-by: Alexandre Courbot Tested-by: Danilo Krummrich Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260928-dev-pin-init-sync-v1-1-e591c8e9cf0e@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index c5b5a0ce5c4e..4e77c7af86f9 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -446,7 +446,8 @@ fn init_fields(fields: &Punctuated, pinned: bool) - // Allow `non_snake_case` since the same warning is going to be reported for the // struct field. #[allow(unused_variables, non_snake_case)] - let #ident = #guard.let_binding(); + // Include `mut` so that `Pin<&mut T>` bindings can be reborrowed via `.as_mut()`. + let mut #ident = #guard.let_binding(); }, Member::Unnamed(_) => quote!(), }; From d99668f5ed5839a1a8f061dd5a47ab5d0c7dac74 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 28 Sep 2026 18:32:20 +0100 Subject: [PATCH 29/91] rust: pin-init: internal: init: improve diagnostics from field checks Use span of the field name for generated alignment and field duplication checks. This avoids pointing diagnostics to the full macro invocation, but just the part of it going wrong. Before the change: error[E0062]: field `0` specified more than once --> tests/ui/compile-fail/init/tuple_duplicate_field.rs:7:37 | 7 | let _ = pin_init!(Tuple { 0: 1, 0: 2, 1: 3 }); | ------------------------^------------ | | | | | used more than once | first use of `0` After the change: error[E0062]: field `0` specified more than once --> tests/ui/compile-fail/init/tuple_duplicate_field.rs:7:37 | 7 | let _ = pin_init!(Tuple { 0: 1, 0: 2, 1: 3 }); | - ^ used more than once | | | first use of `0` Link: https://patch.msgid.link/20260928-dev-pin-init-sync-v1-2-e591c8e9cf0e@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 65 +++++++++++++++++++----------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 4e77c7af86f9..f6f3e2f8cda6 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -478,42 +478,61 @@ fn make_field_check( init_kind: InitKind, path: &Path, ) -> TokenStream { - let field_attrs: Vec<_> = fields + let align_checks: TokenStream = fields .iter() - .filter_map(|f| f.kind.member().map(|_| &f.attrs)) + .filter_map(|f| { + let member = f.kind.member()?; + let span = member.span().resolved_at(Span::mixed_site()); + let attrs = &f.attrs; + + Some(quote_spanned! {span => + // Create references to ensure that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` for value-initialization case has the same requirement. + #(#attrs)* + let _ = &(*slot).#member; + }) + }) + .collect(); + + let fake_field_init: TokenStream = fields + .iter() + .filter_map(|f| { + let member = f.kind.member()?; + let span = member.span().resolved_at(Span::mixed_site()); + let attrs = &f.attrs; + + Some(quote_spanned! {span => + #(#attrs)* + #member: loop {}, + }) + }) .collect(); - let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.member()).collect(); let zeroing_trailer = match init_kind { InitKind::Normal => None, InitKind::Zeroing => Some(quote! { ..::core::mem::zeroed() }), }; + let field_dup_checks = quote_spanned! { Span::mixed_site() => + // If the zeroing trailer is not present, this checks that all fields have been + // mentioned exactly once. If the zeroing trailer is present, all missing fields will be + // zeroed, so this checks that all fields have been mentioned at most once. The use of + // struct initializer will still generate very natural error messages for any misuse. + ::core::ptr::write(slot, #path { + #fake_field_init + #zeroing_trailer + }) + }; + quote_spanned! { Span::mixed_site() => #[allow(unreachable_code)] // We use unreachable code to perform field checks. They're still checked by the compiler. // SAFETY: this code is never executed. let _ = || unsafe { - // Create references to ensure that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` for value-initialization case has the same requirement. - #( - #(#field_attrs)* - let _ = &(*slot).#field_name; - )* - - // If the zeroing trailer is not present, this checks that all fields have been - // mentioned exactly once. If the zeroing trailer is present, all missing fields will be - // zeroed, so this checks that all fields have been mentioned at most once. The use of - // struct initializer will still generate very natural error messages for any misuse. - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: loop {}, - )* - #zeroing_trailer - }) + #align_checks + #field_dup_checks }; } } From b2cb983f9d59ddc0ede2a4976443fda6297778af Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 28 Sep 2026 18:32:21 +0100 Subject: [PATCH 30/91] rust: pin-init: internal: init: use span of `<-` or `:` for the init/write call. Currently we use the full span of the field initializer as the span of the generated init call. This produces overlapping spans for type mismatch errors, which is hard to read. Use the span of `<-` or `:` for the method name, which wll cause type mismatch errors or trait bound not implemented error to be appear from the `<-` or `:` token, which declutter the output a bit and make it easier to read. Before the change: error[E0277]: `impl pin_init::PinInit` cannot be used to movably initialize `Bar` with error `_` --> tests/ui/compile-fail/init/invalid_init.rs:19:16 | 19 | bar <- Bar::new(), | -------^^^^^^^^^^ | | | | | the trait `Init` is not implemented for `impl pin_init::PinInit` | required by a bound introduced by this call | = note: if your type implements `PinInit` but not `Init`, you might be forgetting a `#[pin]` annotation on fields After the change: error[E0277]: `impl pin_init::PinInit` cannot be used to movably initialize `Bar` with error `_` --> tests/ui/compile-fail/init/invalid_init.rs:19:16 | 19 | bar <- Bar::new(), | -- ^^^^^^^^^^ the trait `Init` is not implemented for `impl pin_init::PinInit` | | | required by a bound introduced by this call | = note: if your type implements `PinInit` but not `Init`, you might be forgetting a `#[pin]` annotation on fields Suggested-by: Benno Lossin Link: https://patch.msgid.link/20260928-dev-pin-init-sync-v1-3-e591c8e9cf0e@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 38 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index f6f3e2f8cda6..6605e04d49df 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -101,7 +101,7 @@ enum InitializerKind { }, Init { member: Member, - _left_arrow_token: Token![<-], + left_arrow_token: Token![<-], value: Expr, }, Code { @@ -415,24 +415,40 @@ fn init_fields(fields: &Punctuated, pinned: bool) - // `mixed_site` ensures that the guard is not accessible to the user-controlled code. let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); - let full_span = kind.span(); + let full_span = kind.span().resolved_at(Span::mixed_site()); let init = match kind { InitializerKind::Value { value, .. } => { - let value = value + let (colon_span, value) = value .as_ref() - .map(|(_, value)| quote!(#value)) - .unwrap_or_else(|| quote!(#member)); + .map(|(colon_token, value)| (colon_token.span(), quote!(#value))) + .unwrap_or_else(|| (member.span(), quote!(#member))); + + // Use `:` as the span of the method name, so the type requirement appears to come + // from `:`. + let write = + format_ident!("write", span = colon_span.resolved_at(Span::mixed_site())); quote_spanned! { full_span => #(#attrs)* - let mut #guard = #slot.write(#value); + let mut #guard = #slot.#write(#value); } } - InitializerKind::Init { value, .. } => { + InitializerKind::Init { + value, + left_arrow_token, + .. + } => { + // Use `<-` as the span of the method name, so the trait bound appears to come from + // `<-`. + let init = format_ident!( + "init", + span = left_arrow_token.span().resolved_at(Span::mixed_site()) + ); + quote_spanned! { full_span => #(#attrs)* - let mut #guard = #slot.init(#value)?; + let mut #guard = #slot.#init(#value)?; } } InitializerKind::Code { .. } => unreachable!(), @@ -690,7 +706,7 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { if lh.peek(Token![<-]) { Ok(Self::Init { member, - _left_arrow_token: input.parse()?, + left_arrow_token: input.parse()?, value: input.parse()?, }) } else if lh.peek(Token![:]) { @@ -819,11 +835,11 @@ fn to_tokens(&self, tokens: &mut TokenStream) { } Self::Init { member, - _left_arrow_token, + left_arrow_token, value, } => { member.to_tokens(tokens); - _left_arrow_token.to_tokens(tokens); + left_arrow_token.to_tokens(tokens); value.to_tokens(tokens); } Self::Code { From c82c75ae11fae66cf070562f9b5241a4663f30cb Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 24 Sep 2026 01:50:51 +0100 Subject: [PATCH 31/91] rust: doctest: trim function name for reproducibility Currently rustdoc will generate function names like "_doctest_main__home_gary_Projects_linux_rust_kernel_io_rs_824_0" for a doctest located at rust/kernel/io.rs:824, when building with separate outdir using `O=`. This creates overlong symbol names and is also not reproducible. [ `rustdoc` already remaps these paths when we use Rust >= 1.95, thus this fix is needed for older toolchains. - Miguel ] Fix it by doing a custom remapping to trim it to something like `_doctest_main_rust_kernel_io_rs_824_0`. Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260924005052.2805129-1-gary@kernel.org [ Added `TODO` comment about the minimum version bump. Reworded title and comment for typos. - Miguel ] Signed-off-by: Miguel Ojeda --- scripts/rustdoc_test_builder.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/rustdoc_test_builder.rs b/scripts/rustdoc_test_builder.rs index df864437cef7..0361e22853f2 100644 --- a/scripts/rustdoc_test_builder.rs +++ b/scripts/rustdoc_test_builder.rs @@ -72,6 +72,13 @@ fn main() { // Figure out a smaller test name based on the generated function name. let name = rustdoc_function_name.split_once("_rust_kernel_").unwrap().1; + // The `rustdoc` function name can include the absolute path when building with `O=` which is + // undesirable and creates overlong symbol names. Remap it to use a relative path. + // TODO: Remove when bumping the minimum version since with Rust >= 1.95 we already do the + // path remapping via the `rustdoc` `--remap-path-prefix` flag. + let trimmed_function_name = format!("_doctest_main_rust_kernel_{name}"); + let body = body.replace(&rustdoc_function_name, &trimmed_function_name); + let path = format!("rust/test/doctests/kernel/{name}"); std::fs::write(path, body.as_bytes()).unwrap(); From 499b8f8a5d0401a6ab83b346a307f923640a024a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 21 Sep 2026 12:35:46 +0100 Subject: [PATCH 32/91] rust: proc-macro2: enable `proc_macro_span` feature The `Span::join` function is very important in generating useful diagnostics. When invoking `syn::Spanned::span`, it attempts to return an overall span that covers the whole expression. If the `join` function is unavailable, however, only the span of the first token is generated. This causes worse diagnostics quality for all macros, notably pin-init, when compared to the pin-init's own diagnostics test suite with the feature enabled. For example, this diagnostic covers only part of the text where replacement is suggested: error: expected nothing or `..Zeroable::init_zeroed()`. --> tests/ui/compile-fail/zeroable/invalid_spread.rs:15:11 | 15 | ..MyZeroable::init_zeroed() | ^^^^^^^^^^ In the upcoming pin-init self-reference series, spans of full types or expressions are used quite heavily to make diagnostics not confusing. Thus, enable the `proc_macro_span` feature. This does make use of the unstable `Span::join` function, but this is for producing diagnostics only, so we do not have a hard reliance on the feature. A very small tweak of proc-macro2 vendored code is needed to make the code compile for 1.85. Link: https://github.com/rust-lang/rust/issues/54725 Link: https://github.com/dtolnay/proc-macro2/pull/542 Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260921115708.4009754-1-gary@kernel.org/ [ Updated example diagnostics, removed unused PathBuf import. - Gary ] Signed-off-by: Gary Guo --- rust/Makefile | 1 + rust/proc-macro2/README.md | 3 ++- rust/proc-macro2/probe/proc_macro_span.rs | 9 --------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index da1a7409d984..5ee9c30c324a 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -100,6 +100,7 @@ zerocopy-envs := \ proc_macro2-cfgs := \ feature="proc-macro" \ wrap_proc_macro \ + proc_macro_span \ $(if $(call rustc-min-version,108800),proc_macro_span_file proc_macro_span_location) proc_macro2-flags := \ diff --git a/rust/proc-macro2/README.md b/rust/proc-macro2/README.md index af044fee4f59..519d01112a02 100644 --- a/rust/proc-macro2/README.md +++ b/rust/proc-macro2/README.md @@ -4,7 +4,8 @@ These source files come from the Rust `proc-macro2` crate, version 1.0.101 (released 2025-08-16), hosted in the repository, licensed under "Apache-2.0 OR MIT" and only modified to add the SPDX license -identifiers and to remove the `unicode-ident` dependency. +identifiers, to remove the `unicode-ident` dependency, and to build +with 1.85 with nightly features enabled. For copyright details, please see: diff --git a/rust/proc-macro2/probe/proc_macro_span.rs b/rust/proc-macro2/probe/proc_macro_span.rs index 892a7eb3e5a0..765bd47dbecd 100644 --- a/rust/proc-macro2/probe/proc_macro_span.rs +++ b/rust/proc-macro2/probe/proc_macro_span.rs @@ -10,7 +10,6 @@ use core::ops::{Range, RangeBounds}; use proc_macro::{Literal, Span}; -use std::path::PathBuf; pub fn byte_range(this: &Span) -> Range { this.byte_range() @@ -32,14 +31,6 @@ pub fn column(this: &Span) -> usize { this.column() } -pub fn file(this: &Span) -> String { - this.file() -} - -pub fn local_file(this: &Span) -> Option { - this.local_file() -} - pub fn join(this: &Span, other: Span) -> Option { this.join(other) } From e91836593fbfb0014c290e9a0bbd86d0b98c1d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thi=C3=A9baud=20Weksteen?= Date: Thu, 17 Sep 2026 15:11:47 +1000 Subject: [PATCH 33/91] rust: macros: allow non-ASCII characters in `authors` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Module authors may have non-ASCII characters in their names. In C, `MODULE_AUTHOR` allows arbitrary string literals which are emitted as raw UTF-8 bytes into the `.modinfo` section, and multiple in-tree modules use non-ASCII author names. Originally, the single `author` field permitted arbitrary string literals including non-ASCII characters. When support for multiple authors was introduced in commit 38559da6afb2 ("rust: module: introduce `authors` key"), it reused the `expect_string_array` helper that had originally been added for module aliases. Because that helper enforced an ASCII check, `authors` inadvertently became restricted to ASCII-only string literals. Later, when the macro parsing was rewritten to use `syn` in commit c578ad703ae9 ("rust: macros: use `syn` to parse `module!` macro"), this restriction was carried over by using AsciiLitStr in `authors` type. Change the element type of `authors` in `ModuleInfo` from `AsciiLitStr` to `LitStr` so that UTF-8 author names are permitted. Fixes: 38559da6afb2 ("rust: module: introduce `authors` key") Signed-off-by: Thiébaud Weksteen Acked-by: Petr Pavlu Link: https://patch.msgid.link/20260917051147.98775-1-tweek@google.com Signed-off-by: Miguel Ojeda --- rust/macros/lib.rs | 2 +- rust/macros/module.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 24f96feaeb34..ea2e232b61b1 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -129,7 +129,7 @@ /// # Supported argument types /// - `type`: type which implements the [`Module`] trait (required). /// - `name`: ASCII string literal of the name of the kernel module (required). -/// - `authors`: array of ASCII string literals of the authors of the kernel module. +/// - `authors`: array of string literals of the authors of the kernel module. /// - `description`: string literal of the description of the kernel module. /// - `license`: ASCII string literal of the license of the kernel module (required). /// - `alias`: array of ASCII string literals of the alias names of the kernel module. diff --git a/rust/macros/module.rs b/rust/macros/module.rs index bc7027f8dbb2..0a7b9e24c960 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -369,7 +369,7 @@ pub(crate) struct ModuleInfo { type_: Type, license: AsciiLitStr, name: AsciiLitStr, - authors: Option>, + authors: Option>, description: Option, alias: Option>, firmware: Option>, From 951d09ca450ca76a695016ea76c3fa653bd20439 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 1 Oct 2026 21:31:03 +0900 Subject: [PATCH 34/91] rust: bug: pass TAINT_WARN to warn_slowpath_fmt() on UML On UML, `warn_flags!` calls `warn_slowpath_fmt()` directly. Its third argument is a taint number, but `warn_on!` passes its bug flags there. As a result, `add_taint()` gets 2304 instead of 9 (`TAINT_WARN`). The warning is not recorded: `/proc/sys/kernel/tainted` stays 0 after `warn_on!` fires, where it should be 512. It also causes an out-of-bounds write in `add_taint()`. Pass `TAINT_WARN` directly, as C does on UML. Cc: stable@vger.kernel.org Fixes: dff64b072708 ("rust: Add warn_on macro") Signed-off-by: FUJITA Tomonori Link: https://patch.msgid.link/20261001123103.2198104-1-tomo@flapping.org Signed-off-by: Miguel Ojeda --- rust/kernel/bug.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index 3566f0234ca4..af8eba4563de 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -85,6 +85,7 @@ macro_rules! warn_flags { ($file:expr, $flags:expr) => { if false { _ = $file; + _ = $flags; } // SAFETY: It is always safe to call `warn_slowpath_fmt()` @@ -93,7 +94,7 @@ macro_rules! warn_flags { $crate::bindings::warn_slowpath_fmt( $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), line!() as $crate::ffi::c_int, - $flags as $crate::ffi::c_uint, + $crate::bindings::TAINT_WARN, ::core::ptr::null(), ); } From cd56a1c858ff34cc8b71674a6bdf2f7a899abf67 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Wed, 16 Sep 2026 23:09:52 +0900 Subject: [PATCH 35/91] rust: bug: add a KUnit test for warn_on! The test runs warn_on!(false) and warn_on!(true) inside a KUnit warning suppression block, then checks that one warning was counted. The counter is incremented by the kernel warning path, not by warn_on! itself, so a count of one means the warning was really reported. warn_flags! has a different definition per architecture. The test does not look at any of them, so it runs on every architecture that supports Rust. The new CONFIG_RUST_BUG_KUNIT_TEST depends on BUG. With CONFIG_BUG=n, warn_on! does nothing and no warning is counted. Signed-off-by: FUJITA Tomonori Link: https://patch.msgid.link/20260916140952.3290924-1-tomo@flapping.org [ Added newlines for clarity. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/Kconfig.test | 11 +++++++++++ rust/kernel/bug.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test index e6a5c7a795f0..966b07872ba1 100644 --- a/rust/kernel/Kconfig.test +++ b/rust/kernel/Kconfig.test @@ -83,4 +83,15 @@ config RUST_BITFIELD_KUNIT_TEST If unsure, say N. +config RUST_BUG_KUNIT_TEST + bool "KUnit tests for the Rust BUG/WARN functionality" if !KUNIT_ALL_TESTS + depends on BUG + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust BUG/WARN functionality. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + endif diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index af8eba4563de..6abed2c055b1 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -153,3 +153,33 @@ macro_rules! warn_on { cond }}; } + +#[cfg(CONFIG_RUST_BUG_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_bug)] +mod tests { + // The counter is incremented by the kernel warning path, not by `warn_on!` + // itself. A count of one means the warning was really reported. + #[test] + fn test_warn_on() { + // SAFETY: `kunit_get_current_test()` is always safe to call (it has + // fallbacks for when no KUnit test is running). + let test = unsafe { bindings::kunit_get_current_test() }; + + // SAFETY: This function runs only as a KUnit test case, so `test` is a + // valid pointer to the running test. + let handle = unsafe { bindings::kunit_start_suppress_warning(test) }; + + assert!(!warn_on!(false)); + assert!(warn_on!(true)); + + // SAFETY: `kunit_suppressed_warning_count()` accepts any value returned by + // `kunit_start_suppress_warning()`. + let suppressed_count = unsafe { bindings::kunit_suppressed_warning_count(handle) }; + + // SAFETY: `test` is valid as above. `kunit_end_suppress_warning()` accepts any + // value returned by `kunit_start_suppress_warning()`. + unsafe { bindings::kunit_end_suppress_warning(test, handle) }; + + assert_eq!(suppressed_count, 1); + } +} From dd47e7d8672b64ae8680f8abefbe4b5c55895cda Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 10 Sep 2026 18:05:42 +0100 Subject: [PATCH 36/91] rust: bitfield: remove unnecessary `#[allow]` The `#[allow(non_camel_case_types)]` was there because register names can be commonly SCREAMING_CASE and thus not CamelCase. However, when a bitfield type is created by itself, this justification does not apply. Thus, remove it. Users that want to use non-standard styles can add the `#[allow]` themselves. For example, the invocation of `bitfield!` inside `register!` macro carry this annotation. Signed-off-by: Gary Guo Acked-by: Alexandre Courbot Link: https://patch.msgid.link/20260910170542.92010-1-gary@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/bitfield.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index a0d089423f21..861522c8cf3e 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -295,7 +295,6 @@ macro_rules! bitfield { $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } ) => { $crate::bitfield!(@core - #[allow(non_camel_case_types)] $(#[$attr])* $vis $name $storage ); $crate::bitfield!(@fields $vis $name $storage { $($fields)* }); From 948440b3f429f84680db58b4bf529b1ed2a20e15 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 10 Jun 2026 12:42:22 +0100 Subject: [PATCH 37/91] rust: mem: add `Align` type Rust's `repr(align())` only works with integers. Add an `Align` type so it can also be specified via const generics. Signed-off-by: Gary Guo Reviewed-by: Andreas Hindborg Link: https://patch.msgid.link/20260610-const-align-v2-1-ccd12f0aedb7@garyguo.net [ Adjusted the title of the new module to match the one used in patch "rust: mem: add `transmute` with deferred size check" (landing via driver-core), which is also closer to the upstream standard library title. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 + rust/kernel/mem.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 rust/kernel/mem.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 4d5c96ddc49c..7225abc64084 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -98,6 +98,7 @@ pub mod kunit; pub mod list; pub mod maple_tree; +pub mod mem; pub mod miscdevice; pub mod mm; pub mod module; diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs new file mode 100644 index 000000000000..33676760bee2 --- /dev/null +++ b/rust/kernel/mem.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Basic utilities for dealing with memory, values, and types. + +// TODO: `Alignment` should be moved here as well, Rust moved it upstream in 1.96. +pub use crate::ptr::Alignment; + +/// Trait indicating that [`Align`] is a valid alignment. +pub trait ValidAlign { + #[doc(hidden)] + type Repr: Copy; +} + +/// Zero-sized type that provides the alignment specified via the generic parameter. +/// +/// # Examples +/// +/// ``` +/// # use kernel::mem::Align; +/// const ALIGN: usize = 128; +/// struct MyStruct { +/// _align: Align, +/// } +/// assert_eq!(align_of::(), ALIGN); +/// ``` +#[repr(transparent)] +#[derive(Default, Clone, Copy)] +pub struct Align([::Repr; 0]) +where + Self: ValidAlign; + +impl Align +where + Self: ValidAlign, +{ + /// Create a new [`Align`]. + #[inline] + pub const fn new() -> Self { + Align([]) + } +} + +macro_rules! impl_align { + () => {}; + ($a:literal $($rest:literal)*) => { + const _: () = { + #[repr(align($a))] + #[derive(Clone, Copy)] + pub struct Repr; + + impl ValidAlign for Align<$a> { + type Repr = Repr; + } + }; + + impl_align!($($rest)*); + } +} + +impl_align!(1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536); + +impl core::fmt::Debug for Align +where + Self: ValidAlign, +{ + #[inline] + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "Align<{N}>") + } +} From 547948954f1ca78d3bb0b38201d205a16dc52d56 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 4 Sep 2026 09:31:09 +0000 Subject: [PATCH 38/91] rust: kbuild: add `no_io_statics` to `core-cfgs` In upstream Rust, PR #155625 moved `std::io::Error` into `core` under the `core_io` feature gate. To support decoding and formatting raw OS errors (`RawOsError`) from within `core` without depending on OS facilities, `core` introduced a global atomic pointer: static OS_FUNCTIONS: atomic::AtomicPtr At runtime, `std` registers its OS error vtable with this pointer via `set_functions()`, which uses `core::sync::atomic` to modify the value of this atomic variable. To ensure that `core` does not introduce usage of `core::sync::atomic` into the kernel image, set the `no_io_statics` cfg, which removes this logic from the build. Note that changing the `OS_FUNCTIONS` global is only possible via the unstable `from_raw_os_error_with_functions()` function from `core`, which is called by `std`. There is no way for us to invoke it from the kernel, as we do not use `std`. [ Pass `-Aunused_variables` to `core` to avoid: error: unused variable: `f` --> .../library/core/src/io/error/os_functions.rs:9:36 | 9 | pub(super) unsafe fn set_functions(f: &'static OsFunctions) { | ^ help: if this is intentional, prefix it with an underscore: `_f` | = note: `-D unused-variables` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(unused_variables)]` I sent a PR upstream [1] to clean it up and to add a test to check that `core` builds are warning-free with the `cfg` enabled, similar to what I added for other `cfg`s. Link: https://github.com/rust-lang/rust/pull/163633 [1] - Miguel ] Link: https://github.com/rust-lang/rust/pull/155625 Link: https://github.com/rust-lang/rust/issues/154046 Link: https://github.com/rust-lang/libs-team/issues/755 Signed-off-by: Alice Ryhl Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260904-no-io-statics-v1-1-a2386eb0d255@google.com Signed-off-by: Miguel Ojeda --- rust/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/Makefile b/rust/Makefile index da1a7409d984..152649f209d8 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -75,7 +75,8 @@ endif cfgs-to-flags = $(patsubst %,--cfg='%',$1) core-cfgs := \ - no_fp_fmt_parse + no_fp_fmt_parse \ + no_io_statics core-edition := $(if $(call rustc-min-version,108700),2024,2021) @@ -85,6 +86,7 @@ core-skip_flags := \ core-flags := \ --edition=$(core-edition) \ + -Aunused_variables \ $(call cfgs-to-flags,$(core-cfgs)) zerocopy-cfgs := \ From 5ec807292c55eff56387f32d5b2a0a81f2926b81 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 11 Aug 2026 14:55:55 +0100 Subject: [PATCH 39/91] rust: macros: vtable: hide generated `HAS_` constants for required methods These constants have to be generated as the implementation-site `#[vtable]` invocation has no way of knowing whether the methods are required or provided-but-overridden. However, these constants will have a fixed value so shouldn't be used. Thus, hide them. Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260811-vtable_macro-v1-1-50c5aab3c211@garyguo.net [ Reworded for typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/macros/vtable.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs index be9a5ed8abe5..1dfc40eb53f8 100644 --- a/rust/macros/vtable.rs +++ b/rust/macros/vtable.rs @@ -9,7 +9,10 @@ Ident, TokenStream, // }; -use quote::ToTokens; +use quote::{ + quote, + ToTokens, // +}; use syn::{ parse_quote, Error, @@ -56,12 +59,18 @@ fn handle_trait(mut item: ItemTrait) -> Result { // We don't know on the implementation-site whether a method is required or provided // so we have to generate a const for all methods. + // However, hide it for required methods as it will always be true. + let doc = if fn_item.default.is_some() { + let comment = + format!("Indicates if the `{name}` method is overridden by the implementor."); + quote!(#[doc = #comment]) + } else { + quote!(#[doc(hidden)]) + }; let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); - let comment = - format!("Indicates if the `{name}` method is overridden by the implementor."); gen_items.push(parse_quote! { #(#cfg_attrs)* - #[doc = #comment] + #doc const #gen_const_name: bool = false; }); } From a52a7596c376ac7bce5a5322c526d96e2780ad17 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 11 Aug 2026 14:55:56 +0100 Subject: [PATCH 40/91] rust: macros: vtable: add `#[optional]` attribute For most users of the `#[vtable]` macro, the optional methods do not have a Rust-side default implementation; a `NULL` pointer is inserted into the vtable and the C subsystem does things differently when it sees `NULL`. Streamline this use case by providing a `#[optional]` attribute that does what the `build_error!(VTABLE_DEFAULT_ERROR)` boilerplate does. Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260811-vtable_macro-v1-2-50c5aab3c211@garyguo.net [ Rewrapped docs. Reworded message, comment and error for typos. Rebased. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/macros/lib.rs | 41 +++++++++++++---------------------------- rust/macros/vtable.rs | 26 +++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index ea2e232b61b1..b84db5c6bdc4 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -156,25 +156,15 @@ pub fn module(input: TokenStream) -> TokenStream { /// associated constant bool for each method in the trait that is set to true if /// the implementer has overridden the associated method. /// -/// For a trait method to be optional, it must have a default implementation. -/// This is also the case for traits annotated with `#[vtable]`, but in this -/// case the default implementation will never be executed. The reason for this -/// is that the functions will be called through function pointers installed in -/// C side vtables. When an optional method is not implemented on a `#[vtable]` -/// trait, a `NULL` entry is installed in the vtable. Thus the default -/// implementation is never called. Since these traits are not designed to be -/// used on the Rust side, it should not be possible to call the default -/// implementation. This is done to ensure that we call the vtable methods -/// through the C vtable, and not through the Rust vtable. Therefore, the -/// default implementation should call `build_error!`, which prevents -/// calls to this function at compile time: -/// -/// ```compile_fail -/// # // Intentionally missing `use`s to simplify `rusttest`. -/// build_error!(VTABLE_DEFAULT_ERROR) -/// ``` -/// -/// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`]. +/// For a trait method to be optional for normal traits, it must have a default +/// implementation. However, for many users of `#[vtable]`, the functions will +/// be called through function pointers installed in C side vtables. When an +/// optional method is not implemented on a `#[vtable]` trait, a `NULL` entry is +/// installed in the vtable; thus the default implementation is never called. If +/// this is the case, `#[optional]` can be applied on the optional method. +/// Methods annotated as such do not need to be implemented nor need a default +/// implementation. Calling these methods on types that do not implement them +/// will fail to build using the `build_error!` mechanism. /// /// This macro should not be used when all functions are required. /// @@ -187,7 +177,6 @@ pub fn module(input: TokenStream) -> TokenStream { /// # Examples /// /// ``` -/// use kernel::error::VTABLE_DEFAULT_ERROR; /// use kernel::prelude::*; /// /// # struct LocalModule; @@ -204,13 +193,11 @@ pub fn module(input: TokenStream) -> TokenStream { /// // Declares a `#[vtable]` trait /// #[vtable] /// pub trait Operations: Send + Sync + Sized { -/// fn foo(&self) -> Result<()> { -/// build_error!(VTABLE_DEFAULT_ERROR) -/// } +/// #[optional] +/// fn foo(&self) -> Result<()>; /// -/// fn bar(&self) -> Result<()> { -/// build_error!(VTABLE_DEFAULT_ERROR) -/// } +/// #[optional] +/// fn bar(&self) -> Result<()>; /// } /// /// struct Foo; @@ -228,8 +215,6 @@ pub fn module(input: TokenStream) -> TokenStream { /// assert_eq!(::HAS_BAR, false); /// # } /// ``` -/// -/// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html #[proc_macro_attribute] pub fn vtable(attr: TokenStream, input: TokenStream) -> TokenStream { parse_macro_input!(attr as syn::parse::Nothing); diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs index 1dfc40eb53f8..981ad2e49647 100644 --- a/rust/macros/vtable.rs +++ b/rust/macros/vtable.rs @@ -49,7 +49,7 @@ fn handle_trait(mut item: ItemTrait) -> Result { }); } - for item in &item.items { + for item in &mut item.items { if let TraitItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; let gen_const_name = Ident::new( @@ -57,6 +57,30 @@ fn handle_trait(mut item: ItemTrait) -> Result { name.span(), ); + if fn_item + .attrs + .extract_if(.., |attr| attr.path().is_ident("optional")) + .count() + != 0 + { + if let Some(default) = &fn_item.default { + Err(Error::new_spanned( + default, + "`#[optional]` methods must not have a default implementation", + ))?; + } + + // Optional methods in Rust need a default implementation. Inject one that fails + // the build at compile time if not overridden. + fn_item.default = Some(parse_quote!({ + ::kernel::build_assert::build_error!( + "This function must not be called, see the `#[vtable]` documentation." + ); + })); + // Ensure that the function is never code generated unless used. + fn_item.attrs.push(parse_quote!(#[inline])); + } + // We don't know on the implementation-site whether a method is required or provided // so we have to generate a const for all methods. // However, hide it for required methods as it will always be true. From a4082121eeb5caa7704734184aa24199e83e19bb Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 11 Aug 2026 14:55:57 +0100 Subject: [PATCH 41/91] rust: miscdevice: use `#[optional]` attribute of `#[vtable]` macro Replace the `build_error!(VTABLE_DEFAULT_ERROR)` boilerplate with the `#[optional]` attribute. Signed-off-by: Gary Guo Acked-by: Greg Kroah-Hartman Link: https://patch.msgid.link/20260811-vtable_macro-v1-3-50c5aab3c211@garyguo.net [ Reworded title for prefix. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/miscdevice.rs | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 8d4b719bd83f..eacea29e3fb0 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -11,10 +11,7 @@ use crate::{ bindings, device::Device, - error::{ - to_result, - VTABLE_DEFAULT_ERROR, // - }, + error::to_result, fs::{ File, Kiocb, // @@ -145,37 +142,33 @@ fn release(device: Self::Ptr, _file: &File) { /// the VMA initialization by calling methods of `vma`. If the function does not return an /// error, the kernel will complete initialization of the VMA according to the properties of /// `vma`. + #[optional] fn mmap( _device: ::Borrowed<'_>, _file: &File, _vma: &VmaNew, - ) -> Result { - build_error!(VTABLE_DEFAULT_ERROR) - } + ) -> Result; /// Read from this miscdevice. - fn read_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterDest<'_>) -> Result { - build_error!(VTABLE_DEFAULT_ERROR) - } + #[optional] + fn read_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterDest<'_>) -> Result; /// Write to this miscdevice. - fn write_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterSource<'_>) -> Result { - build_error!(VTABLE_DEFAULT_ERROR) - } + #[optional] + fn write_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterSource<'_>) -> Result; /// Handler for ioctls. /// /// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`]. /// /// [`kernel::ioctl`]: mod@crate::ioctl + #[optional] fn ioctl( _device: ::Borrowed<'_>, _file: &File, _cmd: u32, _arg: usize, - ) -> Result { - build_error!(VTABLE_DEFAULT_ERROR) - } + ) -> Result; /// Handler for ioctls. /// @@ -185,23 +178,21 @@ fn ioctl( /// that have different layout on 32-bit and 64-bit userspace. If no implementation is /// provided, then `compat_ptr_ioctl` will be used instead. #[cfg(CONFIG_COMPAT)] + #[optional] fn compat_ioctl( _device: ::Borrowed<'_>, _file: &File, _cmd: u32, _arg: usize, - ) -> Result { - build_error!(VTABLE_DEFAULT_ERROR) - } + ) -> Result; /// Show info for this fd. + #[optional] fn show_fdinfo( _device: ::Borrowed<'_>, _m: &SeqFile, _file: &File, - ) { - build_error!(VTABLE_DEFAULT_ERROR) - } + ); } /// A vtable for the file operations of a Rust miscdevice. From 3504d1bce9920b23499ab1af682cdf1eceba7c75 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 2 Sep 2026 22:59:59 +0100 Subject: [PATCH 42/91] rust: kunit: use `file!()` inside `kunit_assert!` This parameter is needed currently because doctests want to override the file. Simplify it by changing it to use `file!()`. Have doctests override `file!()` macro to achieve the current behavior. This allows us to remove the `file` helper and associated Kconfig options. Reviewed-by: David Gow Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260902-kunit-v2-1-eda3ed6c00c5@garyguo.net Signed-off-by: Miguel Ojeda --- Documentation/rust/general-information.rst | 4 ++-- init/Kconfig | 3 --- rust/kernel/kunit.rs | 9 +++++---- rust/macros/helpers.rs | 16 ---------------- rust/macros/kunit.rs | 5 ++--- rust/macros/lib.rs | 5 ----- scripts/rustdoc_test_gen.rs | 10 ++++++++-- 7 files changed, 17 insertions(+), 35 deletions(-) diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst index 09234bed272c..20e3178ebfd4 100644 --- a/Documentation/rust/general-information.rst +++ b/Documentation/rust/general-information.rst @@ -157,5 +157,5 @@ numerical comparisons, one may define a new Kconfig symbol: .. code-block:: kconfig - config RUSTC_HAS_SPAN_FILE - def_bool RUSTC_VERSION >= 108800 + config RUSTC_HAS_FILE_AS_C_STR + def_bool RUSTC_VERSION >= 109100 diff --git a/init/Kconfig b/init/Kconfig index 8583d9f06c52..24b019104d88 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -178,9 +178,6 @@ config LD_CAN_USE_KEEP_IN_OVERLAY # https://github.com/llvm/llvm-project/pull/130661 def_bool LD_IS_BFD || LLD_VERSION >= 210000 -config RUSTC_HAS_SPAN_FILE - def_bool RUSTC_VERSION >= 108800 - config RUSTC_HAS_UNNECESSARY_TRANSMUTES def_bool RUSTC_VERSION >= 108800 diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 91eaff8c186a..12084873b51e 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -58,14 +58,15 @@ pub fn info(args: fmt::Arguments<'_>) { #[doc(hidden)] #[macro_export] macro_rules! kunit_assert { - ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => { + ($name:literal, $diff:expr, $condition:expr $(,)?) => { 'out: { // Do nothing if the condition is `true`. if $condition { break 'out; } - static FILE: &'static $crate::str::CStr = $file; + // Use `file!()` instead of `::core::file!()` here so it can be overridden. + static FILE: &'static $crate::str::CStr = $crate::c_str!(file!()); static LINE: i32 = ::core::line!() as i32 - $diff; static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); @@ -164,10 +165,10 @@ unsafe impl Sync for UnaryAssert {} #[doc(hidden)] #[macro_export] macro_rules! kunit_assert_eq { - ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + ($name:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ // For the moment, we just forward to the expression assert because, for binary asserts, // KUnit supports only a few types (e.g. integers). - $crate::kunit_assert!($name, $file, $diff, $left == $right); + $crate::kunit_assert!($name, $diff, $left == $right); }}; } diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index d18fbf4daa0a..8cc9a20f470e 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -38,22 +38,6 @@ pub(crate) fn value(&self) -> String { } } -pub(crate) fn file() -> String { - #[cfg(not(CONFIG_RUSTC_HAS_SPAN_FILE))] - { - proc_macro::Span::call_site() - .source_file() - .path() - .to_string_lossy() - .into_owned() - } - - #[cfg(CONFIG_RUSTC_HAS_SPAN_FILE)] - { - proc_macro::Span::call_site().file() - } -} - /// Obtain all `#[cfg]` attributes. pub(crate) fn gather_cfg_attrs(attr: &[Attribute]) -> impl Iterator + '_ { attr.iter().filter(|a| a.path().is_ident("cfg")) diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs index ae20ed6768f1..936eff014870 100644 --- a/rust/macros/kunit.rs +++ b/rust/macros/kunit.rs @@ -109,12 +109,11 @@ pub(crate) fn kunit_tests(test_suite: Ident, mut module: ItemMod) -> Result {{ - kernel::kunit_assert!(#test_str, #path, 0, $cond); + kernel::kunit_assert!(#test_str, 0, $cond); }} } }); @@ -122,7 +121,7 @@ macro_rules! assert { #[allow(unused)] macro_rules! assert_eq { ($left:expr, $right:expr $(,)?) => {{ - kernel::kunit_assert_eq!(#test_str, #path, 0, $left, $right); + kernel::kunit_assert_eq!(#test_str, 0, $left, $right); }} } }); diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index b84db5c6bdc4..68386b2ed55f 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -8,11 +8,6 @@ // Stable since Rust 1.87.0. #![feature(extract_if)] -// -// Stable since Rust 1.88.0 under a different name, `proc_macro_span_file`, -// which was added in Rust 1.88.0. This is why `cfg_attr` is used here, i.e. -// to avoid depending on the full `proc_macro_span` on Rust >= 1.88.0. -#![cfg_attr(not(CONFIG_RUSTC_HAS_SPAN_FILE), feature(proc_macro_span))] mod concat_idents; mod export; diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index d087c0d9fcb3..bb57745ec339 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -176,12 +176,18 @@ fn main() { r#"/// Generated `{name}` KUnit test case from a Rust documentation test. #[no_mangle] pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ + /// Overrides the usual [`file!`] macro with one that expands to the real path. + #[allow(unused)] + macro_rules! file {{ + () => {{ "{real_path}" }} + }} + /// Overrides the usual [`assert!`] macro with one that calls KUnit instead. #[allow(unused)] macro_rules! assert {{ ($cond:expr $(,)?) => {{{{ ::kernel::kunit_assert!( - "{kunit_name}", c"{real_path}", __DOCTEST_ANCHOR - {line}, $cond + "{kunit_name}", __DOCTEST_ANCHOR - {line}, $cond ); }}}} }} @@ -191,7 +197,7 @@ macro_rules! assert {{ macro_rules! assert_eq {{ ($left:expr, $right:expr $(,)?) => {{{{ ::kernel::kunit_assert_eq!( - "{kunit_name}", c"{real_path}", __DOCTEST_ANCHOR - {line}, $left, $right + "{kunit_name}", __DOCTEST_ANCHOR - {line}, $left, $right ); }}}} }} From e9a35442ef91f7f599a95e04928e80748fba2998 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 2 Sep 2026 23:00:00 +0100 Subject: [PATCH 43/91] rust: kunit: use `line!()` inside `kunit_assert!` The `diff` parameter is needed currently because doctests want to override the line number. Simplify it by changing it to use `line!()`. Have doctests override `line!()` macro to achieve the current behavior. An existing doctest requires `line!()` to expand to a literal; so update it to use `::core::line!()` instead. Reviewed-by: David Gow Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260902-kunit-v2-2-eda3ed6c00c5@garyguo.net [ Removed spurious `///`-to-`//` change and adjusted `line!` to match. Added a comment to match `file!`'s. Made `NOTE:` more precise. Reworded to say it is only one doctest being updated. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/kunit.rs | 9 +++++---- rust/kernel/str.rs | 2 +- rust/macros/kunit.rs | 4 ++-- scripts/rustdoc_test_gen.rs | 14 +++++++++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 12084873b51e..25ff272d29b2 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -58,7 +58,7 @@ pub fn info(args: fmt::Arguments<'_>) { #[doc(hidden)] #[macro_export] macro_rules! kunit_assert { - ($name:literal, $diff:expr, $condition:expr $(,)?) => { + ($name:literal, $condition:expr $(,)?) => { 'out: { // Do nothing if the condition is `true`. if $condition { @@ -67,7 +67,8 @@ macro_rules! kunit_assert { // Use `file!()` instead of `::core::file!()` here so it can be overridden. static FILE: &'static $crate::str::CStr = $crate::c_str!(file!()); - static LINE: i32 = ::core::line!() as i32 - $diff; + // Use `line!()` instead of `::core::line!()` here so it can be overridden. + static LINE: i32 = line!() as i32; static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); // SAFETY: FFI call without safety requirements. @@ -165,10 +166,10 @@ unsafe impl Sync for UnaryAssert {} #[doc(hidden)] #[macro_export] macro_rules! kunit_assert_eq { - ($name:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + ($name:literal, $left:expr, $right:expr $(,)?) => {{ // For the moment, we just forward to the expression assert because, for binary asserts, // KUnit supports only a few types (e.g. integers). - $crate::kunit_assert!($name, $diff, $left == $right); + $crate::kunit_assert!($name, $left == $right); }}; } diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index b3caa9a1c898..644b4279a116 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -411,7 +411,7 @@ fn as_ref(&self) -> &BStr { /// const BAD: &CStr = c_str!("literal"); /// /// // `c_str!` is still needed for static non-literal C strings. -/// const GOOD: &CStr = c_str!(concat!(file!(), ":", line!(), ": My CStr!")); +/// const GOOD: &CStr = c_str!(concat!(file!(), ":", ::core::line!(), ": My CStr!")); /// ``` #[macro_export] macro_rules! c_str { diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs index 936eff014870..e9152b9d51f9 100644 --- a/rust/macros/kunit.rs +++ b/rust/macros/kunit.rs @@ -113,7 +113,7 @@ pub(crate) fn kunit_tests(test_suite: Ident, mut module: ItemMod) -> Result {{ - kernel::kunit_assert!(#test_str, 0, $cond); + kernel::kunit_assert!(#test_str, $cond); }} } }); @@ -121,7 +121,7 @@ macro_rules! assert { #[allow(unused)] macro_rules! assert_eq { ($left:expr, $right:expr $(,)?) => {{ - kernel::kunit_assert_eq!(#test_str, 0, $left, $right); + kernel::kunit_assert_eq!(#test_str, $left, $right); }} } }); diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index bb57745ec339..7159cd6e726d 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -182,12 +182,20 @@ macro_rules! file {{ () => {{ "{real_path}" }} }} + /// Overrides the usual [`line!`] macro with one that expands to the real line number. + #[allow(unused)] + macro_rules! line {{ + // NOTE: This does not expand to a literal, but a constant expression. + // Therefore code that depends on `line!()` expanding to a literal needs special adjustment. + () => {{ const {{ ::core::line!() - __DOCTEST_ANCHOR + {line} }} }} + }} + /// Overrides the usual [`assert!`] macro with one that calls KUnit instead. #[allow(unused)] macro_rules! assert {{ ($cond:expr $(,)?) => {{{{ ::kernel::kunit_assert!( - "{kunit_name}", __DOCTEST_ANCHOR - {line}, $cond + "{kunit_name}", $cond ); }}}} }} @@ -197,7 +205,7 @@ macro_rules! assert {{ macro_rules! assert_eq {{ ($left:expr, $right:expr $(,)?) => {{{{ ::kernel::kunit_assert_eq!( - "{kunit_name}", __DOCTEST_ANCHOR - {line}, $left, $right + "{kunit_name}", $left, $right ); }}}} }} @@ -219,7 +227,7 @@ macro_rules! assert_eq {{ /// The anchor where the test code body starts. #[allow(unused)] - static __DOCTEST_ANCHOR: i32 = ::core::line!() as i32 + {body_offset} + 2; + static __DOCTEST_ANCHOR: u32 = ::core::line!() + {body_offset} + 2; {{ #![allow(unreachable_pub, clippy::disallowed_names)] {body} From a41ec2936d2fecdb958e4f4417781771bcfa683d Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 2 Sep 2026 23:00:01 +0100 Subject: [PATCH 44/91] rust: doctest: generate Rust kunit test suites For doctest, instead of generating C FFI functions, generate a Rust test suite with `#[kunit_tests]` and `#[test]` attributes. This makes the C glue no longer needed. Reviewed-by: David Gow Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260902-kunit-v2-3-eda3ed6c00c5@garyguo.net [ Removed generated file from `rust/.gitignore` and added it to `scripts/remove-stale-files`. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/.gitignore | 1 - rust/Makefile | 4 +-- scripts/remove-stale-files | 2 ++ scripts/rustdoc_test_gen.rs | 69 +++++-------------------------------- 4 files changed, 12 insertions(+), 64 deletions(-) diff --git a/rust/.gitignore b/rust/.gitignore index d3829ffab80b..5ba6b490d1e9 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -3,7 +3,6 @@ bindings_generated.rs bindings_helpers_generated.rs doctests_kernel_generated.rs -doctests_kernel_generated_kunit.c uapi_generated.rs exports_*_generated.h doc/ diff --git a/rust/Makefile b/rust/Makefile index 152649f209d8..97f577ed0b8f 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -39,10 +39,8 @@ obj-$(CONFIG_RUST) += exports.o always-$(CONFIG_RUST) += host/libproc_macro2.rlib host/libquote.rlib host/libsyn.rlib always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs -always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o -obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o always-$(subst y,$(CONFIG_RUST),$(CONFIG_JUMP_LABEL)) += kernel/generated_arch_static_branch_asm.rs ifndef CONFIG_UML @@ -402,7 +400,7 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< $< $(rustdoc_test_kernel_quiet); \ $(objtree)/scripts/rustdoc_test_gen -%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \ +%/doctests_kernel_generated.rs: \ $(src)/kernel/lib.rs $(obj)/kernel.o \ $(objtree)/scripts/rustdoc_test_builder \ $(objtree)/scripts/rustdoc_test_gen FORCE diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index 6e39fa8540df..f1f776caae00 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -26,3 +26,5 @@ rm -f scripts/selinux/genheaders/genheaders rm -f *.spec rm -f lib/test_fortify.log + +rm -f rust/doctests_kernel_generated_kunit.c diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index 7159cd6e726d..754c5aafcce4 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -125,9 +125,7 @@ fn main() { // Sort paths. paths.sort(); - let mut rust_tests = String::new(); - let mut c_test_declarations = String::new(); - let mut c_test_cases = String::new(); + let mut tests = String::new(); let mut body = String::new(); let mut last_file = String::new(); let mut number = 0; @@ -172,10 +170,10 @@ fn main() { use std::fmt::Write; write!( - rust_tests, + tests, r#"/// Generated `{name}` KUnit test case from a Rust documentation test. -#[no_mangle] -pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ +#[test] +fn {kunit_name}() {{ /// Overrides the usual [`file!`] macro with one that expands to the real path. #[allow(unused)] macro_rules! file {{ @@ -190,26 +188,6 @@ macro_rules! line {{ () => {{ const {{ ::core::line!() - __DOCTEST_ANCHOR + {line} }} }} }} - /// Overrides the usual [`assert!`] macro with one that calls KUnit instead. - #[allow(unused)] - macro_rules! assert {{ - ($cond:expr $(,)?) => {{{{ - ::kernel::kunit_assert!( - "{kunit_name}", $cond - ); - }}}} - }} - - /// Overrides the usual [`assert_eq!`] macro with one that calls KUnit instead. - #[allow(unused)] - macro_rules! assert_eq {{ - ($left:expr, $right:expr $(,)?) => {{{{ - ::kernel::kunit_assert_eq!( - "{kunit_name}", $left, $right - ); - }}}} - }} - // Many tests need the prelude, so provide it by default. #[allow(unused)] use ::kernel::prelude::*; @@ -238,14 +216,9 @@ macro_rules! assert_eq {{ "# ) .unwrap(); - - write!(c_test_declarations, "void {kunit_name}(struct kunit *);\n").unwrap(); - write!(c_test_cases, " KUNIT_CASE({kunit_name}),\n").unwrap(); } - let rust_tests = rust_tests.trim(); - let c_test_declarations = c_test_declarations.trim(); - let c_test_cases = c_test_cases.trim(); + let tests = tests.trim(); write!( BufWriter::new(File::create("rust/doctests_kernel_generated.rs").unwrap()), @@ -271,34 +244,10 @@ impl ModuleMetadata for LocalModule {{ }}; }} -{rust_tests} -"# - ) - .unwrap(); - - write!( - BufWriter::new(File::create("rust/doctests_kernel_generated_kunit.c").unwrap()), - r#"/* - * `kernel` crate documentation tests. - */ - -#include - -{c_test_declarations} - -static struct kunit_case test_cases[] = {{ - {c_test_cases} - {{ }} -}}; - -static struct kunit_suite test_suite = {{ - .name = "rust_doctests_kernel", - .test_cases = test_cases, -}}; - -kunit_test_suite(test_suite); - -MODULE_LICENSE("GPL"); +#[kernel::macros::kunit_tests(rust_doctests_kernel)] +mod tests {{ +{tests} +}} "# ) .unwrap(); From 2dee3d3adcadcc57d62ba6608cd02f50b0c10264 Mon Sep 17 00:00:00 2001 From: Nicolas Schier Date: Wed, 17 Jun 2026 16:09:48 +0200 Subject: [PATCH 45/91] rust: kbuild: Replace and dissolve procmacro-extension variable Use make's $(basename) function [1] and substitution reference [2] instead of $(patsubst) to extract crate names for rust procmacro targets. As there is only one user of the procmacro-extension variable, do the crate name extraction in-place and remove procmacro-extension. The procmacro-extension variable is used to record the binary file suffix of dynamically loadable rust libraries -- which is 'so' on Linux and others and 'dylib' in MacOS -- to remove that very suffix from a target file name for reconstructing the corresponding crate name in cmd_rustc_procmacro. Extracting the crate name from the target name in-place is shorter and reduces complexity. Link: https://www.gnu.org/software/make/manual/html_node/File-Name-Functions.html # [1] Link: https://www.gnu.org/software/make/manual/html_node/Substitution-Refs.html # [2] Reviewed-by: Tamir Duberstein Tested-by: Tamir Duberstein Signed-off-by: Nicolas Schier Link: https://lore.kernel.org/linux-kbuild/CANiq72nmPPF=6AUp5mcDJVni80ORQr0AQ-4uCZaE+CzB5kV=eg@mail.gmail.com/ Link: https://patch.msgid.link/20260617-rust-kbuild-replace-procmacro-extension-v2-1-667bdc3a6435@fritz.com [ Rebased. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 97f577ed0b8f..320ab6bb36c4 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -51,7 +51,6 @@ endif ifdef CONFIG_RUST procmacro-name = $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name $(1) --crate-type proc-macro - Date: Tue, 6 Oct 2026 17:29:31 +0200 Subject: [PATCH 46/91] rust: pin-init: internal: init: forget guards in reverse order Currently, the forget order is the same as initialization order, this is the wrong way around compared to their drop order when going out-of-scope. The wrong order will cause borrow checker errors if there are lifetime dependencies between the guards. This does not matter for now because there cannot be any lifetime dependency; it would matter for self-referential borrows. Reverse the order to be consistent with their out-of-scope drop order. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-1-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 6605e04d49df..f0b23d0311ac 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -361,8 +361,7 @@ fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKi /// Generate the code that initializes the fields of the struct using the initializers in `field`. fn init_fields(fields: &Punctuated, pinned: bool) -> TokenStream { - let mut guards = vec![]; - let mut guard_attrs = vec![]; + let mut forget_guards = vec![]; let mut res = TokenStream::new(); for InitializerField { attrs, kind } in fields { let cfgs = { @@ -474,17 +473,19 @@ fn init_fields(fields: &Punctuated, pinned: bool) - #binding }); - guards.push(guard); - guard_attrs.push(cfgs); + forget_guards.push(quote_spanned! { span => + #(#cfgs)* + ::core::mem::forget(#guard); + }); } + + forget_guards.reverse(); + quote! { #res // If execution reaches this point, all fields have been initialized. Therefore we can now // dismiss the guards by forgetting them. - #( - #(#guard_attrs)* - ::core::mem::forget(#guards); - )* + #(#forget_guards)* } } From da2b9009d445efb826a9eb535c4d62129fee6b35 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:32 +0200 Subject: [PATCH 47/91] rust: pin-init: internal: init: set span location of guards Set the span location of the guards to point to the identifier. The generated diagnostic is still not ideal, as ideally it points the place where the identifier is used rather than the identifier name in initializer, but is still an improvement to status quo. Before the change: error[E0505]: cannot move out of value because it is borrowed --> tests/ui/compile-fail/init/accessor_lifetime.rs:8:13 | 8 | let _ = init!(Foo { | _____________^ 9 | | x: 0, 10 | | _: { 11 | | let _: &'static usize = x; | | -------------- type annotation requires that borrow lasts for `'static` 12 | | }, 13 | | }); | | ^ | | | | |______move out of value occurs here | borrow of value occurs here After the change: error[E0505]: cannot move out of value because it is borrowed --> tests/ui/compile-fail/init/accessor_lifetime.rs:9:9 | 9 | x: 0, | ^ | | | move out of value occurs here | borrow of value occurs here 10 | _: { 11 | let _: &'static usize = x; | -------------- type annotation requires that borrow lasts for `'static` Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-2-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index f0b23d0311ac..80e4dc068d2e 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -413,7 +413,7 @@ fn init_fields(fields: &Punctuated, pinned: bool) - }; // `mixed_site` ensures that the guard is not accessible to the user-controlled code. - let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + let guard = format_ident!("__{ident}_guard", span = span); let full_span = kind.span().resolved_at(Span::mixed_site()); let init = match kind { From 25e82b177c8bad26509e1e989795f63ef08e9dc5 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:33 +0200 Subject: [PATCH 48/91] rust: pin-init: internal: add utility to handle generics merging There're many cases where we want to merge multiple generics together. Currently this is done by cloning generics and modifying it, making the code less readable. Add a utility code that allow printing multiple generics together, ensuring correct ordering between lifetime and other parameters. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-3-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 10 +- rust/pin-init/internal/src/util.rs | 164 ++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 6 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 982a38ceea93..4ffa35bbbccd 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -321,10 +321,12 @@ fn generate_projections( is_tuple_struct: bool, fields: &[FieldInfo<'_>], ) -> TokenStream { - let (impl_generics, ty_generics, _) = generics.split_for_impl(); - let mut generics_with_pin_lt = generics.clone(); - generics_with_pin_lt.params.insert(0, parse_quote!('__pin)); - let (_, ty_generics_with_pin_lt, whr) = generics_with_pin_lt.split_for_impl(); + let pin_lt_generics: Generics = parse_quote!(<'__pin>); + let generics_with_pin_lt = CombinedGenerics(vec![&pin_lt_generics, generics]); + + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + let (_, ty_generics_with_pin_lt, _) = generics_with_pin_lt.split_for_impl(); + let projection = format_ident!("{ident}Projection"); let this = format_ident!("this"); diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index 0b5c28b21091..2a57a25b70ae 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::{Ident, TokenStream}; -use quote::format_ident; -use syn::{Attribute, Index, Member}; +use quote::{format_ident, ToTokens}; +use syn::{Attribute, GenericParam, Generics, Index, Member, Token}; pub(crate) trait AttrListExt { fn extract_cfg_attrs(&mut self) -> Vec; @@ -52,3 +52,163 @@ fn display_name(&self) -> String { } } } + +pub(crate) struct CombinedGenerics<'a>(pub(crate) Vec<&'a Generics>); +pub(crate) struct CombinedImplGenerics<'a>(&'a CombinedGenerics<'a>); +pub(crate) struct CombinedTypeGenerics<'a>(&'a CombinedGenerics<'a>); + +impl CombinedGenerics<'_> { + pub(crate) fn split_for_impl( + &self, + ) -> ( + CombinedImplGenerics<'_>, + CombinedTypeGenerics<'_>, + // A stub type so `split_for_impl` signature matches that of `syn`'s. + impl Sized, + ) { + (CombinedImplGenerics(self), CombinedTypeGenerics(self), ()) + } +} + +impl ToTokens for CombinedGenerics<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + // Most of the time we are only adding lifetimes, so we prefer to place + // the user provided generics last. + self.0 + .last() + .and_then(|x| x.lt_token) + .unwrap_or_default() + .to_tokens(tokens); + + let comma: Token![,] = Default::default(); + + // Output lifetimes first. + for generics in self.0.iter() { + for param in generics.params.pairs() { + if let GenericParam::Lifetime(lt) = param.value() { + lt.to_tokens(tokens); + param.punct().unwrap_or(&&comma).to_tokens(tokens); + } + } + } + + for generics in self.0.iter() { + for param in generics.params.pairs() { + if let GenericParam::Lifetime(_) = param.value() { + continue; + }; + param.value().to_tokens(tokens); + param.punct().unwrap_or(&&comma).to_tokens(tokens); + } + } + + self.0 + .last() + .and_then(|x| x.gt_token) + .unwrap_or_default() + .to_tokens(tokens); + } +} + +impl ToTokens for CombinedImplGenerics<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0 + .0 + .last() + .and_then(|x| x.lt_token) + .unwrap_or_default() + .to_tokens(tokens); + + let comma: Token![,] = Default::default(); + + // Output lifetimes first. + for generics in self.0 .0.iter() { + for param in generics.params.pairs() { + if let GenericParam::Lifetime(lt) = param.value() { + lt.to_tokens(tokens); + param.punct().unwrap_or(&&comma).to_tokens(tokens); + } + } + } + + for generics in self.0 .0.iter() { + for param in generics.params.pairs() { + // Leave out defaults. + match param.value() { + GenericParam::Lifetime(_) => continue, + GenericParam::Type(param) => { + param.ident.to_tokens(tokens); + if !param.bounds.is_empty() { + param + .colon_token + .unwrap_or_else(Default::default) + .to_tokens(tokens); + param.bounds.to_tokens(tokens); + } + } + GenericParam::Const(param) => { + param.const_token.to_tokens(tokens); + param.ident.to_tokens(tokens); + param.colon_token.to_tokens(tokens); + param.ty.to_tokens(tokens); + } + } + param.punct().unwrap_or(&&comma).to_tokens(tokens); + } + } + + self.0 + .0 + .last() + .and_then(|x| x.gt_token) + .unwrap_or_default() + .to_tokens(tokens); + } +} + +impl ToTokens for CombinedTypeGenerics<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0 + .0 + .last() + .and_then(|x| x.lt_token) + .unwrap_or_default() + .to_tokens(tokens); + + let comma: Token![,] = Default::default(); + + // Output lifetimes first. + for generics in self.0 .0.iter() { + for param in generics.params.pairs() { + if let GenericParam::Lifetime(lt) = param.value() { + // Leave out bounds + lt.lifetime.to_tokens(tokens); + param.punct().unwrap_or(&&comma).to_tokens(tokens); + } + } + } + + for generics in self.0 .0.iter() { + for param in generics.params.pairs() { + // Leave out bounds and defaults. + match param.value() { + GenericParam::Lifetime(_) => continue, + GenericParam::Type(param) => { + param.ident.to_tokens(tokens); + } + GenericParam::Const(param) => { + param.ident.to_tokens(tokens); + } + } + param.punct().unwrap_or(&&comma).to_tokens(tokens); + } + } + + self.0 + .0 + .last() + .and_then(|x| x.gt_token) + .unwrap_or_default() + .to_tokens(tokens); + } +} From 003a8953d2f1c310550ac477a508b7be910b9827 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:34 +0200 Subject: [PATCH 49/91] rust: pin-init: internal: add utility for extract a single attribute More attributes are to be added which has the requirement that they should only be specified once. Add a utility code to extract attributes for a specific path and report error when appeared multiple times. Convert `#[pin]` parsing to use the added utility. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-4-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 9 ++------- rust/pin-init/internal/src/util.rs | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 4ffa35bbbccd..ac83f0f73e61 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -137,12 +137,7 @@ pub(crate) fn pin_data( .iter_mut() .enumerate() .map(|(index, field)| { - let len = field.attrs.len(); - field.attrs.retain(|a| !a.path().is_ident("pin")); - let pinned_count = len - field.attrs.len(); - if pinned_count > 1 { - dcx.error(&field, "#[pin] attribute specified more than once"); - } + let pinned = field.attrs.extract_single_attr(dcx, "pin").is_some(); assert!( !field.attrs.iter().any(|a| a.path().is_ident("cfg")), @@ -159,7 +154,7 @@ pub(crate) fn pin_data( FieldInfo { field: &*field, member, - pinned: pinned_count != 0, + pinned, } }) .collect(); diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index 2a57a25b70ae..3ce498cea754 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -4,8 +4,15 @@ use quote::{format_ident, ToTokens}; use syn::{Attribute, GenericParam, Generics, Index, Member, Token}; +use crate::DiagCtxt; + pub(crate) trait AttrListExt { fn extract_cfg_attrs(&mut self) -> Vec; + + /// Extract attribute with identifier `path`. + /// + /// Report error if the attribute appears multiple times. + fn extract_single_attr(&mut self, dcx: &DiagCtxt, path: &str) -> Option; } impl AttrListExt for Vec { @@ -25,6 +32,24 @@ fn extract_cfg_attrs(&mut self) -> Vec { cfg } + + fn extract_single_attr(&mut self, dcx: &DiagCtxt, path: &str) -> Option { + // FIXME: Replace with `extract_if` when MSRV >= 1.85. + let attr_pos = self.iter().position(|attr| attr.path().is_ident(path))?; + let attr = self.remove(attr_pos); + self.retain(|attr| { + if !attr.path().is_ident(path) { + return true; + } + + dcx.error( + attr, + format!("`#[{path}]` attribute specified more than once"), + ); + false + }); + Some(attr) + } } pub(crate) trait MemberExt { From 6fbdebfeb02f36c5ec187aea34e47d219b866253 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:35 +0200 Subject: [PATCH 50/91] rust: pin-init: internal: pin_data: move projection struct into `const _: ()` block Most users would not need to mention the generated projection struct. With the upcoming self-referential support, there needs to be additional projection structs. Instead of give them publicly visible names, simply make the name not publicly accessible. A heavy user of pin projection outside kernel is for async programming. I've checked tokio codebase there is a single use case and that's for enums. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-5-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index ac83f0f73e61..3acced1fe806 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -186,10 +186,10 @@ pub(crate) fn pin_data( Ok(quote! { #struct_ - #projections // We put the rest into this const item, because it then will not be accessible to anything // outside. const _: () = { + #projections #the_pin_data #unpin_impl #drop_impl @@ -322,7 +322,6 @@ fn generate_projections( let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, ty_generics_with_pin_lt, _) = generics_with_pin_lt.split_for_impl(); - let projection = format_ident!("{ident}Projection"); let this = format_ident!("this"); let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields @@ -371,13 +370,13 @@ fn generate_projections( let (projection_def, projection_init) = if is_tuple_struct { ( quote! { - #vis struct #projection #generics_with_pin_lt ( + #vis struct __Projection #generics_with_pin_lt ( #(#fields_decl)* ::core::marker::PhantomData<&'__pin mut ()>, ) #whr; }, quote! { - #projection( + __Projection( #(#fields_proj)* ::core::marker::PhantomData, ) @@ -386,7 +385,7 @@ fn generate_projections( } else { ( quote! { - #vis struct #projection #generics_with_pin_lt + #vis struct __Projection #generics_with_pin_lt #whr { #(#fields_decl)* @@ -394,7 +393,7 @@ fn generate_projections( } }, quote! { - #projection { + __Projection { #(#fields_proj)* ___pin_phantom_data: ::core::marker::PhantomData, } @@ -422,7 +421,7 @@ impl #impl_generics #ident #ty_generics #[inline] #vis fn project<'__pin>( self: ::core::pin::Pin<&'__pin mut Self>, - ) -> #projection #ty_generics_with_pin_lt { + ) -> __Projection #ty_generics_with_pin_lt { // SAFETY: we only give access to `&mut` for fields not structurally pinned. let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; #projection_init From 93bfdb53067cca9b89632022bd3f4560e03ac015 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:36 +0200 Subject: [PATCH 51/91] rust: pin-init: internal: pin_data: extract `#[cfg]` processing to its own function The `pin_data` function is getting big. Split the `#[cfg]` handling which does not interact with other part of expansion to its own `expand_with_cfg` function. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-6-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/lib.rs | 3 ++- rust/pin-init/internal/src/pin_data.rs | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index 07b3d33d2282..01e19fcd31fb 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -22,7 +22,8 @@ #[proc_macro_attribute] pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { - DiagCtxt::for_item(|dcx| pin_data::pin_data(syn::parse(args)?, syn::parse(input)?, dcx)).into() + DiagCtxt::for_item(|dcx| pin_data::expand_with_cfg(syn::parse(args)?, syn::parse(input)?, dcx)) + .into() } #[proc_macro_attribute] diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 3acced1fe806..a4d6965c1572 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -8,7 +8,8 @@ punctuated::Punctuated, spanned::Spanned, visit_mut::VisitMut, - Field, Fields, Generics, Ident, Index, Item, Member, PathSegment, Type, TypePath, Visibility, + Field, Fields, Generics, Ident, Index, Item, ItemStruct, Member, PathSegment, Type, TypePath, + Visibility, }; use crate::{ @@ -54,7 +55,7 @@ struct FieldInfo<'a> { pinned: bool, } -pub(crate) fn pin_data( +pub(crate) fn expand_with_cfg( args: Args, input: Item, dcx: &mut DiagCtxt, @@ -120,6 +121,14 @@ pub(crate) fn pin_data( )); } + expand(args, struct_, dcx) +} + +fn expand( + args: Args, + mut struct_: ItemStruct, + dcx: &mut DiagCtxt, +) -> Result { // The generics might contain the `Self` type. Since this macro will define a new type with the // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed // to this struct definition. Therefore we have to replace `Self` with the concrete name. From ec87b5938354046ca79a3ed7fcd89a2cb9469f11 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:37 +0200 Subject: [PATCH 52/91] rust: pin-init: internal: pin_data: rename `'__pin` lifetime to `'__this` "pin" is a very overloaded word in pin-init. For the generated projection structs, the lifetime refer to that of the whole struct, so rename the lifetime used in expansion from `'__pin` to `'__this` instead. The `__pin_phantom_data` field is renamed to `__this` for the same reason and for clarity purpose, capture a `PhantomData` of the mutable reference to the struct. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-7-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index a4d6965c1572..6e18bc9e890a 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -325,11 +325,11 @@ fn generate_projections( is_tuple_struct: bool, fields: &[FieldInfo<'_>], ) -> TokenStream { - let pin_lt_generics: Generics = parse_quote!(<'__pin>); - let generics_with_pin_lt = CombinedGenerics(vec![&pin_lt_generics, generics]); + let this_lt_generics: Generics = parse_quote!(<'__this>); + let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); - let (_, ty_generics_with_pin_lt, _) = generics_with_pin_lt.split_for_impl(); + let (_, ty_generics_with_this_lt, _) = generics_with_this_lt.split_for_impl(); let this = format_ident!("this"); @@ -348,7 +348,7 @@ fn generate_projections( if field.pinned { ( quote!( - #vis #name ::core::pin::Pin<&'__pin mut #ty>, + #vis #name ::core::pin::Pin<&'__this mut #ty>, ), quote!( // SAFETY: this field is structurally pinned. @@ -358,7 +358,7 @@ fn generate_projections( } else { ( quote!( - #vis #name &'__pin mut #ty, + #vis #name &'__this mut #ty, ), quote!( #name &mut #this.#member, @@ -379,9 +379,9 @@ fn generate_projections( let (projection_def, projection_init) = if is_tuple_struct { ( quote! { - #vis struct __Projection #generics_with_pin_lt ( + #vis struct __Projection #generics_with_this_lt ( #(#fields_decl)* - ::core::marker::PhantomData<&'__pin mut ()>, + ::core::marker::PhantomData<&'__this mut #ident #ty_generics>, ) #whr; }, quote! { @@ -394,17 +394,17 @@ fn generate_projections( } else { ( quote! { - #vis struct __Projection #generics_with_pin_lt + #vis struct __Projection #generics_with_this_lt #whr { #(#fields_decl)* - ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, + __this: ::core::marker::PhantomData<&'__this mut #ident #ty_generics>, } }, quote! { __Projection { #(#fields_proj)* - ___pin_phantom_data: ::core::marker::PhantomData, + __this: ::core::marker::PhantomData, } }, ) @@ -428,9 +428,9 @@ impl #impl_generics #ident #ty_generics /// These fields are **not** structurally pinned: #(#[doc = #not_structurally_pinned_fields_docs])* #[inline] - #vis fn project<'__pin>( - self: ::core::pin::Pin<&'__pin mut Self>, - ) -> __Projection #ty_generics_with_pin_lt { + #vis fn project<'__this>( + self: ::core::pin::Pin<&'__this mut Self>, + ) -> __Projection #ty_generics_with_this_lt { // SAFETY: we only give access to `&mut` for fields not structurally pinned. let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; #projection_init From 0d9aa4b994379c6e0099737221ed421deb7e1a31 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 6 Oct 2026 17:29:38 +0200 Subject: [PATCH 53/91] rust: pin-init: internal: pin_data: parse into `StructInfo` before generating Organize parsing and generation into two separate phases, with a single `StructInfo` in the middle. This allows analysis and processing to happen with a single struct and avoid having to pass different information into the token stream generation functions. A drawback of this approach is that as we consume the `Fields`, we cannot expand the original struct using `quote!(#struct_)` anymore, and the field needs to be re-constructed in `generate_struct_def` function. However, this is only a temporary drawback, as the upcoming pin-init self-referential feature will require modification of the generated struct anyway. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261006-dev-selfref-dep-v1-8-da716f740d6d@garyguo.net Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 162 +++++++++++++++++-------- 1 file changed, 112 insertions(+), 50 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 6e18bc9e890a..03e893cf5475 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -8,8 +8,7 @@ punctuated::Punctuated, spanned::Spanned, visit_mut::VisitMut, - Field, Fields, Generics, Ident, Index, Item, ItemStruct, Member, PathSegment, Type, TypePath, - Visibility, + Field, Fields, Generics, Index, Item, ItemStruct, Member, PathSegment, Type, TypePath, }; use crate::{ @@ -49,12 +48,19 @@ fn to_tokens(&self, tokens: &mut TokenStream) { } } -struct FieldInfo<'a> { - field: &'a Field, +struct FieldInfo { + field: Field, member: Member, pinned: bool, } +struct StructInfo { + args: Args, + struct_: ItemStruct, + fields: Vec, + is_tuple_struct: bool, +} + pub(crate) fn expand_with_cfg( args: Args, input: Item, @@ -141,11 +147,11 @@ fn expand( replacer.visit_fields_mut(&mut struct_.fields); let is_tuple_struct = matches!(struct_.fields, Fields::Unnamed(_)); - let fields: Vec> = struct_ + let fields: Vec = struct_ .fields - .iter_mut() + .into_iter() .enumerate() - .map(|(index, field)| { + .map(|(index, mut field)| { let pinned = field.attrs.extract_single_attr(dcx, "pin").is_some(); assert!( @@ -161,17 +167,25 @@ fn expand( }; FieldInfo { - field: &*field, + field, member, pinned, } }) .collect(); - for field in &fields { + struct_.fields = Fields::Unit; + let info = StructInfo { + args, + struct_, + fields, + is_tuple_struct, + }; + + for field in &info.fields { if !field.pinned && is_phantom_pinned(&field.field.ty) { dcx.warn( - field.field, + &field.field, format!( "The field {} of type `PhantomPinned` only has an effect \ if it has the `#[pin]` attribute", @@ -181,20 +195,14 @@ fn expand( } } - let unpin_impl = generate_unpin_impl(&struct_.ident, &struct_.generics, &fields); - let drop_impl = generate_drop_impl(&struct_.ident, &struct_.generics, args); - let projections = generate_projections( - &struct_.vis, - &struct_.ident, - &struct_.generics, - is_tuple_struct, - &fields, - ); - let the_pin_data = - generate_the_pin_data(&struct_.vis, &struct_.ident, &struct_.generics, &fields); + let struct_def = generate_struct_def(&info); + let unpin_impl = generate_unpin_impl(&info); + let drop_impl = generate_drop_impl(&info); + let projections = generate_projections(&info); + let the_pin_data = generate_the_pin_data(&info); Ok(quote! { - #struct_ + #struct_def // We put the rest into this const item, because it then will not be accessible to anything // outside. const _: () = { @@ -230,17 +238,63 @@ fn is_phantom_pinned(ty: &Type) -> bool { } } -fn generate_unpin_impl( - ident: &Ident, - generics: &Generics, - fields: &[FieldInfo<'_>], -) -> TokenStream { +fn generate_struct_def(info: &StructInfo) -> TokenStream { + let ItemStruct { + attrs, + vis, + struct_token, + ident, + generics, + fields: _, + semi_token, + } = &info.struct_; + + let generated_fields = info.fields.iter().map(|field| { + let Field { + attrs, + vis, + mutability: _, + ident, + colon_token, + ty, + } = &field.field; + + quote! { + #(#attrs)* #vis #ident #colon_token #ty + } + }); + + let whr = &generics.where_clause; + + if info.is_tuple_struct { + quote!( + #(#attrs)* + #vis + #struct_token #ident #generics (#(#generated_fields,)*) #whr + #semi_token + ) + } else { + quote!( + #(#attrs)* + #vis + #struct_token #ident #generics #whr { + #(#generated_fields,)* + } + #semi_token + ) + } +} + +fn generate_unpin_impl(info: &StructInfo) -> TokenStream { + let ItemStruct { + generics, ident, .. + } = &info.struct_; let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let predicates = whr .map(|x| &x.predicates) .unwrap_or(const { &Punctuated::new() }); - let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { + let pinned_fields = info.fields.iter().filter(|f| f.pinned).map(|f| { let ident = f.member.as_ident(); let ty = &f.field.ty; quote!( @@ -271,9 +325,12 @@ impl #impl_generics ::core::marker::Unpin for #ident #ty_generics } } -fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenStream { +fn generate_drop_impl(info: &StructInfo) -> TokenStream { + let ItemStruct { + generics, ident, .. + } = &info.struct_; let (impl_generics, ty_generics, whr) = generics.split_for_impl(); - let has_pinned_drop = matches!(args, Args::PinnedDrop(_)); + let has_pinned_drop = matches!(info.args, Args::PinnedDrop(_)); // We need to disallow normal `Drop` implementation, the exact behavior depends on whether // `PinnedDrop` was specified in `args`. if has_pinned_drop { @@ -318,13 +375,13 @@ impl #impl_generics } } -fn generate_projections( - vis: &Visibility, - ident: &Ident, - generics: &Generics, - is_tuple_struct: bool, - fields: &[FieldInfo<'_>], -) -> TokenStream { +fn generate_projections(info: &StructInfo) -> TokenStream { + let ItemStruct { + vis, + ident, + generics, + .. + } = &info.struct_; let this_lt_generics: Generics = parse_quote!(<'__this>); let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); @@ -333,14 +390,15 @@ fn generate_projections( let this = format_ident!("this"); - let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields + let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = info + .fields .iter() .map(|field| { let Field { vis, ty, .. } = &field.field; let member = &field.member; // The projection of a tuple struct is a tuple struct itself, so its fields are // positional and must not be named. - let name = (!is_tuple_struct).then(|| { + let name = (!info.is_tuple_struct).then(|| { let ident = field.member.as_ident(); quote!(#ident:) }); @@ -367,16 +425,18 @@ fn generate_projections( } }) .collect(); - let structurally_pinned_fields_docs = fields + let structurally_pinned_fields_docs = info + .fields .iter() .filter(|f| f.pinned) .map(|f| format!(" - {}", f.member.display_name())); - let not_structurally_pinned_fields_docs = fields + let not_structurally_pinned_fields_docs = info + .fields .iter() .filter(|f| !f.pinned) .map(|f| format!(" - {}", f.member.display_name())); let docs = format!(" Pin-projections of [`{ident}`]"); - let (projection_def, projection_init) = if is_tuple_struct { + let (projection_def, projection_init) = if info.is_tuple_struct { ( quote! { #vis struct __Projection #generics_with_this_lt ( @@ -439,22 +499,24 @@ impl #impl_generics #ident #ty_generics } } -fn generate_the_pin_data( - vis: &Visibility, - struct_name: &Ident, - generics: &Generics, - fields: &[FieldInfo<'_>], -) -> TokenStream { +fn generate_the_pin_data(info: &StructInfo) -> TokenStream { + let ItemStruct { + vis, + ident: struct_name, + generics, + .. + } = &info.struct_; let (impl_generics, ty_generics, whr) = generics.split_for_impl(); // For every field, we create an initializing projection function according to its projection // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with // `Unpinned` which allows initialization via `Init`. - let field_accessors = fields + let field_accessors = info + .fields .iter() .map(|f| { - let Field { vis, ty, .. } = f.field; + let Field { vis, ty, .. } = &f.field; let field_name = f.member.as_ident(); let member = &f.member; let pin_marker = if f.pinned { From d45ef897ecf8d34ca25a3772552a8c75260fe5e1 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 8 Oct 2026 14:15:29 +0200 Subject: [PATCH 54/91] kbuild: rust: allow `clippy::comparison_chain` globally The lint would suggest comparison chains to be converted to match on `Ordering` instead. The suggestions do not always produce more readable code, and also it can produce false-positive when not all ordering has a branch. The issue is known upstream and thus it has been downgraded from style lints to pedantic lints, which makes it allow-by-default since Rust 1.87.0. The current kernel MSRV is 1.85.0, and thus the lint still triggers; allow it globally to match the updated upstream behaviour. Link: https://rust-lang.zulipchat.com/#narrow/channel/257328-clippy/topic/demote.20some.20lints.20to.20.60pedantic.60 Link: https://rust-lang.github.io/rust-clippy/master/#comparison_chain Acked-by: Benno Lossin Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20261008-dev-selfref-v2-1-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 0f1b80100b47..2839c6c5045b 100644 --- a/Makefile +++ b/Makefile @@ -476,6 +476,7 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS) # These flags apply to all Rust code in the tree, including the kernel and # host programs. # +# `-Aclippy::comparison_chain`: the lint has been demoted since 1.87. # `-Aclippy::unwrap_or_default`: the lint is buggy [1] and ignores our # MSRV. It can trigger depending on the optimization level. # [1] https://github.com/rust-lang/rust-clippy/issues/17379 @@ -494,6 +495,7 @@ export rust_common_flags := --edition=2021 \ -Wclippy::cast_lossless \ -Aclippy::collapsible_if \ -Aclippy::collapsible_match \ + -Aclippy::comparison_chain \ -Wclippy::ignored_unit_patterns \ -Aclippy::incompatible_msrv \ -Wclippy::mut_mut \ From 414b5c482e910335d1b2698711fb9da03a89ac93 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 30 Apr 2026 18:49:47 +0100 Subject: [PATCH 55/91] rust: pin-init: internal: pin_data: infer self-referential struct As a first step towards adding self-referential data structures in pin-init, add parsing support. Scan all field types for unbounded lifetimes, and if the names that of fields, it is inferred as a self-referential field lifetime. No explicit annotations are supported yet. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-2-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 162 ++++++++++++++++++++++++- rust/pin-init/internal/src/util.rs | 94 +++++++++++++- 2 files changed, 253 insertions(+), 3 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 03e893cf5475..cf6142cd656d 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +use std::collections::{BTreeMap, BTreeSet}; + use proc_macro2::TokenStream; use quote::{format_ident, quote, ToTokens}; use syn::{ @@ -7,8 +9,10 @@ parse_quote, parse_quote_spanned, punctuated::Punctuated, spanned::Spanned, + visit::Visit, visit_mut::VisitMut, - Field, Fields, Generics, Index, Item, ItemStruct, Member, PathSegment, Type, TypePath, + Field, Fields, Generics, Ident, Index, Item, ItemStruct, Lifetime, Member, PathSegment, Type, + TypePath, }; use crate::{ @@ -48,10 +52,69 @@ fn to_tokens(&self, tokens: &mut TokenStream) { } } +/// Description of how a field is borrowed. +#[derive(Clone, Copy, Default, PartialEq, Eq)] +enum BorrowedKind { + /// Implicitly inferreed. + #[default] + Shared, +} + +/// Information about a borrowed field. +#[expect(unused)] +struct BorrowedInfo { + kind: BorrowedKind, + /// Field lifetime for this field. + lifetime: Lifetime, +} + +#[derive(Clone, Copy, Default, PartialEq, Eq)] +enum Variance { + /// Implicitly inferred variance. + #[default] + Covariant, +} + +/// Information about field lifetimes captured in a type. +#[expect(unused)] +struct Capture { + variance: Variance, + /// Lifetime to be captured. + lifetime: Lifetime, +} + +impl std::borrow::Borrow for Capture { + fn borrow(&self) -> &Lifetime { + &self.lifetime + } +} + +impl PartialEq for Capture { + fn eq(&self, other: &Self) -> bool { + self.lifetime == other.lifetime + } +} + +impl Eq for Capture {} + +impl PartialOrd for Capture { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Capture { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lifetime.cmp(&other.lifetime) + } +} + struct FieldInfo { field: Field, member: Member, pinned: bool, + borrowed: Option, + captures: BTreeSet, } struct StructInfo { @@ -59,6 +122,7 @@ struct StructInfo { struct_: ItemStruct, fields: Vec, is_tuple_struct: bool, + self_referential: bool, } pub(crate) fn expand_with_cfg( @@ -147,7 +211,22 @@ fn expand( replacer.visit_fields_mut(&mut struct_.fields); let is_tuple_struct = matches!(struct_.fields, Fields::Unnamed(_)); - let fields: Vec = struct_ + + // Collect all bound lifetimes from generics. + let bound_lifetimes: BTreeSet<&Lifetime> = + struct_.generics.lifetimes().map(|x| &x.lifetime).collect(); + // Collect all fields. + let field_idx_map: BTreeMap = struct_ + .fields + .iter() + .enumerate() + .filter_map(|(index, field)| Some((field.ident.clone()?, index))) + .collect(); + + // Keep track on fields being implicitly borrowed by being mentioned. + let mut implicitly_borrowed = BTreeSet::new(); + + let mut fields: Vec = struct_ .fields .into_iter() .enumerate() @@ -166,16 +245,88 @@ fn expand( }), }; + let mut captures = BTreeSet::new(); + let wildcard_variance = Variance::default(); + + // Infer lifetime based on the field referenced. + // Bound lifetimes from struct generics take priority. + // + // For example, + // ``` + // struct Foo<'a> { + // bar: &'a (), + // a: u32, + // } + // ``` + // would not be inferred as self-referential because `'a` is already bound by the + // struct generics. + Lifetime::visitor(|lt| { + if bound_lifetimes.contains(lt) || captures.contains(lt) { + return; + } + + if !field_idx_map.contains_key(<.ident) { + dcx.error( + lt, + format!("`{lt}` is neither a lifetime in generics nor a field name"), + ); + return; + } + + captures.insert(Capture { + variance: wildcard_variance, + lifetime: lt.clone(), + }); + }) + .visit_type(&field.ty); + + for capture in captures.iter() { + implicitly_borrowed.insert(capture.lifetime.ident.clone()); + } + FieldInfo { field, member, pinned, + borrowed: None, + captures, } }) .collect(); + for field_name in implicitly_borrowed.into_iter() { + let field = &mut fields[field_idx_map[&field_name]]; + + // If field is not explicit marked as borrowed, infer a shared borrow. + if field.borrowed.is_none() { + field.borrowed = Some(BorrowedInfo { + kind: BorrowedKind::Shared, + // Obtaining from `field` instead of `field_name` for the correct span. + lifetime: Lifetime::from_ident(&field.member.as_ident()), + }); + } + } + + // Check that field lifetimes do not appear in the bounds. + Lifetime::visitor(|lt| { + if bound_lifetimes.contains(<) { + return; + } + + if field_idx_map.contains_key(<.ident) { + // Forbid the use of field lifetimes within bounds. + dcx.error(lt, "field lifetimes cannot be used in bounds"); + } + + // Otherwise this is completely unbound. Let Rust compiler produce that error instead. + }) + .visit_generics(&struct_.generics); + struct_.fields = Fields::Unit; let info = StructInfo { + self_referential: fields + .iter() + .any(|f| !f.captures.is_empty() || f.borrowed.is_some()), args, struct_, fields, @@ -195,6 +346,13 @@ fn expand( } } + if info.self_referential { + dcx.error( + &info.struct_.ident, + "self-referential support is not fully implemented", + ); + } + let struct_def = generate_struct_def(&info); let unpin_impl = generate_unpin_impl(&info); let drop_impl = generate_drop_impl(&info); diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index 3ce498cea754..67ebb333710f 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -1,8 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +use std::collections::BTreeSet; + use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, ToTokens}; -use syn::{Attribute, GenericParam, Generics, Index, Member, Token}; +use syn::{ + visit::Visit, Attribute, BoundLifetimes, GenericParam, Generics, Index, Lifetime, Member, Token, +}; use crate::DiagCtxt; @@ -237,3 +241,91 @@ fn to_tokens(&self, tokens: &mut TokenStream) { .to_tokens(tokens); } } + +pub(crate) trait LifetimeExt { + /// Get a visitor that call the provided function for all unbound lifetimes. + fn visitor<'a>(f: impl FnMut(&'a Lifetime)) -> impl Visit<'a>; + + /// Obtain a lifetime from a identifier. + /// + /// The created lifetime has the same span. + fn from_ident(ident: &Ident) -> Self; +} + +impl LifetimeExt for Lifetime { + fn visitor<'a>(f: impl FnMut(&'a Lifetime)) -> impl Visit<'a> { + LifetimeVisitor { + bound: BTreeSet::new(), + visit: f, + } + } + + fn from_ident(ident: &Ident) -> Self { + Lifetime { + apostrophe: ident.span(), + ident: ident.clone(), + } + } +} + +struct LifetimeVisitor<'a, F> { + bound: BTreeSet<&'a Lifetime>, + visit: F, +} + +impl<'a, F> LifetimeVisitor<'a, F> { + fn with_bound_lifetimes( + &mut self, + bound: Option<&'a BoundLifetimes>, + f: impl FnOnce(&mut Self), + ) { + // In case the type includes a lifetime binder, e.g. `dyn for<'a> Foo`, + // the lifetimes in the binder are bound and should not be visited. + + let mut to_remove = Vec::new(); + if let Some(bound) = bound { + for lt in &bound.lifetimes { + let GenericParam::Lifetime(lt) = lt else { + continue; + }; + if !self.bound.contains(&<.lifetime) { + self.bound.insert(<.lifetime); + to_remove.push(<.lifetime); + } + } + } + + f(self); + + for lt in to_remove { + self.bound.remove(lt); + } + } +} + +impl<'a, F: FnMut(&'a Lifetime)> Visit<'a> for LifetimeVisitor<'a, F> { + fn visit_lifetime(&mut self, lt: &'a Lifetime) { + if lt.ident == "static" { + return; + } + + if !self.bound.contains(lt) { + (self.visit)(lt); + } + } + + fn visit_trait_bound(&mut self, bound: &'a syn::TraitBound) { + self.with_bound_lifetimes(bound.lifetimes.as_ref(), |this| { + this.visit_path(&bound.path) + }); + } + + fn visit_type_bare_fn(&mut self, bare_fn: &'a syn::TypeBareFn) { + self.with_bound_lifetimes(bare_fn.lifetimes.as_ref(), |this| { + for input in bare_fn.inputs.iter() { + this.visit_bare_fn_arg(input); + } + this.visit_return_type(&bare_fn.output); + }); + } +} From 9ad9425015e267f6fccc065980f88625d0fa6fcb Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 1 May 2026 18:32:29 +0100 Subject: [PATCH 56/91] rust: pin-init: internal: pin_data: rewrite fields that borrow others Fields that borrow other fields have lifetimes that are within the struct and these are not part of the struct generics. Therefore, these fields need to have their lifetime erased. A naive implementation would be to replace their lifetimes with `'static`. However, doing so is unsound for multiple reasons: * Users may directly access such field with field access syntax, and get exposed with wrong lifetime; * Auto trait implementations will cause the struct to be implementing auto traits when the type only implements the auto trait for specific lifetime. This is similar to how specialization can be unsound if specialized on lifetime. Create a `Erase` type, which has `for<'a> fn(&'a ()) -> Foo<'a>` as generic parameter. Internally, it uses a helper trait to resolve that to `Foo<'static>`. The first issue is solved by not exposing any public accessor on that type. The second issue is solved by add custom `Send` and `Sync` implementations that requires `Send` to be implemented for all lifetimes, thus closing the lifetime specialization hole. The actual implementation is a bit more convoluted because it supports erasing multiple lifetimes. This is more or less a stable polyfill of the unstable `unsafe_binder` feature, without `unsafe_binders`'s no drop glue requirement. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-3-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 16 ++++++ rust/pin-init/src/__internal.rs | 77 ++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index cf6142cd656d..36678843251c 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -417,6 +417,22 @@ fn generate_struct_def(info: &StructInfo) -> TokenStream { ty, } = &field.field; + let mut ty = ty.to_token_stream(); + + // Replace lifetime for self-referential fields. + if !field.captures.is_empty() { + // Build a chain `for<'a> fn(&'a ()) -> ... -> (Ty,)`. Such type will have a `EraseLt` + // implementation and thus may be used inside `Erase`. + ty = quote!((#ty,)); + + for borrow in field.captures.iter().rev() { + let lt = &borrow.lifetime; + ty = quote!(for<#lt> fn(&#lt()) -> #ty); + } + + ty = quote!(::pin_init::__internal::Erase<#ty>); + }; + quote! { #(#attrs)* #vis #ident #colon_token #ty } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index cba53b8c3c94..0a8247473cbc 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -385,3 +385,80 @@ unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { Err(()) } } +/// Polyfill of `FnOnce` trait to be able to reference output via associated type. +pub trait FnOutput { + type Output; +} + +macro_rules! impl_fn_output { + () => {}; + ($ret:ident, $($arg:ident,)*) => { + impl FnOutput<($($arg,)*)> for This + where + This: FnOnce($($arg,)*) -> $ret, + { + type Output = $ret; + } + impl_fn_output!($($arg,)*); + }; +} + +impl_fn_output!(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U,); + +/// Lifetime erasure facility. +/// +/// Say we have `exists<'a, 'b> Foo<'a, 'b>` and we want to store it. There's no concrete +/// lifetimes we can use, so we want to erase the lifetime. +/// +/// Such erasure can be encoded as +/// `Erased fn(&'a ()) -> for<'b> fn(&'b()) -> (Foo<'a, 'b>,)`. +/// +/// This can be considered the stable version of Rust's `unsafe_binder` feature, without the +/// no-drop-glue requirement. +#[repr(transparent)] +#[allow(private_bounds)] +pub struct Erase(F::Erased); + +/// Helper trait to resolve the erased lifetime. +trait EraseLt { + type Erased; +} + +impl EraseLt for (T,) { + type Erased = T; +} + +impl EraseLt for T +where + T: for<'a> FnOutput<(&'a (),), Output: EraseLt>, +{ + type Erased = <>::Output as EraseLt>::Erased; +} + +// The default `Send` and `Sync` are not sufficient, because one can use lifetime specialization to +// implement `Send` or `Sync` for a concrete instance of lifetime. Use HRTB to ensure that the type +// will only implement `Send` or `Sync` if it's implemented for *all* erased lifetimes. + +// SAFETY: Trivial, no lifetime to erase. +unsafe impl Send for Erase<(T,)> {} + +// SAFETY: If we erased a lifetime, then the type needs to be `Send` for across *all* that +// lifetimes. +unsafe impl Send for Erase +where + F: for<'a> FnOutput<(&'a (),), Output: EraseLt>, + for<'a> Erase<>::Output>: Send, +{ +} + +// SAFETY: Trivial, no lifetime to erase. +unsafe impl Sync for Erase<(T,)> {} + +// SAFETY: If we erased a lifetime, then the type needs to be `Sync` for across *all* that +// lifetimes. +unsafe impl Sync for Erase +where + F: for<'a> FnOutput<(&'a (),), Output: EraseLt>, + for<'a> Erase<>::Output>: Sync, +{ +} From 73511e489bb0cb41d65a99b6f9df4f73a2f4827a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 22 Sep 2026 21:42:34 +0100 Subject: [PATCH 57/91] rust: pin-init: internal: pin_data: pin borrowed fields with wrapper For fields that are borrowed, a mutable reference to the struct no longer mean that it has the permission to access these fields. Therefore, the memory that they refer to must be pinned. Wrap these fields inside a `Borrowed` struct which pins it. They may be accessed directly (if they're not themselves referencing other struct fields), so implement a `Deref`. As such fields are always pinned, there is no need to generate a conditional `Unpin` implementations that implements `Unpin` when all fields are. Simply generate a never satisfiable `Unpin` implementation to prevent user from adding their own. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-4-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 19 +++++++++++++++++++ rust/pin-init/src/__internal.rs | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 36678843251c..a3e492c455da 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -433,6 +433,10 @@ fn generate_struct_def(info: &StructInfo) -> TokenStream { ty = quote!(::pin_init::__internal::Erase<#ty>); }; + if field.borrowed.is_some() { + ty = quote!(::pin_init::__internal::Borrowed<#ty>); + } + quote! { #(#attrs)* #vis #ident #colon_token #ty } @@ -468,6 +472,20 @@ fn generate_unpin_impl(info: &StructInfo) -> TokenStream { .map(|x| &x.predicates) .unwrap_or(const { &Punctuated::new() }); + if info.self_referential { + // Self-referential structs must always be pinned. + return quote! { + #[doc(hidden)] + impl #impl_generics ::core::marker::Unpin for #ident #ty_generics + where + // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds` + // feature . + for<'__dummy> ::core::marker::PhantomPinned: ::core::marker::Unpin, + #predicates + {} + }; + } + let pinned_fields = info.fields.iter().filter(|f| f.pinned).map(|f| { let ident = f.member.as_ident(); let ty = &f.field.ty; @@ -475,6 +493,7 @@ fn generate_unpin_impl(info: &StructInfo) -> TokenStream { #ident: #ty ) }); + quote! { // This struct will be used for the unpin analysis. It is needed, because only structurally // pinned fields are relevant whether the struct should implement `Unpin`. diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 0a8247473cbc..74dc23506d97 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -5,6 +5,9 @@ //! These items must not be used outside of this crate and the pin-init-internal crate located at //! `../internal`. +use core::marker::PhantomPinned; +use core::ops::Deref; + use super::*; /// Zero-sized type used to mark a type as invariant. @@ -462,3 +465,19 @@ unsafe impl Sync for Erase for<'a> Erase<>::Output>: Sync, { } + +/// Wrapper for borrowed fields. +/// +/// This should be switched to `UnsafePinned` when it is stable. +/// NOTE: This type needs to be covariant; Rust's 1.89+'s `UnsafePinned` is invariant. +#[repr(transparent)] +pub struct Borrowed(PhantomPinned, T); + +impl Deref for Borrowed { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.1 + } +} From 29213c00ccaa55e8ba01cfc79a93e000b0fcacc4 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 22 Sep 2026 22:58:11 +0100 Subject: [PATCH 58/91] rust: pin-init: internal: pin_data: teach drop check about generics that cannot dangle Lifetimes not needed by drop glue are considered by Rust's drop check to be considered `#[may_dangle]`. In case for a self-referential struct, we may have fields which need lifetime of borrowed fields in their drop glue, so compiler's automatic check is insufficient. Code like this: #[pin_data] struct SelfRef<'a> { borrow: PrintOnDrop<&'owner str>, owner: &'a str, } may access `owner` during the drop, however Rust will determine that since `'a` only is used in `owner`, the `'a` lifetime may dangle during drop. This is undesirable for pin-init self references, because `&'a str` could be coerced to `&'owner str` and this could further coerce if there're implied outlives, e.g. `&'earlier_field &'owner ()` would allow `&'owner str` to further coerce to `&'earlier_field`. Thus, if any self-referential field require field lifetime access in `Drop` impl, we would need to ensure that the all generic parameters visible by self-referential fields would strictly outlive the struct. And this can be done by a simple `Drop` impl that does nothing. Without a dropck eye patch, presence of `Drop` impl, albeit empty, tells the drop check that the strict outlive relation is needed. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-5-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/src/__internal.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 74dc23506d97..32bd6d8c39a1 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -473,6 +473,35 @@ unsafe impl Sync for Erase #[repr(transparent)] pub struct Borrowed(PhantomPinned, T); +// Lifetimes not needed by drop glue are considered by Rust's drop check to be considered +// `#[may_dangle]`. In case for a self-referential struct, we may have fields which need lifetime of +// borrowed fields in their drop glue, so compiler's automatic check is insufficient. +// +// Code like this: +// ``` +// #[pin_data] +// struct SelfRef<'a> { +// borrow: PrintOnDrop<&'owner str>, +// owner: &'a str, +// } +// ``` +// may access `owner` during the drop, however Rust will determine that since `'a` only is used in +// `owner`, the `'a` lifetime may dangle during drop. +// +// This is undesirable for pin-init self references, because `&'a str` could be coerecd to +// `&'owner str` and this could further coerce if there're implied outlives, e.g. +// `&'earlier_field &'owner ()` would allow `&'owner str` to further coerce to `&'earlier_field`. +// +// Thus, if any self-referential field require field lifetime access in `Drop` impl, we would need +// to ensure that the all generic parameters visible by self-referential fields would strictly +// outlive the struct. And this can be done by a simple `Drop` impl that does nothing. Without a +// dropck eye patch, presence of `Drop` impl, albeit empty, tells the drop check that the strict +// outlive relation is needed. +impl Drop for Borrowed { + #[inline(always)] + fn drop(&mut self) {} +} + impl Deref for Borrowed { type Target = T; From 71293ee9f3a8b222dc72bfa41275bd05c29eedbd Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 1 May 2026 20:50:50 +0100 Subject: [PATCH 59/91] rust: pin-init: internal: pin_data: self-referential drop order checks Check drop order to ensure that usage of lifetime inside self-referential struct is consistent with the order that the fields will dropped in drop glue. First, fields are checked according to their index to ensure that if `a` borrows from `b`, `b` must outlive `a`. This is simple and produces a very good diagnostic when misused. Lifetime bounds can also be indirectly crafted with implied bounds that make fields well-formed. For example, in this struct struct Foo { x: &'b &'a (), a: String, y: PrintOnDrop<&'b str>, b: String, } `&'b &'a ()` will imply that `a` outlive `b`, which is inconsistent with the actual drop order. For this case, create a `__drop_order_check` function with field lifetimes and outlive relationship of them as generic parameter, and ask Rust to prove that the types are well-formed inside the generated function, to ensure that the bad implied bounds cannot happen. The `__drop_order_check` also need to correlate lifetimes or types captured by generics and the field lifetimes. Do this by inserting outlive bounds when a field mentions a specific type or lifetime parameter. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-6-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 207 ++++++++++++++++++++++++- rust/pin-init/internal/src/util.rs | 65 +++++++- 2 files changed, 263 insertions(+), 9 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index a3e492c455da..ae0e169a00c6 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -2,8 +2,8 @@ use std::collections::{BTreeMap, BTreeSet}; -use proc_macro2::TokenStream; -use quote::{format_ident, quote, ToTokens}; +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ parse::{End, Nothing, Parse}, parse_quote, parse_quote_spanned, @@ -11,8 +11,8 @@ spanned::Spanned, visit::Visit, visit_mut::VisitMut, - Field, Fields, Generics, Ident, Index, Item, ItemStruct, Lifetime, Member, PathSegment, Type, - TypePath, + Field, Fields, GenericParam, Generics, Ident, Index, Item, ItemStruct, Lifetime, LifetimeParam, + Member, PathSegment, Type, TypePath, }; use crate::{ @@ -115,14 +115,19 @@ struct FieldInfo { pinned: bool, borrowed: Option, captures: BTreeSet, + generic_lt_captures: BTreeSet, + generic_ty_captures: BTreeSet, } struct StructInfo { args: Args, struct_: ItemStruct, fields: Vec, + field_idx_map: BTreeMap, is_tuple_struct: bool, self_referential: bool, + /// Field lifetime generics. + field_lts: Generics, } pub(crate) fn expand_with_cfg( @@ -215,6 +220,8 @@ fn expand( // Collect all bound lifetimes from generics. let bound_lifetimes: BTreeSet<&Lifetime> = struct_.generics.lifetimes().map(|x| &x.lifetime).collect(); + // Collect all type parameters from generics. + let type_params: BTreeSet<&Ident> = struct_.generics.type_params().map(|x| &x.ident).collect(); // Collect all fields. let field_idx_map: BTreeMap = struct_ .fields @@ -248,6 +255,9 @@ fn expand( let mut captures = BTreeSet::new(); let wildcard_variance = Variance::default(); + let mut generic_lt_captures = BTreeSet::new(); + let mut generic_ty_captures = BTreeSet::new(); + // Infer lifetime based on the field referenced. // Bound lifetimes from struct generics take priority. // @@ -261,7 +271,12 @@ fn expand( // would not be inferred as self-referential because `'a` is already bound by the // struct generics. Lifetime::visitor(|lt| { - if bound_lifetimes.contains(lt) || captures.contains(lt) { + if bound_lifetimes.contains(lt) { + generic_lt_captures.insert(lt.clone()); + return; + } + + if captures.contains(lt) { return; } @@ -284,12 +299,21 @@ fn expand( implicitly_borrowed.insert(capture.lifetime.ident.clone()); } + GenericParam::maybe_type_params_visitor(|ident| { + if type_params.contains(ident) { + generic_ty_captures.insert(ident.clone()); + } + }) + .visit_type(&field.ty); + FieldInfo { field, member, pinned, borrowed: None, captures, + generic_lt_captures, + generic_ty_captures, } }) .collect(); @@ -322,6 +346,58 @@ fn expand( }) .visit_generics(&struct_.generics); + // Create a lifetime parameter for each field. + let borrowed_fields: Vec<_> = fields.iter().filter_map(|f| f.borrowed.as_ref()).collect(); + let mut field_lts = Generics { + lt_token: None, + params: borrowed_fields + .iter() + .map(|borrowed| { + GenericParam::Lifetime(LifetimeParam { + attrs: Vec::new(), + lifetime: borrowed.lifetime.clone(), + colon_token: None, + bounds: Default::default(), + }) + }) + .collect(), + gt_token: None, + where_clause: None, + }; + + // Insert necessary bounds to make types well-formed. + for field in fields.iter() { + let Some(borrowed) = &field.borrowed else { + continue; + }; + let field_lt = &borrowed.lifetime; + + // For each borrowed field that borrows from other fields, we need to insert outlive bounds. + for capture in &field.captures { + let lt = &capture.lifetime; + field_lts + .make_where_clause() + .predicates + .push(parse_quote!(#lt: #field_lt)); + } + + // For each borrowed field that references a generic, we also need to insert their outlive + // bounds so they can refer to generics. + for lt in field.generic_lt_captures.iter() { + field_lts + .make_where_clause() + .predicates + .push(parse_quote!(#lt: #field_lt)); + } + + for ty in field.generic_ty_captures.iter() { + field_lts + .make_where_clause() + .predicates + .push(parse_quote!(#ty: #field_lt)); + } + } + struct_.fields = Fields::Unit; let info = StructInfo { self_referential: fields @@ -330,7 +406,9 @@ fn expand( args, struct_, fields, + field_idx_map, is_tuple_struct, + field_lts, }; for field in &info.fields { @@ -356,6 +434,7 @@ fn expand( let struct_def = generate_struct_def(&info); let unpin_impl = generate_unpin_impl(&info); let drop_impl = generate_drop_impl(&info); + let drop_order_check = generate_drop_order_check(dcx, &info); let projections = generate_projections(&info); let the_pin_data = generate_the_pin_data(&info); @@ -364,6 +443,7 @@ fn expand( // We put the rest into this const item, because it then will not be accessible to anything // outside. const _: () = { + #drop_order_check #projections #the_pin_data #unpin_impl @@ -568,6 +648,123 @@ impl #impl_generics } } +fn generate_drop_order_check(dcx: &mut DiagCtxt, info: &StructInfo) -> TokenStream { + let ItemStruct { + ident: struct_name, + generics, + .. + } = &info.struct_; + + // If the struct is not self-referential then we can just skip. + if !info.self_referential { + return quote!(); + } + + // Make sure fields are dropped earlier than the fields that they borrow. + for (i, field) in info.fields.iter().enumerate() { + let ident = field.member.as_ident(); + for capture in &field.captures { + let borrowed_field = &capture.lifetime.ident; + + if let Some(&borrowed_idx) = info.field_idx_map.get(borrowed_field) { + if i == borrowed_idx { + // We need a strict outlive relationship, in case the lifetime is needed by the + // field's drop glue. + dcx.error( + borrowed_field, + format!("field `{ident}` cannot borrow from itself"), + ); + } else if i > borrowed_idx { + dcx.error( + borrowed_field, + format!("field `{ident}` borrows `{borrowed_field}`, but drops later"), + ); + } + } + } + } + + // The check above is necessary, but not sufficient. + // + // Consider this case: + // ``` + // struct Foo { + // x: &'b &'a (), + // a: String, + // y: PrintOnDrop<&'b str>, + // b: String, + // } + // ``` + // we need to ensure that `b` will strictly outlive `a`. + // + // Rust needs to ensure that types are well-formed; in the above example, `&'b &'a ()` is + // well-formed only if `a` outlive `b`. To avoid requiring everyone from having to express this + // bound explicitly when declaring a struct, the `'b: 'a` bound is inferred by the Rust + // compiler. However this causes an issue, where now `&'a str` can be coerced to `&'b str` + // because compiler thinks that it shorten the lifetime. We'll be able to put a reference to `a` + // into `y`; but `a` drops first, so when `y` drops, it accesses `a` and causes a + // use-after-free! + // + // Therefore, we must ensure the types contained within the struct has their implied bound being + // consistent with the actual lifetime relationship. We create a `__drop_order_check` function, + // with known lifetime bounds as bounds on the function, and asks Rust to *prove* that the types + // are wellformed, given the bounds that we understand. + + let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts, generics]); + + let (_, ty_generics, _) = generics.split_for_impl(); + let (impl_generics_with_field_lt, _, whr_with_field_lt) = + generics_with_field_lt.split_for_impl(); + + // Prove the wellformedness of struct fields with regarding to the bounds of + // `__drop_order_check`. + // + // Consider this case: + // ``` + // struct Foo { + // x: &'b &'a (), + // a: String, + // y: PrintOnDrop<&'b str>, + // b: String, + // } + // ``` + // we need to ensure that `b` will strictly outlive `a`. + // + // Rust needs to ensure that types are well-formed; in the above example, `&'b &'a ()` is + // well-formed only if `a` outlive `b`. To avoid requiring everyone from having to express this + // bound explicitly when declaring a struct, the `'b: 'a` bound is inferred by the Rust + // compiler. However this causes an issue, where now `&'a str` can be coerced to `&'b str` + // because compiler thinks that it shorten the lifetime. We'll be able to put a reference to `a` + // into `y`; but `a` drops first, so when `y` drops, it accesses `a` and causes a + // use-after-free! + // + // Rust needs to *prove* the wellformedness of the type below, taking into account only the + // explicitly defined bounds plus the bounds implied by the lifetime-erased struct (but not + // the full implied bound between the field lifetimes). + let wf_proofs = info.fields.iter().rev().map(|f| { + let ty = &f.field.ty; + let span = ty.span().resolved_at(Span::mixed_site()); + let ident = f.member.as_ident(); + let lt = f.borrowed.as_ref().map(|b| &b.lifetime); + quote_spanned!(span => + let #ident: &#lt mut #ty = loop {}; + ) + }); + + let struct_span = struct_name.span().resolved_at(Span::mixed_site()); + quote_spanned! {struct_span => + #[allow(non_snake_case, unused)] + fn __drop_order_check #impl_generics_with_field_lt ( + // This must be present so the function can *assume* the implied bounds on the erased + // struct. For example, if the struct has `&'a T`, Rust will infer `T: 'a`; we still + // want to assume these bounds as they are not relevant to the field lifetimes. + _: &#struct_name #ty_generics, + ) #whr_with_field_lt { + #(#wf_proofs)* + } + } +} + fn generate_projections(info: &StructInfo) -> TokenStream { let ItemStruct { vis, diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index 67ebb333710f..f19712a46a30 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -5,7 +5,8 @@ use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, ToTokens}; use syn::{ - visit::Visit, Attribute, BoundLifetimes, GenericParam, Generics, Index, Lifetime, Member, Token, + visit::Visit, Attribute, BoundLifetimes, GenericParam, Generics, Index, Lifetime, Member, + Token, TypePath, }; use crate::DiagCtxt; @@ -85,6 +86,7 @@ fn display_name(&self) -> String { pub(crate) struct CombinedGenerics<'a>(pub(crate) Vec<&'a Generics>); pub(crate) struct CombinedImplGenerics<'a>(&'a CombinedGenerics<'a>); pub(crate) struct CombinedTypeGenerics<'a>(&'a CombinedGenerics<'a>); +pub(crate) struct CombinedWhereClauses<'a>(&'a CombinedGenerics<'a>); impl CombinedGenerics<'_> { pub(crate) fn split_for_impl( @@ -92,10 +94,13 @@ pub(crate) fn split_for_impl( ) -> ( CombinedImplGenerics<'_>, CombinedTypeGenerics<'_>, - // A stub type so `split_for_impl` signature matches that of `syn`'s. - impl Sized, + CombinedWhereClauses<'_>, ) { - (CombinedImplGenerics(self), CombinedTypeGenerics(self), ()) + ( + CombinedImplGenerics(self), + CombinedTypeGenerics(self), + CombinedWhereClauses(self), + ) } } @@ -242,6 +247,31 @@ fn to_tokens(&self, tokens: &mut TokenStream) { } } +impl ToTokens for CombinedWhereClauses<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0 + .0 + .iter() + .filter_map(|x| Some(x.where_clause.as_ref()?.where_token)) + .next_back() + .unwrap_or_default() + .to_tokens(tokens); + + let comma: Token![,] = Default::default(); + + for generics in self.0 .0.iter() { + let Some(where_clause) = &generics.where_clause else { + continue; + }; + + where_clause.predicates.to_tokens(tokens); + if !where_clause.predicates.empty_or_trailing() { + comma.to_tokens(tokens); + } + } + } +} + pub(crate) trait LifetimeExt { /// Get a visitor that call the provided function for all unbound lifetimes. fn visitor<'a>(f: impl FnMut(&'a Lifetime)) -> impl Visit<'a>; @@ -329,3 +359,30 @@ fn visit_type_bare_fn(&mut self, bare_fn: &'a syn::TypeBareFn) { }); } } + +pub(crate) trait GenericParamExt { + fn maybe_type_params_visitor<'a>(f: impl FnMut(&'a Ident)) -> impl Visit<'a>; +} + +impl GenericParamExt for GenericParam { + fn maybe_type_params_visitor<'a>(f: impl FnMut(&'a Ident)) -> impl Visit<'a> { + struct TypeParamVisitor(F); + + impl<'a, F> Visit<'a> for TypeParamVisitor + where + F: FnMut(&'a Ident), + { + fn visit_type_path(&mut self, ty: &'a TypePath) { + if ty.qself.is_none() + && ty.path.leading_colon.is_none() + && ty.path.segments[0].arguments.is_none() + { + (self.0)(&ty.path.segments[0].ident); + } + syn::visit::visit_type_path(self, ty); + } + } + + TypeParamVisitor(f) + } +} From 5729b5908312576f0f626683aac0b5965df0f434 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 2 May 2026 22:08:21 +0100 Subject: [PATCH 60/91] rust: pin-init: internal: pin_data: check covariance of self-referential fields We implicitly infer covariance for fields that self-references. This needs to be checked to ensure that the fields are really covariant, so the rest of expansion code can rely on this fact. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-7-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 75 +++++++++++++++++++++++++- rust/pin-init/internal/src/util.rs | 24 ++++++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index ae0e169a00c6..84a1fbacac9a 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -76,7 +76,6 @@ enum Variance { } /// Information about field lifetimes captured in a type. -#[expect(unused)] struct Capture { variance: Variance, /// Lifetime to be captured. @@ -435,6 +434,7 @@ fn expand( let unpin_impl = generate_unpin_impl(&info); let drop_impl = generate_drop_impl(&info); let drop_order_check = generate_drop_order_check(dcx, &info); + let variance_check = generate_variance_check(&info); let projections = generate_projections(&info); let the_pin_data = generate_the_pin_data(&info); @@ -444,6 +444,7 @@ fn expand( // outside. const _: () = { #drop_order_check + #variance_check #projections #the_pin_data #unpin_impl @@ -765,6 +766,78 @@ fn __drop_order_check #impl_generics_with_field_lt ( } } +/// Produce variance checks, so we can ensure that the variance of lifetimes captured by field types +/// actually match our expectation. +fn generate_variance_check(info: &StructInfo) -> TokenStream { + if !info.self_referential { + return quote!(); + } + + let mut checks = Vec::new(); + + for f in info.fields.iter() { + let covariant_captures: Vec<_> = f + .captures + .iter() + .filter(|b| b.variance == Variance::Covariant) + .map(|b| &b.lifetime) + .collect(); + if covariant_captures.is_empty() { + continue; + } + + let ident = f.member.as_ident(); + // Use the span of type for better error message. + let span = f.field.ty.span().resolved_at(Span::mixed_site()); + + let other_field_lifetimes = Generics { + lt_token: None, + params: f + .captures + .iter() + .filter(|b| b.variance != Variance::Covariant) + .map(|b| GenericParam::Lifetime(LifetimeParam::new(b.lifetime.clone()))) + .collect(), + gt_token: None, + where_clause: None, + }; + + let long = Lifetime::new("'__long", span); + let long_ty = f + .field + .ty + .replace_lifetimes(&covariant_captures, &vec![&long; covariant_captures.len()]); + + let short = Lifetime::new("'__short", span); + let short_ty = f + .field + .ty + .replace_lifetimes(&covariant_captures, &vec![&short; covariant_captures.len()]); + + let check_name = format_ident!("__{ident}_covariance", span = span); + + // Add `<'__long: '__short, 'short>` as additional generics. + let covariance_check_generics = parse_quote!(<#long: #short, #short>); + let combined_generics = CombinedGenerics(vec![ + &covariance_check_generics, + &other_field_lifetimes, + &info.struct_.generics, + ]); + let (combined_impl_generics, _, whr) = combined_generics.split_for_impl(); + + checks.push(quote_spanned!(span => + // Emit a check to ensure the type is *really* covariant for soundness. + fn #check_name #combined_impl_generics (long: #long_ty) -> #short_ty #whr { + long + } + )); + } + + quote!( + #(#checks)* + ) +} + fn generate_projections(info: &StructInfo) -> TokenStream { let ItemStruct { vis, diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index f19712a46a30..59054f5934fb 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -5,8 +5,8 @@ use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, ToTokens}; use syn::{ - visit::Visit, Attribute, BoundLifetimes, GenericParam, Generics, Index, Lifetime, Member, - Token, TypePath, + parse_quote, visit::Visit, Attribute, BoundLifetimes, GenericParam, Generics, Index, Lifetime, + Member, Token, Type, TypePath, }; use crate::DiagCtxt; @@ -386,3 +386,23 @@ fn visit_type_path(&mut self, ty: &'a TypePath) { TypeParamVisitor(f) } } + +pub(crate) trait TypeExt { + fn replace_lifetimes(&self, needle: &[&Lifetime], replacement: &[&Lifetime]) -> Type; +} + +impl TypeExt for Type { + fn replace_lifetimes(&self, needle: &[&Lifetime], replacement: &[&Lifetime]) -> Type { + if needle.is_empty() { + return self.clone(); + } + + parse_quote!( + < + for<#(#needle,)*> fn(#(&#needle (),)*) -> #self + as + ::pin_init::__internal::FnOutput<(#(&#replacement (),)*)> + >::Output + ) + } +} From c7ec00c230b5014757a76f3f285c7fa880321163 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 25 Apr 2026 18:27:37 +0100 Subject: [PATCH 61/91] rust: pin-init: internal: pin_data: implement initialization of borrowed structs We now have the checks to ensure that lifetime relations are what is expected, we can generate the slot projections in `generate_pin_data` so self-referential struct can be implemented. New slot and guard types are defined (`SelfRefSlot` and `SelfRefDropGuard`) which gives the generated let bindings longer lifetime than the guard themselves. Have `__make_closure` take `data` back as an argument. This gives `#[pin_data]` an opportunity to change the type to add lifetimes. Higher-ranked trait bound on `__make_closure` is used to ensure that the initialization closure cannot make arbitrary assumptions of those lifetimes. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-8-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/init.rs | 42 +++++- rust/pin-init/internal/src/pin_data.rs | 116 +++++++++++++++-- rust/pin-init/src/__internal.rs | 169 ++++++++++++++++++++++++- rust/pin-init/src/lib.rs | 1 + 4 files changed, 306 insertions(+), 22 deletions(-) diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 80e4dc068d2e..ded2b5283376 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -288,8 +288,10 @@ fn assert_zeroable(_: *mut T) }, }; // `mixed_site` ensures that the data is not accessible to the user-controlled code. - let init_fields = init_fields(&fields, pinned); + let init_fields = make_field_init(&fields, pinned, false); + let drop_check = make_field_init(&fields, pinned, true); let field_check = make_field_check(&fields, init_kind, &path); + Ok(quote_spanned! { Span::mixed_site() => { // Get the data about fields from the supplied type. let data = { @@ -303,17 +305,29 @@ fn assert_zeroable(_: *mut T) // Ensure that `data` really is of type `data` and help with type inference: let init = data.__make_closure::<_, #error>( - move |slot| { + move |slot, data_lt| { #zeroable_check #this - #init_fields + // Generate init twice, which is mostly identical except for lifetimes. + if true { + // In this path, we use the field lifetime from HRTB to prevent environment + // lifetime from entering the fields, and to ensure that the dependency of + // fields is consistent with the field drop of the struct. + #init_fields + } else { + // In this path, we use local lifetime, to make sure that if initialization + // fails, the destructor execution will not cause lifetime issues. This is + // separate as implied bounds between field lifetimes can be inconsistent with + // that of the drop. + #drop_check + } #field_check // SAFETY: we are the `init!` macro that is allowed to call this. Ok(unsafe { ::pin_init::__internal::InitOk::new() }) } ); let init = move |slot| -> ::core::result::Result<(), #error> { - init(slot).map(|__InitOk| ()) + init(slot, data.__with_lt()).map(|__InitOk| ()) }; // SAFETY: TODO unsafe { ::pin_init::#init_from_closure::<_, #error>(init) } @@ -360,7 +374,11 @@ fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKi } /// Generate the code that initializes the fields of the struct using the initializers in `field`. -fn init_fields(fields: &Punctuated, pinned: bool) -> TokenStream { +fn make_field_init( + fields: &Punctuated, + pinned: bool, + dropck: bool, +) -> TokenStream { let mut forget_guards = vec![]; let mut res = TokenStream::new(); for InitializerField { attrs, kind } in fields { @@ -388,13 +406,18 @@ fn init_fields(fields: &Punctuated, pinned: bool) - let span = Span::mixed_site().located_at(ident.span()); let slot = if pinned { + let data = if !dropck { + quote_spanned!(span => data_lt) + } else { + quote_spanned!(span => data.__with_lt()) + }; quote_spanned! { span => // SAFETY: // - `slot` is valid and properly aligned. // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned. // - `make_field_check` prevents `#member` from being used twice, therefore // `(*slot).#member` is exclusively accessed and has not been initialized. - (unsafe { data.#ident(slot) }) + (unsafe { #data.#ident(slot) }) } } else { quote_spanned! { span => @@ -455,6 +478,11 @@ fn init_fields(fields: &Punctuated, pinned: bool) - // A tuple field has no name that could be bound here (the `_0` identifiers are considered // implementation detail and not user-facing). + let let_binding_method = if !dropck { + format_ident!("let_binding", span = span) + } else { + format_ident!("let_binding_in_dropck", span = span) + }; let binding = match member { Member::Named(ident) => quote_spanned! { span => #(#cfgs)* @@ -462,7 +490,7 @@ fn init_fields(fields: &Punctuated, pinned: bool) - // struct field. #[allow(unused_variables, non_snake_case)] // Include `mut` so that `Pin<&mut T>` bindings can be reborrowed via `.as_mut()`. - let mut #ident = #guard.let_binding(); + let mut #ident = #guard.#let_binding_method(); }, Member::Unnamed(_) => quote!(), }; diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 84a1fbacac9a..a46c4dcd7da5 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -61,7 +61,6 @@ enum BorrowedKind { } /// Information about a borrowed field. -#[expect(unused)] struct BorrowedInfo { kind: BorrowedKind, /// Field lifetime for this field. @@ -969,12 +968,36 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { generics, .. } = &info.struct_; - let (impl_generics, ty_generics, whr) = generics.split_for_impl(); - // For every field, we create an initializing projection function according to its projection - // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be - // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with - // `Unpinned` which allows initialization via `Init`. + // Wrap in `CombinedGenerics` because it's ty generics will always output `<>`, so it can be + // used with `for`. + let field_lts = CombinedGenerics(vec![&info.field_lts]); + let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts, generics]); + + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); + let (impl_generics_with_lt, ty_generics_with_field_lt, whr_with_field_lt) = + generics_with_field_lt.split_for_impl(); + + // Wrap each field in a `PhantomInvariant`. For borrowed fields, additionally + // use `&#lt mut #ty` so the `lt` becomes associated with `#ty` which deduces + // implied bounds. + let phantom_fields = info.fields.iter().map(|f| { + let ty = &f.field.ty; + let ident = f.member.as_ident(); + + if let Some(borrowed) = &f.borrowed { + let lt = &borrowed.lifetime; + quote!( + #ident: ::pin_init::__internal::PhantomInvariant<&#lt mut #ty>, + ) + } else { + quote!( + #ident: ::pin_init::__internal::PhantomInvariant<#ty>, + ) + } + }); + let field_accessors = info .fields .iter() @@ -987,6 +1010,30 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { } else { quote!(Unpinned) }; + + let (slot_ty, slot_arg) = match &f.borrowed { + None => (quote!(Slot), quote!()), + Some(BorrowedInfo { + kind: BorrowedKind::Shared, + lifetime, + }) => ( + // For borrowed fields, create a `SelfRefSlot`, which after initialization + // turns into a `SelfRefDropGuard` instead of `DropGuard`. + // + // They're mostly the same, except that `SelfRefDropGuard` returns `&'field T` + // instead of `&'guard T` for let bindings; this allows it to be used to be + // used to initialize other fields. + // + // The soundness of doing so relies on fact that `__make_init` requires a + // higher-ranked trait bound on the closure. Within the closure (which is the + // caller of the generated slot projection functions here), it can make no + // assumptions on the lifetime except for those implied by the struct's bounds, + // and we have validated them in `generate_drop_check`. + quote!(SelfRefSlot), + quote!(#lifetime,), + ), + }; + quote! { /// # Safety /// @@ -1001,19 +1048,54 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { #vis unsafe fn #field_name( self, slot: *mut #struct_name #ty_generics, - ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> { + ) -> ::pin_init::__internal::#slot_ty< + #slot_arg ::pin_init::__internal::#pin_marker, #ty + > { + // CAST: `as _` is needed to convert types wrapped inside `SelfRef`. // SAFETY: // - If `#pin_marker` is `Pinned`, the corresponding field is structurally // pinned. // - Other safety requirements follows the safety requirement. - unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#member) } + // - If `#slot_ty` is `SelfRefSlot`, the lifetime `#lt` represents that of the + // field. + unsafe { ::pin_init::__internal::#slot_ty::new(&raw mut (*slot).#member as _) } } } }) .collect::(); + quote! { - // We declare this struct which will host all of the projection function for our type. It - // will be invariant over all generic parameters which are inherited from the struct. + // We declare this struct which will host all of the projection function for our type. + #[doc(hidden)] + #[allow(non_snake_case)] + #vis struct __PinDataLt #generics_with_field_lt + #whr_with_field_lt + { + #(#phantom_fields)* + __pin_phantom: ::core::marker::PhantomData<#struct_name #ty_generics>, + } + + impl #impl_generics_with_lt ::core::clone::Clone for __PinDataLt #ty_generics_with_field_lt + #whr_with_field_lt + { + fn clone(&self) -> Self { *self } + } + + impl #impl_generics_with_lt ::core::marker::Copy for __PinDataLt #ty_generics_with_field_lt + #whr_with_field_lt + {} + + #[allow(dead_code)] // Some functions might never be used and private. + #[expect(clippy::missing_safety_doc)] + impl #impl_generics_with_lt __PinDataLt #ty_generics_with_field_lt + #whr_with_field_lt + { + #field_accessors + } + + // Declare a type that serves as the entry point of interaction with the `pin_init!` macro. + // We use this type instead of defining methods directly on user's type to avoid possibility + // of name conflicts. #[doc(hidden)] #vis struct __ThePinData #generics #whr @@ -1032,7 +1114,6 @@ impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics #whr {} - #[allow(dead_code)] // Some functions might never be used and private. impl #impl_generics __ThePinData #ty_generics #whr { @@ -1040,13 +1121,20 @@ impl #impl_generics __ThePinData #ty_generics #[inline(always)] #vis fn __make_closure<__F, __E>(self, f: __F) -> __F where - __F: FnOnce(*mut #struct_name #ty_generics) -> - ::core::result::Result<::pin_init::__internal::InitOk, __E>, + __F: for #field_lt_ty_generics ::core::ops::FnOnce( + *mut #struct_name #ty_generics, + __PinDataLt #ty_generics_with_field_lt + ) -> ::core::result::Result<::pin_init::__internal::InitOk, __E>, { f } - #field_accessors + #[inline(always)] + #vis fn __with_lt #field_lts(self) -> __PinDataLt #ty_generics_with_field_lt { + // Generate a zeroed to avoid naming all fields. + // SAFETY: `__PinDataLt` only contains phantom fields. + unsafe { ::core::mem::zeroed() } + } } // SAFETY: We have added the correct projection functions above to `__ThePinData` and diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 32bd6d8c39a1..b639c9204a7d 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -109,10 +109,15 @@ impl InitData { #[inline(always)] pub fn __make_closure(self, f: F) -> F where - F: FnOnce(*mut T) -> Result, + F: FnOnce(*mut T, Self) -> Result, { f } + + #[inline(always)] + pub fn __with_lt(self) -> Self { + self + } } /// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. @@ -322,6 +327,12 @@ pub fn let_binding(&mut self) -> &mut T { // SAFETY: Per type invariant. unsafe { &mut *self.ptr } } + + /// Create a let binding for accessor use in dropck. + #[inline] + pub fn let_binding_in_dropck(&mut self) -> &mut T { + self.let_binding() + } } impl DropGuard { @@ -332,6 +343,12 @@ pub fn let_binding(&mut self) -> Pin<&mut T> { // pinned per type invariant. unsafe { Pin::new_unchecked(&mut *self.ptr) } } + + /// Create a let binding for accessor use in dropck. + #[inline] + pub fn let_binding_in_dropck(&mut self) -> Pin<&mut T> { + self.let_binding() + } } impl Drop for DropGuard { @@ -342,6 +359,156 @@ fn drop(&mut self) { } } +/// Represent an uninitialized field in a pinned struct that will be referenced by other fields. +/// +/// # Invariants +/// +/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory +/// and will live longer than `'a`. +/// - If `P` is `Pinned`, then `ptr` is structurally pinned. +pub struct SelfRefSlot<'a, P, T: ?Sized> { + pub ptr: *mut T, + pub _phantom: PhantomData<(P, &'a mut T)>, +} + +impl<'a, P, T: ?Sized> SelfRefSlot<'a, P, T> { + /// # Safety + /// + /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed + /// memory and will live longer than `'a`. + /// - If `P` is `Pinned`, then `ptr` is structurally pinned. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + _phantom: PhantomData, + } + } + + /// Initialize the field by value. + #[inline] + pub fn write(self, value: T) -> SelfRefDropGuard<'a, P, T> + where + T: Sized, + { + // SAFETY: `self.ptr` is a valid and aligned pointer for write. + unsafe { self.ptr.write(value) } + // SAFETY: + // - `self.ptr` is valid, properly aligned and live longer than `'a` per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + // - If `P` is `Pinned`, `self.ptr` is pinned. + unsafe { SelfRefDropGuard::new(self.ptr) } + } +} + +impl<'a, T: ?Sized> SelfRefSlot<'a, Unpinned, T> { + /// Initialize the field. + #[inline] + pub fn init(self, init: impl Init) -> Result, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `ptr`; + // also `self` is consumed so it cannot be touched further. + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid, properly aligned and live longer than `'a` per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { SelfRefDropGuard::new(self.ptr) }) + } +} + +impl<'a, T: ?Sized> SelfRefSlot<'a, Pinned, T> { + /// Initialize the field. + #[inline] + pub fn init(self, init: impl PinInit) -> Result, E> { + // SAFETY: + // - `ptr` is valid + // - when `Err` is returned, we also propagate the error without touching `ptr`; + // also `self` is consumed so it cannot be touched further. + // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid, properly aligned and live longer than `'a` per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { SelfRefDropGuard::new(self.ptr) }) + } +} +/// When a value of this type is dropped, it drops a `T`. +/// +/// Can be forgotten to prevent the drop. +/// +/// # Invariants +/// +/// - `ptr` is valid, properly aligned and live longer than `'a`. +/// - `*ptr` is initialized and owned by this guard. +/// - if `P` is `Pinned`, `ptr` is pinned. +pub struct SelfRefDropGuard<'a, P, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<(P, &'a mut T)>, +} + +impl<'a, P, T: ?Sized> SelfRefDropGuard<'a, P, T> { + /// Creates a drop guard and transfer the ownership of the pointer content. + /// + /// The ownership is only relinquished if the guard is forgotten via [`core::mem::forget`]. + /// + /// # Safety + /// + /// - `ptr` is valid, properly aligned and live longer than `'a`. + /// - `*ptr` is initialized, and the ownership is transferred to this guard. + /// - if `P` is `Pinned`, `ptr` is pinned. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: By safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } +} + +impl<'a, T: ?Sized> SelfRefDropGuard<'a, Unpinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &'a T { + // SAFETY: Per type invariant. + unsafe { &*self.ptr } + } + + /// Create a let binding for accessor use in dropck. + #[inline] + pub fn let_binding_in_dropck(&mut self) -> &T { + self.let_binding() + } +} + +impl<'a, T: ?Sized> SelfRefDropGuard<'a, Pinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> Pin<&'a T> { + // SAFETY: `self.ptr` is valid, properly aligned, live longer than `'a`, initialized, + // exclusively accessible and pinned per type invariant. + unsafe { Pin::new_unchecked(&*self.ptr) } + } + + /// Create a let binding for accessor use in dropck. + #[inline] + pub fn let_binding_in_dropck(&mut self) -> Pin<&T> { + self.let_binding() + } +} + +impl Drop for SelfRefDropGuard<'_, P, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. + unsafe { ptr::drop_in_place(self.ptr) } + } +} + /// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely /// created struct. This is needed, because the `drop` function is safe, but should not be called /// manually. diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index d1ed0561bb27..9ffa76434310 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -923,6 +923,7 @@ macro_rules! assert_pinned { let data = <$ty as $crate::__internal::HasInitData>::__init_data(); let data = $crate::__internal::HasPinData::__pin_data(data); _ = data + .__with_lt() .$field(ptr) .init($crate::__internal::AlwaysFail::<$field_ty>::new()); }; From 71b0637185270a657333bb378d36e6bc79d64518 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 26 Sep 2026 00:25:21 +0100 Subject: [PATCH 62/91] rust: pin-init: internal: pin_data: project self-referential fields This adds the projection for fields that are shared borrowed or that borrows other fields but is covariant. Both cases allow a shared reference to be accessed. No mutable references can be created for these cases for different reasons: * For fields that are shared borrowed, aliasing restriction prevents creation of mutable reference * For fields that borrow other fields, their proper type contains field lifetimes. These lifetimes cannot be made available in the returned `project` struct (because there is no way to represent existential lifetime in return position). For covariant types, it is possible to shorten these lifetimes to that of `&self`; but doing so requires the reference to also be covariant over the pointee type, so we cannot give out `&mut` as it is invariant over the pointee. Due to field-referencing fields being wrapped inside `Erase`, the normal accessor syntax stop working; create accessor methods for these fields instead. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-9-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 109 +++++++++++++++++++++---- rust/pin-init/src/__internal.rs | 12 +++ 2 files changed, 107 insertions(+), 14 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index a46c4dcd7da5..002489a9b36c 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -12,7 +12,7 @@ visit::Visit, visit_mut::VisitMut, Field, Fields, GenericParam, Generics, Ident, Index, Item, ItemStruct, Lifetime, LifetimeParam, - Member, PathSegment, Type, TypePath, + Member, PathSegment, Token, Type, TypePath, }; use crate::{ @@ -844,7 +844,8 @@ fn generate_projections(info: &StructInfo) -> TokenStream { generics, .. } = &info.struct_; - let this_lt_generics: Generics = parse_quote!(<'__this>); + let this_lt = Lifetime::new("'__this", Span::mixed_site()); + let this_lt_generics: Generics = parse_quote!(<#this_lt>); let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); @@ -855,33 +856,68 @@ fn generate_projections(info: &StructInfo) -> TokenStream { let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = info .fields .iter() - .map(|field| { - let Field { vis, ty, .. } = &field.field; - let member = &field.member; + .map(|f| { + let vis = &f.field.vis; + let ident = f.member.as_ident(); + let member = &f.member; // The projection of a tuple struct is a tuple struct itself, so its fields are // positional and must not be named. - let name = (!info.is_tuple_struct).then(|| { - let ident = field.member.as_ident(); - quote!(#ident:) - }); + let name = (!info.is_tuple_struct).then(|| quote!(#ident:)); - if field.pinned { + // if `f.ty` contains field lifetimes, which we need to replace them with shorter + // `'__this` lifetime as field lifetimes are not available in this context. + let all_lifetimes: Vec<_> = f.captures.iter().map(|b| &b.lifetime).collect(); + let ty = f + .field + .ty + .replace_lifetimes(&all_lifetimes, &vec![&this_lt; all_lifetimes.len()]); + + // Fields sharedly borrowed by other fields can only be shared accessed. Fields that + // references other field and are covariant can also only be given shared reference + // as mutable reference is invariant. + let mut_token: Option = if f.borrowed.is_none() && f.captures.is_empty() { + Some(Default::default()) + } else { + None + }; + + let mut accessor = quote!(&#mut_token #this.#member); + if !f.captures.is_empty() || f.borrowed.is_some() { + accessor = quote!( + // SAFETY: we have `Erase<..>` which we know is layout compatible with `f.ty`. + // Field lifetimes in `f.ty` can be shortened to `#ty` due to covariance. + unsafe { ::core::mem::transmute::<_, &#mut_token #ty>(#accessor) } + ) + } + + if !f.captures.iter().all(|b| b.variance == Variance::Covariant) { + // If the type is not covariant, it must omitted, as projection shortens the + // lifetime to `'__this`. ( quote!( - #vis #name ::core::pin::Pin<&'__this mut #ty>, + #vis #name ::pin_init::__internal::NotVisible<&'__this #mut_token #ty>, + ), + quote!( + #name ::pin_init::__internal::NotVisible::new(), + ), + ) + } else if f.pinned { + ( + quote!( + #vis #name ::core::pin::Pin<&'__this #mut_token #ty>, ), quote!( // SAFETY: this field is structurally pinned. - #name unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#member) }, + #name unsafe { ::core::pin::Pin::new_unchecked(#accessor) }, ), ) } else { ( quote!( - #vis #name &'__this mut #ty, + #vis #name &'__this #mut_token #ty, ), quote!( - #name &mut #this.#member, + #name #accessor, ), ) } @@ -931,6 +967,49 @@ fn generate_projections(info: &StructInfo) -> TokenStream { }, ) }; + + // For fields that references other fields, field access syntax stops working as they're wrapped + // behind `Erase` because their actual lifetime is not on the struct. + // + // Generate an accessor method for them. + let mut accessors = Vec::new(); + for f in info.fields.iter() { + let ident = f.member.as_ident(); + let member = &f.member; + + if f.captures.is_empty() { + // They can be accessed normally, no accessor to be generated. + continue; + } + + if f.captures.iter().all(|b| b.variance == Variance::Covariant) { + let f_doc = format!("Access the `{ident}` field on a shared reference of `Self`."); + let vis = &f.field.vis; + + // Use the span of type for better error message. + let span = f.field.ty.span().resolved_at(Span::mixed_site()); + + let all_lifetimes: Vec<_> = f.captures.iter().map(|b| &b.lifetime).collect(); + let ty = f + .field + .ty + .replace_lifetimes(&all_lifetimes, &vec![&this_lt; all_lifetimes.len()]); + + accessors.push(quote_spanned!(span => + #[doc = #f_doc] + #[inline] + #[allow(clippy::mut_from_ref)] // false positive when `&&mut` is returned. + #vis fn #ident<#this_lt>(&#this_lt self) -> &#this_lt #ty { + // SAFETY: we have `Erased<..>` which we know is layout compatible with `f.ty`. + // Field lifetimes in `f.ty` can be shortened to `#ty` due to covariance. + unsafe { ::core::mem::transmute(&self.#member) } + } + )) + } else { + continue; + } + } + quote! { #[doc = #docs] // Allow `non_snake_case` since the same warning will be emitted on @@ -957,6 +1036,8 @@ impl #impl_generics #ident #ty_generics let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; #projection_init } + + #(#accessors)* } } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index b639c9204a7d..d44a5b1f3362 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -5,6 +5,8 @@ //! These items must not be used outside of this crate and the pin-init-internal crate located at //! `../internal`. +#![expect(clippy::new_without_default, reason = "private API")] + use core::marker::PhantomPinned; use core::ops::Deref; @@ -677,3 +679,13 @@ fn deref(&self) -> &T { &self.1 } } + +/// An alias of `PhantomData` but with a name to aid user in case of misuse. +pub struct NotVisible(PhantomData); + +impl NotVisible { + #[inline(always)] + pub fn new() -> Self { + Self(PhantomData) + } +} From b8d10c864a76cfbe9d3d94b35482683219aca712 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 2 May 2026 22:29:08 +0100 Subject: [PATCH 63/91] rust: pin-init: internal: pin_data: add `with_project` method The `project` method needs to perform covariant coercion on covariant fields, causing them to no longer being mutable. Implement a `with_project` that does not require covariant coercion by using higher-ranked trait bounds, thus allow the fields to be assignable inside the callback. This mechanism can also be used to access non-covariant fields. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-10-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 141 ++++++++++++++++++++++++- 1 file changed, 136 insertions(+), 5 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 002489a9b36c..78044cbc5a23 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -846,10 +846,17 @@ fn generate_projections(info: &StructInfo) -> TokenStream { } = &info.struct_; let this_lt = Lifetime::new("'__this", Span::mixed_site()); let this_lt_generics: Generics = parse_quote!(<#this_lt>); - let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); + // Wrap in `CombinedGenerics` because it's ty generics will always output `<>`, so it can be + // used with `for`. + let field_lts = CombinedGenerics(vec![&info.field_lts]); + let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); + let generics_with_this_field_lt = + CombinedGenerics(vec![&this_lt_generics, &info.field_lts, generics]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); let (_, ty_generics_with_this_lt, _) = generics_with_this_lt.split_for_impl(); + let (_, ty_generics_with_this_field_lt, _) = generics_with_this_field_lt.split_for_impl(); let this = format_ident!("this"); @@ -923,16 +930,78 @@ fn generate_projections(info: &StructInfo) -> TokenStream { } }) .collect(); - let structurally_pinned_fields_docs = info + + let (fields_decl_lt, fields_proj_lt): (Vec<_>, Vec<_>) = info + .fields + .iter() + .map(|f| { + let vis = &f.field.vis; + let ident = f.member.as_ident(); + let member = &f.member; + let name = (!info.is_tuple_struct).then(|| quote!(#ident:)); + + let ty = &f.field.ty; + + // Fields shared-referenced by other fields can only be shared accessed. + let mut_token: Option = if f.borrowed.is_none() { + Some(Default::default()) + } else { + None + }; + + let mut accessor = quote!(&#mut_token #this.#member); + if !f.captures.is_empty() || f.borrowed.is_some() { + accessor = quote!( + // SAFETY: we have `Erase<..>` which we know is layout compatible with `f.ty`. + // We cannot include explicit type name here as the field lifetimes are nameable + // in this context, so `for<'field_name> ..` would fail. + unsafe { ::core::mem::transmute(#accessor) } + ) + } + + // In `with_project`, borrowed fields have their field lifetime available, so use it + // instead of `'__this`. + let lt = if f.borrowed.is_some() { + Lifetime::from_ident(&ident) + } else { + this_lt.clone() + }; + + if f.pinned { + ( + quote!( + #vis #name ::core::pin::Pin<&#lt #mut_token #ty>, + ), + quote!( + // SAFETY: this field is structurally pinned. + #name unsafe { ::core::pin::Pin::new_unchecked(#accessor) }, + ), + ) + } else { + ( + quote!( + #vis #name &#lt #mut_token #ty, + ), + quote!( + #name #accessor, + ), + ) + } + }) + .collect(); + + let structurally_pinned_fields_docs: Vec<_> = info .fields .iter() .filter(|f| f.pinned) - .map(|f| format!(" - {}", f.member.display_name())); - let not_structurally_pinned_fields_docs = info + .map(|f| format!(" - {}", f.member.display_name())) + .collect(); + let not_structurally_pinned_fields_docs: Vec<_> = info .fields .iter() .filter(|f| !f.pinned) - .map(|f| format!(" - {}", f.member.display_name())); + .map(|f| format!(" - {}", f.member.display_name())) + .collect(); let docs = format!(" Pin-projections of [`{ident}`]"); let (projection_def, projection_init) = if info.is_tuple_struct { ( @@ -968,6 +1037,42 @@ fn generate_projections(info: &StructInfo) -> TokenStream { ) }; + let projection_lt = format_ident!("__ProjectionLt"); + let (projection_lt_def, projection_lt_init) = if info.is_tuple_struct { + ( + quote! { + #vis struct #projection_lt #generics_with_this_field_lt ( + #(#fields_decl_lt)* + ::core::marker::PhantomData<&'__this mut #ident #ty_generics>, + ) #whr; + }, + quote! { + #projection_lt( + #(#fields_proj_lt)* + ::core::marker::PhantomData, + ) + }, + ) + } else { + ( + quote! { + #vis struct #projection_lt #generics_with_this_field_lt + #whr + { + #(#fields_decl_lt)* + ___pin_phantom_data: + ::core::marker::PhantomData<&'__this mut #ident #ty_generics>, + } + }, + quote! { + #projection_lt { + #(#fields_proj_lt)* + ___pin_phantom_data: ::core::marker::PhantomData, + } + }, + ) + }; + // For fields that references other fields, field access syntax stops working as they're wrapped // behind `Erase` because their actual lifetime is not on the struct. // @@ -1018,6 +1123,13 @@ fn generate_projections(info: &StructInfo) -> TokenStream { #[doc(hidden)] #projection_def + #[doc = #docs] + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(dead_code, non_snake_case)] + #[doc(hidden)] + #projection_lt_def + impl #impl_generics #ident #ty_generics #whr { @@ -1037,6 +1149,25 @@ impl #impl_generics #ident #ty_generics #projection_init } + /// Pin-projects all fields of `Self` with proper lifetime. + /// + /// These fields are structurally pinned: + #(#[doc = #structurally_pinned_fields_docs])* + /// + /// These fields are **not** structurally pinned: + #(#[doc = #not_structurally_pinned_fields_docs])* + #[inline] + #vis fn with_project<'__this, R>( + self: ::core::pin::Pin<&'__this mut Self>, + f: impl for #field_lt_ty_generics ::core::ops::FnOnce( + #projection_lt #ty_generics_with_this_field_lt + ) -> R, + ) -> R { + // SAFETY: we only give access to `&mut` for fields not structurally pinned. + let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; + f(#projection_lt_init) + } + #(#accessors)* } } From 8f6607d1e51d082f3eabf82adf62e0c732e1a5d6 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 29 Sep 2026 13:58:15 +0100 Subject: [PATCH 64/91] rust: pin-init: internal: pin_data: enable self-referential support Enable self-referential support, and add example and test cases. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-11-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/examples/selfref.rs | 35 ++++++++++++++++++++++++++ rust/pin-init/internal/src/pin_data.rs | 7 ------ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 rust/pin-init/examples/selfref.rs diff --git a/rust/pin-init/examples/selfref.rs b/rust/pin-init/examples/selfref.rs new file mode 100644 index 000000000000..b5cdf96b6d1c --- /dev/null +++ b/rust/pin-init/examples/selfref.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::disallowed_names)] + +use pin_init::*; + +#[pin_data] +struct SelfRef { + part: &'str str, + str: String, +} + +fn use_self_ref() { + stack_pin_init!(let foo = pin_init!(SelfRef { + str: "hello world".to_owned(), + part: &str[..5], + })); + + // Access via projection. + println!("{}", foo.as_mut().project().part); + + // Access via accessor. + println!("{}", foo.part()); + + // Access via `with_project`, gives mutable reference. + foo.as_mut().with_project(|proj| { + *proj.part = &proj.str[5..]; + }); + + println!("{}", foo.part()); +} + +fn main() { + use_self_ref(); +} diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 78044cbc5a23..ad98bb9f553f 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -422,13 +422,6 @@ fn expand( } } - if info.self_referential { - dcx.error( - &info.struct_.ident, - "self-referential support is not fully implemented", - ); - } - let struct_def = generate_struct_def(&info); let unpin_impl = generate_unpin_impl(&info); let drop_impl = generate_drop_impl(&info); From f53736dd800db574f508ba6bdab6804c795f79a6 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 1 Oct 2026 14:37:21 +0100 Subject: [PATCH 65/91] rust: pin-init: internal: pin_data: allow lifetime to be shortened per field drop order Add the outlive relations per field drop order. This allows a single lifetime to be used when a field potentially borrow from two different fields, by allowing the longer-living field lifetime to be shortened to a shorter-living field lifetime. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-12-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 82 +++++++++++++------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index ad98bb9f553f..f27dff95a11a 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -107,6 +107,7 @@ fn cmp(&self, other: &Self) -> std::cmp::Ordering { } } +#[expect(unused)] struct FieldInfo { field: Field, member: Member, @@ -124,8 +125,8 @@ struct StructInfo { field_idx_map: BTreeMap, is_tuple_struct: bool, self_referential: bool, - /// Field lifetime generics. - field_lts: Generics, + /// Field lifetime generics with outlive chain. + field_lts_outlive_chain: Generics, } pub(crate) fn expand_with_cfg( @@ -346,16 +347,20 @@ fn expand( // Create a lifetime parameter for each field. let borrowed_fields: Vec<_> = fields.iter().filter_map(|f| f.borrowed.as_ref()).collect(); - let mut field_lts = Generics { + let mut field_lts_outlive_chain = Generics { lt_token: None, params: borrowed_fields .iter() - .map(|borrowed| { + .zip(std::iter::once(None).chain(borrowed_fields.iter().map(Some))) + .map(|(borrowed, prev)| { GenericParam::Lifetime(LifetimeParam { attrs: Vec::new(), lifetime: borrowed.lifetime.clone(), colon_token: None, - bounds: Default::default(), + bounds: prev + .iter() + .map(|borrowed| borrowed.lifetime.clone()) + .collect(), }) }) .collect(), @@ -363,36 +368,26 @@ fn expand( where_clause: None, }; - // Insert necessary bounds to make types well-formed. - for field in fields.iter() { - let Some(borrowed) = &field.borrowed else { - continue; - }; - let field_lt = &borrowed.lifetime; - - // For each borrowed field that borrows from other fields, we need to insert outlive bounds. - for capture in &field.captures { - let lt = &capture.lifetime; - field_lts - .make_where_clause() - .predicates - .push(parse_quote!(#lt: #field_lt)); - } - - // For each borrowed field that references a generic, we also need to insert their outlive - // bounds so they can refer to generics. - for lt in field.generic_lt_captures.iter() { - field_lts - .make_where_clause() - .predicates - .push(parse_quote!(#lt: #field_lt)); - } - - for ty in field.generic_ty_captures.iter() { - field_lts - .make_where_clause() - .predicates - .push(parse_quote!(#ty: #field_lt)); + if let Some(last_borrowed_field) = borrowed_fields.last() { + let field_lt = &last_borrowed_field.lifetime; + for param in struct_.generics.params.iter() { + match param { + GenericParam::Lifetime(param) => { + let lt = ¶m.lifetime; + field_lts_outlive_chain + .make_where_clause() + .predicates + .push(parse_quote!(#lt: #field_lt)); + } + GenericParam::Type(param) => { + let ty = ¶m.ident; + field_lts_outlive_chain + .make_where_clause() + .predicates + .push(parse_quote!(#ty: #field_lt)); + } + GenericParam::Const(_) => (), + } } } @@ -406,7 +401,7 @@ fn expand( fields, field_idx_map, is_tuple_struct, - field_lts, + field_lts_outlive_chain, }; for field in &info.fields { @@ -703,7 +698,7 @@ fn generate_drop_order_check(dcx: &mut DiagCtxt, info: &StructInfo) -> TokenStre // with known lifetime bounds as bounds on the function, and asks Rust to *prove* that the types // are wellformed, given the bounds that we understand. - let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts, generics]); + let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts_outlive_chain, generics]); let (_, ty_generics, _) = generics.split_for_impl(); let (impl_generics_with_field_lt, _, whr_with_field_lt) = @@ -842,10 +837,13 @@ fn generate_projections(info: &StructInfo) -> TokenStream { // Wrap in `CombinedGenerics` because it's ty generics will always output `<>`, so it can be // used with `for`. - let field_lts = CombinedGenerics(vec![&info.field_lts]); + let field_lts = CombinedGenerics(vec![&info.field_lts_outlive_chain]); let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); - let generics_with_this_field_lt = - CombinedGenerics(vec![&this_lt_generics, &info.field_lts, generics]); + let generics_with_this_field_lt = CombinedGenerics(vec![ + &this_lt_generics, + &info.field_lts_outlive_chain, + generics, + ]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); let (_, ty_generics_with_this_lt, _) = generics_with_this_lt.split_for_impl(); @@ -1176,8 +1174,8 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { // Wrap in `CombinedGenerics` because it's ty generics will always output `<>`, so it can be // used with `for`. - let field_lts = CombinedGenerics(vec![&info.field_lts]); - let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts, generics]); + let field_lts = CombinedGenerics(vec![&info.field_lts_outlive_chain]); + let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts_outlive_chain, generics]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); From c660201f8245df541041420b3a2087207c8e2ed9 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 24 Sep 2026 13:30:18 +0100 Subject: [PATCH 66/91] rust: pin-init: internal: pin_data: parse explicit `#[borrowed]` annotation `#[borrowed]` attribute explicitly marks a field as potentially being borrowed by other fields. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-13-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 36 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index f27dff95a11a..9b584e4c6d5d 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -11,8 +11,8 @@ spanned::Spanned, visit::Visit, visit_mut::VisitMut, - Field, Fields, GenericParam, Generics, Ident, Index, Item, ItemStruct, Lifetime, LifetimeParam, - Member, PathSegment, Token, Type, TypePath, + Attribute, Field, Fields, GenericParam, Generics, Ident, Index, Item, ItemStruct, Lifetime, + LifetimeParam, Member, Meta, PathSegment, Token, Type, TypePath, }; use crate::{ @@ -55,11 +55,25 @@ fn to_tokens(&self, tokens: &mut TokenStream) { /// Description of how a field is borrowed. #[derive(Clone, Copy, Default, PartialEq, Eq)] enum BorrowedKind { - /// Implicitly inferreed. + /// `#[borrowed]`, or implicitly inferreed. #[default] Shared, } +impl BorrowedKind { + fn parse(dcx: &mut DiagCtxt, attrs: &mut Vec) -> Option { + let attr = attrs.extract_single_attr(dcx, "borrowed")?; + + Some(if let Meta::Path(_) = attr.meta { + BorrowedKind::Shared + } else { + // Swallow the error and recover by inferring shared. + dcx.error(attr.path(), "unexpected `#[borrowed]` attribute"); + BorrowedKind::Shared + }) + } +} + /// Information about a borrowed field. struct BorrowedInfo { kind: BorrowedKind, @@ -305,11 +319,25 @@ fn expand( }) .visit_type(&field.ty); + let borrowed = BorrowedKind::parse(dcx, &mut field.attrs).and_then(|kind| { + let lifetime = Lifetime::from_ident(&member.as_ident()); + + if bound_lifetimes.contains(&lifetime) { + dcx.error( + &lifetime, + format!("`{lifetime}` appear in generics and would conflict with field lifetime"), + ); + return None; + } + + Some(BorrowedInfo { kind, lifetime }) + }); + FieldInfo { field, member, pinned, - borrowed: None, + borrowed, captures, generic_lt_captures, generic_ty_captures, From 0b326dd96e773ef5e980f62f5e1b4fb05c4f983a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 6 May 2026 18:12:59 +0100 Subject: [PATCH 67/91] rust: pin-init: internal: pin_data: support mutable borrows Allow fields to be mutably referenced by other fields in addition to shared references. In order for this to be sound, the fields that can be mutably borrowed are blocked from being accessed via field access syntax or projection to maintain the aliasing requirements. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-14-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/examples/selfref.rs | 13 ++++ rust/pin-init/internal/src/pin_data.rs | 94 +++++++++++++++++++++++--- rust/pin-init/src/__internal.rs | 66 ++++++++++++++---- 3 files changed, 149 insertions(+), 24 deletions(-) diff --git a/rust/pin-init/examples/selfref.rs b/rust/pin-init/examples/selfref.rs index b5cdf96b6d1c..5e9494dcc17e 100644 --- a/rust/pin-init/examples/selfref.rs +++ b/rust/pin-init/examples/selfref.rs @@ -8,12 +8,18 @@ struct SelfRef { part: &'str str, str: String, + + mut_part: &'mut_str mut str, + #[borrowed(mut)] + mut_str: String, } fn use_self_ref() { stack_pin_init!(let foo = pin_init!(SelfRef { str: "hello world".to_owned(), part: &str[..5], + mut_str: "hello world".to_owned(), + mut_part: &mut mut_str[..5], })); // Access via projection. @@ -28,6 +34,13 @@ fn use_self_ref() { }); println!("{}", foo.part()); + + // Access fields that mutable borrow others are similar to those of shared borrow. + println!("{}", foo.as_mut().project().mut_part); + println!("{}", foo.mut_part()); + foo.as_mut().with_project(|proj| { + proj.mut_part.make_ascii_uppercase(); + }); } fn main() { diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 9b584e4c6d5d..9910e14b9926 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -5,7 +5,7 @@ use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ - parse::{End, Nothing, Parse}, + parse::{End, Nothing, Parse, ParseStream}, parse_quote, parse_quote_spanned, punctuated::Punctuated, spanned::Spanned, @@ -58,6 +58,8 @@ enum BorrowedKind { /// `#[borrowed]`, or implicitly inferreed. #[default] Shared, + // `#[borrowed(mut)]`. + Mutable, } impl BorrowedKind { @@ -67,9 +69,17 @@ fn parse(dcx: &mut DiagCtxt, attrs: &mut Vec) -> Option { Some(if let Meta::Path(_) = attr.meta { BorrowedKind::Shared } else { - // Swallow the error and recover by inferring shared. - dcx.error(attr.path(), "unexpected `#[borrowed]` attribute"); - BorrowedKind::Shared + match attr.parse_args_with(|input: ParseStream<'_>| { + let _: Token![mut] = input.parse()?; + Ok(BorrowedKind::Mutable) + }) { + Ok(v) => v, + Err(err) => { + // Swallow the error and recover by inferring shared. + dcx.error(attr.path(), err); + BorrowedKind::Shared + } + } }) } } @@ -515,8 +525,17 @@ fn generate_struct_def(info: &StructInfo) -> TokenStream { let mut ty = ty.to_token_stream(); - // Replace lifetime for self-referential fields. - if !field.captures.is_empty() { + // Replace lifetime for self-referential fields. For mutable fields, this uses `Erase` to + // block direct access. + if !field.captures.is_empty() + || matches!( + field.borrowed, + Some(BorrowedInfo { + kind: BorrowedKind::Mutable, + .. + }) + ) + { // Build a chain `for<'a> fn(&'a ()) -> ... -> (Ty,)`. Such type will have a `EraseLt` // implementation and thus may be used inside `Erase`. ty = quote!((#ty,)); @@ -916,9 +935,18 @@ fn generate_projections(info: &StructInfo) -> TokenStream { ) } - if !f.captures.iter().all(|b| b.variance == Variance::Covariant) { + if !f.captures.iter().all(|b| b.variance == Variance::Covariant) + || matches!( + f.borrowed, + Some(BorrowedInfo { + kind: BorrowedKind::Mutable, + .. + }) + ) + { // If the type is not covariant, it must omitted, as projection shortens the // lifetime to `'__this`. + // Mutable borrow must be omitted for aliasing reason. ( quote!( #vis #name ::pin_init::__internal::NotVisible<&'__this #mut_token #ty>, @@ -986,7 +1014,25 @@ fn generate_projections(info: &StructInfo) -> TokenStream { this_lt.clone() }; - if f.pinned { + if matches!( + f.borrowed, + Some(BorrowedInfo { + kind: BorrowedKind::Mutable, + .. + }) + ) { + // If the type is not covariant, it must omitted, as projection shortens the + // lifetime to `'__this`. + // Mutable borrow must be omitted for aliasing reason. + ( + quote!( + #vis #name ::pin_init::__internal::NotVisible<&#lt #mut_token #ty>, + ), + quote!( + #name ::pin_init::__internal::NotVisible::new(), + ), + ) + } else if f.pinned { ( quote!( #vis #name ::core::pin::Pin<&#lt #mut_token #ty>, @@ -1106,6 +1152,17 @@ fn generate_projections(info: &StructInfo) -> TokenStream { continue; } + if matches!( + f.borrowed, + Some(BorrowedInfo { + kind: BorrowedKind::Mutable, + .. + }) + ) { + // Mutably borrowed fields cannot be accessed directly under any circumstance. + continue; + } + if f.captures.iter().all(|b| b.variance == Variance::Covariant) { let f_doc = format!("Access the `{ident}` field on a shared reference of `Self`."); let vis = &f.field.vis; @@ -1261,7 +1318,26 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { // assumptions on the lifetime except for those implied by the struct's bounds, // and we have validated them in `generate_drop_check`. quote!(SelfRefSlot), - quote!(#lifetime,), + quote!(#lifetime, ::pin_init::__internal::Shared, ), + ), + Some(BorrowedInfo { + kind: BorrowedKind::Mutable, + lifetime, + }) => ( + // For borrowed fields, create a `SelfRefSlot`, which after initialization + // turns into a `SelfRefDropGuard` instead of `DropGuard`. + // + // They're mostly the same, except that `SelfRefDropGuard` returns `&'field T` + // instead of `&'guard T` for let bindings; this allows it to be used to be + // used to initialize other fields. + // + // The soundness of doing so relies on fact that `__make_init` requires a + // higher-ranked trait bound on the closure. Within the closure (which is the + // caller of the generated slot projection functions here), it can make no + // assumptions on the lifetime except for those implied by the struct's bounds, + // and we have validated them in `generate_drop_check`. + quote!(SelfRefSlot), + quote!(#lifetime, ::pin_init::__internal::Mutable, ), ), }; diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index d44a5b1f3362..c013d94ab5aa 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -361,6 +361,9 @@ fn drop(&mut self) { } } +pub struct Shared; +pub struct Mutable; + /// Represent an uninitialized field in a pinned struct that will be referenced by other fields. /// /// # Invariants @@ -368,12 +371,12 @@ fn drop(&mut self) { /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory /// and will live longer than `'a`. /// - If `P` is `Pinned`, then `ptr` is structurally pinned. -pub struct SelfRefSlot<'a, P, T: ?Sized> { - pub ptr: *mut T, - pub _phantom: PhantomData<(P, &'a mut T)>, +pub struct SelfRefSlot<'a, M, P, T: ?Sized> { + ptr: *mut T, + _phantom: PhantomData<(M, P, &'a mut T)>, } -impl<'a, P, T: ?Sized> SelfRefSlot<'a, P, T> { +impl<'a, M, P, T: ?Sized> SelfRefSlot<'a, M, P, T> { /// # Safety /// /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed @@ -390,7 +393,7 @@ pub unsafe fn new(ptr: *mut T) -> Self { /// Initialize the field by value. #[inline] - pub fn write(self, value: T) -> SelfRefDropGuard<'a, P, T> + pub fn write(self, value: T) -> SelfRefDropGuard<'a, M, P, T> where T: Sized, { @@ -404,10 +407,10 @@ pub fn write(self, value: T) -> SelfRefDropGuard<'a, P, T> } } -impl<'a, T: ?Sized> SelfRefSlot<'a, Unpinned, T> { +impl<'a, M, T: ?Sized> SelfRefSlot<'a, M, Unpinned, T> { /// Initialize the field. #[inline] - pub fn init(self, init: impl Init) -> Result, E> { + pub fn init(self, init: impl Init) -> Result, E> { // SAFETY: // - `self.ptr` is valid and properly aligned. // - when `Err` is returned, we also propagate the error without touching `ptr`; @@ -421,10 +424,13 @@ pub fn init(self, init: impl Init) -> Result SelfRefSlot<'a, Pinned, T> { +impl<'a, M, T: ?Sized> SelfRefSlot<'a, M, Pinned, T> { /// Initialize the field. #[inline] - pub fn init(self, init: impl PinInit) -> Result, E> { + pub fn init( + self, + init: impl PinInit, + ) -> Result, E> { // SAFETY: // - `ptr` is valid // - when `Err` is returned, we also propagate the error without touching `ptr`; @@ -447,12 +453,12 @@ pub fn init(self, init: impl PinInit) -> Result { +pub struct SelfRefDropGuard<'a, M, P, T: ?Sized> { ptr: *mut T, - phantom: PhantomData<(P, &'a mut T)>, + phantom: PhantomData<(M, P, &'a mut T)>, } -impl<'a, P, T: ?Sized> SelfRefDropGuard<'a, P, T> { +impl<'a, M, P, T: ?Sized> SelfRefDropGuard<'a, M, P, T> { /// Creates a drop guard and transfer the ownership of the pointer content. /// /// The ownership is only relinquished if the guard is forgotten via [`core::mem::forget`]. @@ -472,7 +478,7 @@ pub unsafe fn new(ptr: *mut T) -> Self { } } -impl<'a, T: ?Sized> SelfRefDropGuard<'a, Unpinned, T> { +impl<'a, T: ?Sized> SelfRefDropGuard<'a, Shared, Unpinned, T> { /// Create a let binding for accessor use. #[inline] pub fn let_binding(&mut self) -> &'a T { @@ -487,7 +493,7 @@ pub fn let_binding_in_dropck(&mut self) -> &T { } } -impl<'a, T: ?Sized> SelfRefDropGuard<'a, Pinned, T> { +impl<'a, T: ?Sized> SelfRefDropGuard<'a, Shared, Pinned, T> { /// Create a let binding for accessor use. #[inline] pub fn let_binding(&mut self) -> Pin<&'a T> { @@ -503,7 +509,37 @@ pub fn let_binding_in_dropck(&mut self) -> Pin<&T> { } } -impl Drop for SelfRefDropGuard<'_, P, T> { +impl<'a, T: ?Sized> SelfRefDropGuard<'a, Mutable, Unpinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &'a mut T { + // SAFETY: Per type invariant. + unsafe { &mut *self.ptr } + } + + /// Create a let binding for accessor use in dropck. + #[inline] + pub fn let_binding_in_dropck(&mut self) -> &mut T { + self.let_binding() + } +} + +impl<'a, T: ?Sized> SelfRefDropGuard<'a, Mutable, Pinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> Pin<&'a mut T> { + // SAFETY: `self.ptr` is valid, properly aligned, live longer than `'a`, initialized, + // exclusively accessible and pinned per type invariant. + unsafe { Pin::new_unchecked(&mut *self.ptr) } + } + + #[inline] + pub fn let_binding_in_dropck(&mut self) -> Pin<&mut T> { + self.let_binding() + } +} + +impl Drop for SelfRefDropGuard<'_, M, P, T> { #[inline] fn drop(&mut self) { // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. From 59a9b11bccdb308f242ba6abc4638d64816781b1 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 24 Sep 2026 13:30:18 +0100 Subject: [PATCH 68/91] rust: pin-init: internal: pin_data: parse explicit `#[uses]` annotation Add support for explicit self-referential annotations. `#[uses]` attribute is used to mark what other field lifetimes are captured by this field, and also the variance of the type in respect to the field lifetimes. For example, #[uses('a: covariant, 'b: invariant)] indicates that the field captures the field lifetime `'a` covariantly, and field lifetime `'b` invariantly. Many types are covariant, so this is the default variance if the variance is omitted (e.g. `#[uses('a)]`), consistent with the automatically inferred borrow. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-15-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 99 ++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 9910e14b9926..ad43f0bbc4c1 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -21,6 +21,9 @@ }; pub(crate) mod kw { + syn::custom_keyword!(covariant); + syn::custom_keyword!(invariant); + syn::custom_keyword!(contravariant); syn::custom_keyword!(PinnedDrop); } @@ -93,9 +96,37 @@ struct BorrowedInfo { #[derive(Clone, Copy, Default, PartialEq, Eq)] enum Variance { - /// Implicitly inferred variance. + /// `covariant` annotation, or implicitly inferred. #[default] Covariant, + // `invariant` annotation. + Invariant, +} + +impl Parse for Variance { + fn parse(input: ParseStream<'_>) -> syn::Result { + let lh = input.lookahead1(); + Ok(if lh.peek(kw::covariant) { + let _: kw::covariant = input.parse()?; + Variance::Covariant + } else if lh.peek(kw::invariant) { + let _: kw::invariant = input.parse()?; + Variance::Invariant + } else if lh.peek(kw::contravariant) { + // Field lifetimes are inherently covariant, so combining with contravariance, + // we would constrain it to be invariant. + let token: kw::contravariant = input.parse()?; + DiagCtxt::current(|dcx| { + dcx.error( + token, + "field lifetimes cannot be `contravariant`; use `invariant` instead", + ) + }); + Variance::Invariant + } else { + Err(lh.error())? + }) + } } /// Information about field lifetimes captured in a type. @@ -131,7 +162,65 @@ fn cmp(&self, other: &Self) -> std::cmp::Ordering { } } -#[expect(unused)] +impl Parse for Capture { + fn parse(input: ParseStream<'_>) -> syn::Result { + let lifetime = input.parse()?; + let variance = if input.peek(Token![:]) { + let _: Token![:] = input.parse()?; + input.parse()? + } else { + // If variance is not explicitly specified, infer covariance by default. + Variance::Covariant + }; + Ok(Capture { variance, lifetime }) + } +} + +impl Capture { + fn parse_list( + dcx: &mut DiagCtxt, + attrs: &mut Vec, + bound_lifetimes: &BTreeSet<&Lifetime>, + field_idx_map: &BTreeMap, + ) -> Option<(BTreeSet, Variance)> { + let attr = attrs.extract_single_attr(dcx, "uses")?; + let punctuated: Punctuated = attr + .parse_args_with(Punctuated::parse_terminated) + .map_err(ErrorGuaranteed::from) + .ok()?; + + // Check for misuses inside attribute. + let mut set = BTreeSet::new(); + for borrow in punctuated { + let lt = &borrow.lifetime; + if set.contains(&borrow) { + dcx.error(lt, format!("lifetime `{lt}` is mentioned more than once")); + continue; + } + + if bound_lifetimes.contains(lt) { + dcx.error( + lt, + format!("`{lt}` is a struct generics and cannot be used in `#[uses]`"), + ); + continue; + } + + if lt.ident != "_" && !field_idx_map.contains_key(<.ident) { + dcx.error(lt, format!("`{lt}` is not a field name")); + continue; + } + + set.insert(borrow); + } + + let wildcard = Lifetime::new("'_", Span::mixed_site()); + let wildcard_variance = set.take(&wildcard).map(|b| b.variance).unwrap_or_default(); + Some((set, wildcard_variance)) + } +} + +#[allow(unused)] struct FieldInfo { field: Field, member: Member, @@ -275,8 +364,10 @@ fn expand( }), }; - let mut captures = BTreeSet::new(); - let wildcard_variance = Variance::default(); + // Parse `#[uses]` attribute. + let (mut captures, wildcard_variance) = + Capture::parse_list(dcx, &mut field.attrs, &bound_lifetimes, &field_idx_map) + .unwrap_or_default(); let mut generic_lt_captures = BTreeSet::new(); let mut generic_ty_captures = BTreeSet::new(); From 91a255fea7306c5b07f6b26d740be31fdc8bfa41 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 9 Sep 2026 19:37:00 +0100 Subject: [PATCH 69/91] rust: pin-init: internal: pin_data: make field lifetime invariance imply type invariance If invariant field captures a field lifetime, then we also need to make sure that field is also invariant. Imagine this struct: #[pin_data] struct SelfRef<'a> { #[uses('outer: invariant)] part: Mutex<&'outer str>, outer: &'a String, } fn new<'a>(str: &'a String) -> impl PinInit, Infallible> { pin_init!(SelfRef { outer: str, part: Mutex::new(*outer), }) } If we make this struct covariant over `'a`, then we can have the following case: let mut long = "hello world".to_owned(); let s = Box::pin_init(new(&long)).unwrap(); { let mut short = "hello world".to_owned(); // If `s` is covariant, this would be okay, because we shorten from // `SelfRef<'long>` to `SelfRef<'short>`. s.with_project_ref(|p| { *p.part.lock().unwrap() = &short; }); } Conceptually, a field's type must outlive the field's lifetime, so if we have a covariant `'a`, we can have a shortened `SelfRef<'short_a>` where `'outer` outlives `'short_a`. But the wellformedness requirement of the field will imply `'short_a: 'outer`, which enables the `&'short_a` to `'outer` coercion, effectively making the field lifetime `'f` behave covariantly, too, breaking the requirement that it is invariant. Therefore, compute an invariant closure and use an additional generics on `Borrowed` to allow capturing things invariantly. Note that we do capture all parameters explicitly rather than capture the field type invariantly, because if type aliases are involved, we might be syntactically determining that the field uses a type parameter but actually not, causing the invariance enforcement to be missed. Outlive bounds between field lifetimes can also cause the same issue (an invariant field lifetime cannot be a lower bound of a covariant field lifetime). Since we cannot deduce whether any implied bounds from type wellformness would create such outlive relationships, prevent such bounds from happening by using a split outlive chain. Note that this is not a soundness hole in itself in absence of `with_project_ref`, because with single field accessors only, the field lifetimes of the fields are not connected; in methods like `with_project` lifetimes are invariant so shortening cannot happen. However, such method is likely desirable, so include the variance rule before it has been heavily relied upon. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-16-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 175 ++++++++++++++++++++++++- rust/pin-init/src/__internal.rs | 8 +- 2 files changed, 174 insertions(+), 9 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index ad43f0bbc4c1..77c8ce260403 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -92,6 +92,8 @@ struct BorrowedInfo { kind: BorrowedKind, /// Field lifetime for this field. lifetime: Lifetime, + // Variance of the field lifetime, as captured by other fields. + lt_variance: Variance, } #[derive(Clone, Copy, Default, PartialEq, Eq)] @@ -238,8 +240,11 @@ struct StructInfo { field_idx_map: BTreeMap, is_tuple_struct: bool, self_referential: bool, - /// Field lifetime generics with outlive chain. + /// Field lifetime genercis with outlive chain. field_lts_outlive_chain: Generics, + /// Field lifetime genercis with outlive chain, with one chain for covariant fields and one + /// chain for invariant fields. + field_lts_split_variance_outlive_chain: Generics, } pub(crate) fn expand_with_cfg( @@ -431,7 +436,7 @@ fn expand( return None; } - Some(BorrowedInfo { kind, lifetime }) + Some(BorrowedInfo { kind, lifetime, lt_variance: Variance::default() }) }); FieldInfo { @@ -455,6 +460,7 @@ fn expand( kind: BorrowedKind::Shared, // Obtaining from `field` instead of `field_name` for the correct span. lifetime: Lifetime::from_ident(&field.member.as_ident()), + lt_variance: Variance::Covariant, }); } } @@ -474,8 +480,63 @@ fn expand( }) .visit_generics(&struct_.generics); + // Obtain an closure of invariant fields. + // + // If a field lifetime is used invariantly, we also need to make sure that + // for each generic parameter `T: 'field` bound, `T` is also captured + // invariantly (same is true for lifetime parameters). For example, say we + // have a struct `SelfRef<'a>` and a field `f: &'a ()`. For wellformedness, + // we would have `'a: 'f`. If we have a covariant `'a`, we can observe a + // shortened `SelfRef<'short_a>` where `'f` outlives `'short_a`, conflicting + // with the wellformedness bound `'short_a: 'f`. This will enable the + // `&'short_a ()` to `&'f ()` coercion, which effectively shortens `'f` too, + // so we shortened the field lifetime despite it being invariant. + let mut invariant_field_idx = BTreeSet::new(); + let mut worklist = Vec::new(); + for field in fields.iter() { + for borrow in field.captures.iter() { + if let Variance::Invariant = borrow.variance { + let Some(borrow_idx) = fields + .iter() + .position(|f| f.member.as_ident() == borrow.lifetime.ident) + else { + continue; + }; + if invariant_field_idx.insert(borrow_idx) { + worklist.push(&fields[borrow_idx]); + } + } + } + } + while let Some(field) = worklist.pop() { + for borrow in field.captures.iter() { + let Some(borrow_idx) = fields + .iter() + .position(|f| f.member.as_ident() == borrow.lifetime.ident) + else { + continue; + }; + let borrow = &fields[borrow_idx]; + if invariant_field_idx.insert(borrow_idx) { + worklist.push(borrow); + } + } + } + for idx in invariant_field_idx { + fields[idx].borrowed.as_mut().unwrap().lt_variance = Variance::Invariant; + } + // Create a lifetime parameter for each field. let borrowed_fields: Vec<_> = fields.iter().filter_map(|f| f.borrowed.as_ref()).collect(); + let borrowed_covariant_fields: Vec<_> = borrowed_fields + .iter() + .filter(|f| f.lt_variance == Variance::Covariant) + .collect(); + let borrowed_invariant_fields: Vec<_> = borrowed_fields + .iter() + .filter(|f| f.lt_variance == Variance::Invariant) + .collect(); + let mut field_lts_outlive_chain = Generics { lt_token: None, params: borrowed_fields @@ -520,6 +581,96 @@ fn expand( } } + let mut field_lts_split_variance_outlive_chain = Generics { + lt_token: Some(Default::default()), + params: borrowed_covariant_fields + .iter() + .zip(std::iter::once(None).chain(borrowed_covariant_fields.iter().map(Some))) + .map(|(borrowed, prev)| { + GenericParam::Lifetime(LifetimeParam { + attrs: Vec::new(), + lifetime: borrowed.lifetime.clone(), + colon_token: None, + bounds: prev + .iter() + .map(|borrowed| borrowed.lifetime.clone()) + .collect(), + }) + }) + .chain( + borrowed_invariant_fields + .iter() + .zip(std::iter::once(None).chain(borrowed_invariant_fields.iter().map(Some))) + .map(|(borrowed, prev)| { + GenericParam::Lifetime(LifetimeParam { + attrs: Vec::new(), + lifetime: borrowed.lifetime.clone(), + colon_token: None, + bounds: prev + .iter() + .map(|borrowed| borrowed.lifetime.clone()) + .collect(), + }) + }), + ) + .collect(), + gt_token: Some(Default::default()), + where_clause: None, + }; + + // For `field_lts_split_variance_outlive_chain`, we may need to insert some additional bounds + // for types to be WF. + for field in fields.iter() { + let Some(borrowed) = &field.borrowed else { + continue; + }; + let field_lt = &borrowed.lifetime; + + // For each borrowed field that references a generic, we also need to insert their outlive + // bounds so they can refer to generics. + for lt in field.generic_lt_captures.iter() { + field_lts_split_variance_outlive_chain + .make_where_clause() + .predicates + .push(parse_quote!(#lt: #field_lt)); + } + + for ty in field.generic_ty_captures.iter() { + field_lts_split_variance_outlive_chain + .make_where_clause() + .predicates + .push(parse_quote!(#ty: #field_lt)); + } + + // If a field is invariant, then the invariance closure rule will make all borrowed fields + // to be invariant, so they're already captured in `field_lts_split_variance_outlive_chain`. + if borrowed.lt_variance != Variance::Covariant { + continue; + } + + for capture in field.captures.iter() { + let Some(&idx) = field_idx_map.get(&capture.lifetime.ident) else { + continue; + }; + + let prev_borrowed = fields[idx].borrowed.as_ref().unwrap(); + + // If borrowed field is covariant, it's already captured in + // `field_lts_split_variance_outlive_chain`. + if prev_borrowed.lt_variance == Variance::Invariant { + // Covariant field borrowing an invariant field. This is not captured in the chain + // so we need to add additional bound. This bound is okay, as the invariant lifetime + // is the longer living one, so arbitrary shortening of the covariant one does not + // violate their relation. + let param = field_lts_split_variance_outlive_chain + .lifetimes_mut() + .find(|l| l.lifetime == prev_borrowed.lifetime) + .unwrap(); + param.bounds.push(borrowed.lifetime.clone()); + } + } + } + struct_.fields = Fields::Unit; let info = StructInfo { self_referential: fields @@ -531,6 +682,7 @@ fn expand( field_idx_map, is_tuple_struct, field_lts_outlive_chain, + field_lts_split_variance_outlive_chain, }; for field in &info.fields { @@ -639,8 +791,18 @@ fn generate_struct_def(info: &StructInfo) -> TokenStream { ty = quote!(::pin_init::__internal::Erase<#ty>); }; - if field.borrowed.is_some() { - ty = quote!(::pin_init::__internal::Borrowed<#ty>); + if let Some(borrowed) = &field.borrowed { + let mut invariance_capture = Vec::new(); + if borrowed.lt_variance == Variance::Invariant { + for lt in &field.generic_lt_captures { + invariance_capture.push(quote!(&#lt ())); + } + + for ty in &field.generic_ty_captures { + invariance_capture.push(quote!(::core::marker::PhantomData<#ty>)); + } + } + ty = quote!(::pin_init::__internal::Borrowed<#ty, (#(#invariance_capture,)*)>); } quote! { @@ -836,7 +998,8 @@ fn generate_drop_order_check(dcx: &mut DiagCtxt, info: &StructInfo) -> TokenStre // with known lifetime bounds as bounds on the function, and asks Rust to *prove* that the types // are wellformed, given the bounds that we understand. - let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts_outlive_chain, generics]); + let generics_with_field_lt = + CombinedGenerics(vec![&info.field_lts_split_variance_outlive_chain, generics]); let (_, ty_generics, _) = generics.split_for_impl(); let (impl_generics_with_field_lt, _, whr_with_field_lt) = @@ -1395,6 +1558,7 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { Some(BorrowedInfo { kind: BorrowedKind::Shared, lifetime, + .. }) => ( // For borrowed fields, create a `SelfRefSlot`, which after initialization // turns into a `SelfRefDropGuard` instead of `DropGuard`. @@ -1414,6 +1578,7 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { Some(BorrowedInfo { kind: BorrowedKind::Mutable, lifetime, + .. }) => ( // For borrowed fields, create a `SelfRefSlot`, which after initialization // turns into a `SelfRefDropGuard` instead of `DropGuard`. diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index c013d94ab5aa..9f0f5a34e630 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -676,7 +676,7 @@ unsafe impl Sync for Erase /// This should be switched to `UnsafePinned` when it is stable. /// NOTE: This type needs to be covariant; Rust's 1.89+'s `UnsafePinned` is invariant. #[repr(transparent)] -pub struct Borrowed(PhantomPinned, T); +pub struct Borrowed(PhantomInvariant

, PhantomPinned, T); // Lifetimes not needed by drop glue are considered by Rust's drop check to be considered // `#[may_dangle]`. In case for a self-referential struct, we may have fields which need lifetime of @@ -702,17 +702,17 @@ unsafe impl Sync for Erase // outlive the struct. And this can be done by a simple `Drop` impl that does nothing. Without a // dropck eye patch, presence of `Drop` impl, albeit empty, tells the drop check that the strict // outlive relation is needed. -impl Drop for Borrowed { +impl Drop for Borrowed { #[inline(always)] fn drop(&mut self) {} } -impl Deref for Borrowed { +impl Deref for Borrowed { type Target = T; #[inline(always)] fn deref(&self) -> &T { - &self.1 + &self.2 } } From e55acb73379fc2a5164c741bb1257becd346bc39 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 2 May 2026 22:40:15 +0100 Subject: [PATCH 70/91] rust: pin-init: internal: pin_data: complete invariant borrow support Non-covariant types can already be accessed inside projections. As projections are only generated for `Pin<&mut T>`, they're not accessible otherwise. Add `with_{field_name}` methods so fields can be accessed using closures with just `&T`. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-17-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/examples/selfref.rs | 12 ++++++++++++ rust/pin-init/internal/src/pin_data.rs | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/rust/pin-init/examples/selfref.rs b/rust/pin-init/examples/selfref.rs index 5e9494dcc17e..5212b9255b75 100644 --- a/rust/pin-init/examples/selfref.rs +++ b/rust/pin-init/examples/selfref.rs @@ -6,6 +6,9 @@ #[pin_data] struct SelfRef { + #[uses('_: invariant)] + not_cov: Box bool + 'str>, + part: &'str str, str: String, @@ -20,6 +23,7 @@ fn use_self_ref() { part: &str[..5], mut_str: "hello world".to_owned(), mut_part: &mut mut_str[..5], + not_cov: Box::new(move |s| s == str), })); // Access via projection. @@ -41,6 +45,14 @@ fn use_self_ref() { foo.as_mut().with_project(|proj| { proj.mut_part.make_ascii_uppercase(); }); + + // Access non-covariant type using `with_` accessor. + foo.with_not_cov(|not_cov| { + not_cov(""); + }); + + // Access non-covariant type using `with_project`. + foo.as_mut().with_project(|proj| (proj.not_cov)(proj.str)); } fn main() { diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 77c8ce260403..4b361c510a40 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1441,7 +1441,24 @@ fn generate_projections(info: &StructInfo) -> TokenStream { } )) } else { - continue; + let f_doc = format!("Access the `{ident}` field on a shared reference of `Self`."); + let vis = &f.field.vis; + let with_ident = format_ident!("with_{ident}"); + + let all_lifetimes: Vec<_> = f.captures.iter().map(|b| &b.lifetime).collect(); + let ty = &f.field.ty; + + accessors.push(quote!( + #[doc = #f_doc] + #[inline] + #vis fn #with_ident<'__this, R>( + &'__this self, + f: impl for<#(#all_lifetimes,)*> ::core::ops::FnOnce(&'__this #ty) -> R, + ) -> R { + // SAFETY: we have `Erase<..>` which we know is layout compatible with `f.ty`. + f(unsafe { ::core::mem::transmute(&self.#member) }) + } + )) } } From e2b2e01365120d388c25ddaefa9f1e6a25aa1f28 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 2 May 2026 22:58:33 +0100 Subject: [PATCH 71/91] rust: pin-init: internal: pin_data: perform AST lifetime replacement if possible Currently lifetimes are replaced via function type and `FnOutput` trait. This is very general approach as it uses generic associated type to replace lifetime, so it can even work when macros are involved. This does cause more generated code, and does not render in documentation nicely. Thus, just replace the lifetime in the AST if no macros are involved. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-18-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/util.rs | 112 ++++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs index 59054f5934fb..4d3331b852b4 100644 --- a/rust/pin-init/internal/src/util.rs +++ b/rust/pin-init/internal/src/util.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, ToTokens}; use syn::{ - parse_quote, visit::Visit, Attribute, BoundLifetimes, GenericParam, Generics, Index, Lifetime, - Member, Token, Type, TypePath, + parse_quote, visit::Visit, visit_mut::VisitMut, Attribute, BoundLifetimes, GenericParam, + Generics, Index, Lifetime, Member, Token, Type, TypePath, }; use crate::DiagCtxt; @@ -388,21 +388,113 @@ fn visit_type_path(&mut self, ty: &'a TypePath) { } pub(crate) trait TypeExt { + /// Check if the type includes macro invocations. + /// + /// Proc-macros cannot expand macros and peek into them, so if macro is involved sometimes + /// special handling is required. + fn has_macro(&self) -> bool; + fn replace_lifetimes(&self, needle: &[&Lifetime], replacement: &[&Lifetime]) -> Type; } impl TypeExt for Type { + fn has_macro(&self) -> bool { + struct HasMacro(bool); + + impl<'ast> Visit<'ast> for HasMacro { + fn visit_macro(&mut self, _: &'ast syn::Macro) { + self.0 = true; + } + } + + let mut visitor = HasMacro(false); + visitor.visit_type(self); + visitor.0 + } + fn replace_lifetimes(&self, needle: &[&Lifetime], replacement: &[&Lifetime]) -> Type { if needle.is_empty() { return self.clone(); } - parse_quote!( - < - for<#(#needle,)*> fn(#(&#needle (),)*) -> #self - as - ::pin_init::__internal::FnOutput<(#(&#replacement (),)*)> - >::Output - ) + // If the type has macro, we cannot peek into it. Use some different approach to replace + // the type using GAT. + if self.has_macro() { + return parse_quote!( + < + for<#(#needle,)*> fn(#(&#needle (),)*) -> #self + as + ::pin_init::__internal::FnOutput<(#(&#replacement (),)*)> + >::Output + ); + } + + struct LifetimeReplacer<'a> { + to_replace: BTreeMap<&'a Lifetime, &'a Lifetime>, + } + + impl<'a> LifetimeReplacer<'a> { + fn with_bound_lifetimes( + &mut self, + bound: Option<&BoundLifetimes>, + f: impl FnOnce(&mut Self), + ) { + // In case the type includes a lifetime binder, e.g. `dyn for<'a> Foo`, + // temporarily remove them from to_replace if they're. + + let mut removed = Vec::new(); + if let Some(bound) = bound { + for lt in &bound.lifetimes { + let GenericParam::Lifetime(lt) = lt else { + continue; + }; + if let Some(entry) = self.to_replace.remove_entry(<.lifetime) { + removed.push(entry); + } + } + } + + f(self); + + for (key, val) in removed { + self.to_replace.insert(key, val); + } + } + } + + impl VisitMut for LifetimeReplacer<'_> { + fn visit_lifetime_mut(&mut self, lt: &mut syn::Lifetime) { + if let Some(&replacement) = self.to_replace.get(lt) { + *lt = replacement.clone(); + } + } + + fn visit_trait_bound_mut(&mut self, bound: &mut syn::TraitBound) { + self.with_bound_lifetimes(bound.lifetimes.as_ref(), |this| { + this.visit_path_mut(&mut bound.path) + }); + } + + fn visit_type_bare_fn_mut(&mut self, bare_fn: &mut syn::TypeBareFn) { + self.with_bound_lifetimes(bare_fn.lifetimes.as_ref(), |this| { + for input in bare_fn.inputs.iter_mut() { + this.visit_bare_fn_arg_mut(input); + } + + this.visit_return_type_mut(&mut bare_fn.output); + }); + } + } + + let mut ret = self.clone(); + LifetimeReplacer { + to_replace: needle + .iter() + .copied() + .zip(replacement.iter().copied()) + .collect(), + } + .visit_type_mut(&mut ret); + ret } } From e2ab80587d02ef262df78a48660eef06adeb133b Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 8 Sep 2026 23:54:51 +0100 Subject: [PATCH 72/91] rust: pin-init: internal: pin_data: support shared projection For fields that self-references, it is desirable that projection can happen on shared references too, so lifetime between different fields can be correlated. Add support for that with `with_project_ref`. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-19-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 134 +++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 4b361c510a40..8ab71544f674 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1145,10 +1145,17 @@ fn generate_projections(info: &StructInfo) -> TokenStream { &info.field_lts_outlive_chain, generics, ]); + let generics_with_this_field_ref_lt = CombinedGenerics(vec![ + &this_lt_generics, + &info.field_lts_split_variance_outlive_chain, + generics, + ]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); let (_, ty_generics_with_this_lt, _) = generics_with_this_lt.split_for_impl(); let (_, ty_generics_with_this_field_lt, _) = generics_with_this_field_lt.split_for_impl(); + let (_, ty_generics_with_this_field_ref_lt, _) = + generics_with_this_field_ref_lt.split_for_impl(); let this = format_ident!("this"); @@ -1462,6 +1469,109 @@ fn generate_projections(info: &StructInfo) -> TokenStream { } } + let (fields_ref_decl_lt, fields_ref_proj_lt): (Vec<_>, Vec<_>) = info + .fields + .iter() + .map(|f| { + let vis = &f.field.vis; + let ident = f.member.as_ident(); + let member = &f.member; + let name = (!info.is_tuple_struct).then(|| quote!(#ident:)); + + let ty = &f.field.ty; + + let mut accessor = quote!(&#this.#member); + if !f.captures.is_empty() || f.borrowed.is_some() { + accessor = quote!( + // SAFETY: we have `Erase<..>` which we know is layout compatible with `f.ty`. + // We cannot include explicit type name here as the field lifetimes are nameable + // in this context. + unsafe { ::core::mem::transmute(#accessor) } + ) + } + + // In `with_project`, borrowed fields have their field lifetime available, so use it + // instead of `'__this`. + let lt = if f.borrowed.is_some() { + Lifetime::from_ident(&ident) + } else { + this_lt.clone() + }; + + if matches!( + f.borrowed, + Some(BorrowedInfo { + kind: BorrowedKind::Mutable, + .. + }) + ) { + // Mutable borrow must be omitted for aliasing reason. + ( + quote!( + #vis #name ::pin_init::__internal::NotVisible<&#lt #ty>, + ), + quote!( + #name ::pin_init::__internal::NotVisible::new(), + ), + ) + } else if f.pinned { + ( + quote!( + #vis #name ::core::pin::Pin<&#lt #ty>, + ), + quote!( + // SAFETY: this field is structurally pinned. + #name unsafe { ::core::pin::Pin::new_unchecked(#accessor) }, + ), + ) + } else { + ( + quote!( + #vis #name &#lt #ty, + ), + quote!( + #name #accessor, + ), + ) + } + }) + .collect(); + + let projection_ref_lt = format_ident!("__ProjectionRef"); + let (projection_ref_lt_def, projection_ref_lt_init) = if info.is_tuple_struct { + ( + quote!( + #vis struct #projection_ref_lt #generics_with_this_field_ref_lt( + #(#fields_ref_decl_lt)* + ::core::marker::PhantomData<&'__this #ident #ty_generics>, + ) #whr; + ), + quote!( + #projection_ref_lt( + #(#fields_ref_proj_lt)* + ::core::marker::PhantomData, + ) + ), + ) + } else { + ( + quote! { + #vis struct #projection_ref_lt #generics_with_this_field_ref_lt + #whr + { + #(#fields_ref_decl_lt)* + ___pin_phantom_data: ::core::marker::PhantomData<&'__this #ident #ty_generics>, + } + }, + quote! { + #projection_ref_lt { + #(#fields_ref_proj_lt)* + ___pin_phantom_data: ::core::marker::PhantomData, + } + }, + ) + }; + quote! { #[doc = #docs] // Allow `non_snake_case` since the same warning will be emitted on @@ -1477,6 +1587,12 @@ fn generate_projections(info: &StructInfo) -> TokenStream { #[doc(hidden)] #projection_lt_def + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(dead_code, non_snake_case)] + #[doc(hidden)] + #projection_ref_lt_def + impl #impl_generics #ident #ty_generics #whr { @@ -1515,6 +1631,24 @@ impl #impl_generics #ident #ty_generics f(#projection_lt_init) } + /// Pin-projects all fields of `Self` from a shared reference with proper lifetime. + /// + /// These fields are structurally pinned: + #(#[doc = #structurally_pinned_fields_docs])* + /// + /// These fields are **not** structurally pinned: + #(#[doc = #not_structurally_pinned_fields_docs])* + #[inline] + #vis fn with_project_ref<'__this, R>( + self: ::core::pin::Pin<&'__this Self>, + f: impl for #field_lt_ty_generics ::core::ops::FnOnce( + #projection_ref_lt #ty_generics_with_this_field_ref_lt + ) -> R, + ) -> R { + let #this = ::core::pin::Pin::get_ref(self); + f(#projection_ref_lt_init) + } + #(#accessors)* } } From 73cf61724648ad57dd5664a0e2915e20fd304650 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 9 Sep 2026 23:23:20 +0100 Subject: [PATCH 73/91] rust: pin-init: internal: pin_data: support existential lifetimes There are many cases where structs that have interior mutability, but they do not need the invariance over captured lifetimes as the lifetime is captured upon construction, and new data of that particular lifetime does not flow back into the struct. For these use cases, the same mechanism as pin-init self-reference may be used. Add support for existential lifetimes, introduced by having where clauses such as exists<'a>: 'b The lifetime `'a` above is minted similar to field lifetimes. Because `'a` is an erased lifetime living longer than `'b`, the only variance requirement that we have is that `'b` cannot be contravariant; that defense is fulfilled by adding `PhantomData<&'b ()>` so the struct is either covariant or invariant over `'b`. Acked-by: Benno Lossin Link: https://patch.msgid.link/20261008-dev-selfref-v2-20-e280b3c8fba5@garyguo.net/ Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 253 +++++++++++++++++++++++-- rust/pin-init/src/__internal.rs | 15 ++ 2 files changed, 252 insertions(+), 16 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 8ab71544f674..dbf9084bfd22 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -11,8 +11,9 @@ spanned::Spanned, visit::Visit, visit_mut::VisitMut, - Attribute, Field, Fields, GenericParam, Generics, Ident, Index, Item, ItemStruct, Lifetime, - LifetimeParam, Member, Meta, PathSegment, Token, Type, TypePath, + Attribute, Field, Fields, GenericArgument, GenericParam, Generics, Ident, Index, Item, + ItemStruct, Lifetime, LifetimeParam, Member, Meta, PathArguments, PathSegment, Token, Type, + TypeParamBound, TypePath, WhereClause, WherePredicate, }; use crate::{ @@ -183,6 +184,7 @@ fn parse_list( dcx: &mut DiagCtxt, attrs: &mut Vec, bound_lifetimes: &BTreeSet<&Lifetime>, + exist_lifetimes: &BTreeSet, field_idx_map: &BTreeMap, ) -> Option<(BTreeSet, Variance)> { let attr = attrs.extract_single_attr(dcx, "uses")?; @@ -208,7 +210,10 @@ fn parse_list( continue; } - if lt.ident != "_" && !field_idx_map.contains_key(<.ident) { + if lt.ident != "_" + && !exist_lifetimes.contains(lt) + && !field_idx_map.contains_key(<.ident) + { dcx.error(lt, format!("`{lt}` is not a field name")); continue; } @@ -240,11 +245,15 @@ struct StructInfo { field_idx_map: BTreeMap, is_tuple_struct: bool, self_referential: bool, + /// Existential lifetimes defined. + exist_lts: BTreeSet, /// Field lifetime genercis with outlive chain. field_lts_outlive_chain: Generics, /// Field lifetime genercis with outlive chain, with one chain for covariant fields and one /// chain for invariant fields. field_lts_split_variance_outlive_chain: Generics, + /// Existential lifetime with their outlives bounds. + exist_lts_generics: Generics, } pub(crate) fn expand_with_cfg( @@ -334,6 +343,12 @@ fn expand( let is_tuple_struct = matches!(struct_.fields, Fields::Unnamed(_)); + let exist_lts = if let Some(whr) = &mut struct_.generics.where_clause { + ExistLt::parse(dcx, whr) + } else { + BTreeSet::new() + }; + // Collect all bound lifetimes from generics. let bound_lifetimes: BTreeSet<&Lifetime> = struct_.generics.lifetimes().map(|x| &x.lifetime).collect(); @@ -347,6 +362,16 @@ fn expand( .filter_map(|(index, field)| Some((field.ident.clone()?, index))) .collect(); + for l in &exist_lts { + let lt = &l.lifetime; + if bound_lifetimes.contains(<) { + dcx.error( + lt, + format!("existential `{lt}` conflicts with struct generics"), + ); + } + } + // Keep track on fields being implicitly borrowed by being mentioned. let mut implicitly_borrowed = BTreeSet::new(); @@ -370,9 +395,14 @@ fn expand( }; // Parse `#[uses]` attribute. - let (mut captures, wildcard_variance) = - Capture::parse_list(dcx, &mut field.attrs, &bound_lifetimes, &field_idx_map) - .unwrap_or_default(); + let (mut captures, wildcard_variance) = Capture::parse_list( + dcx, + &mut field.attrs, + &bound_lifetimes, + &exist_lts, + &field_idx_map, + ) + .unwrap_or_default(); let mut generic_lt_captures = BTreeSet::new(); let mut generic_ty_captures = BTreeSet::new(); @@ -399,7 +429,7 @@ fn expand( return; } - if !field_idx_map.contains_key(<.ident) { + if !exist_lts.contains(lt) && !field_idx_map.contains_key(<.ident) { dcx.error( lt, format!("`{lt}` is neither a lifetime in generics nor a field name"), @@ -414,8 +444,10 @@ fn expand( }) .visit_type(&field.ty); - for capture in captures.iter() { - implicitly_borrowed.insert(capture.lifetime.ident.clone()); + for capture in captures.iter(){ + if !exist_lts.contains(&capture.lifetime){ + implicitly_borrowed.insert(capture.lifetime.ident.clone()); + } } GenericParam::maybe_type_params_visitor(|ident| { @@ -495,6 +527,9 @@ fn expand( let mut worklist = Vec::new(); for field in fields.iter() { for borrow in field.captures.iter() { + if exist_lts.contains(&borrow.lifetime) { + continue; + } if let Variance::Invariant = borrow.variance { let Some(borrow_idx) = fields .iter() @@ -510,6 +545,9 @@ fn expand( } while let Some(field) = worklist.pop() { for borrow in field.captures.iter() { + if exist_lts.contains(&borrow.lifetime) { + continue; + } let Some(borrow_idx) = fields .iter() .position(|f| f.member.as_ident() == borrow.lifetime.ident) @@ -581,6 +619,23 @@ fn expand( } } + let exist_lt_generics = Generics { + lt_token: Some(Default::default()), + params: exist_lts + .iter() + .map(|l| { + GenericParam::Lifetime(LifetimeParam { + attrs: Vec::new(), + lifetime: l.lifetime.clone(), + colon_token: Default::default(), + bounds: l.bounds.iter().cloned().collect(), + }) + }) + .collect(), + gt_token: Some(Default::default()), + where_clause: None, + }; + let mut field_lts_split_variance_outlive_chain = Generics { lt_token: Some(Default::default()), params: borrowed_covariant_fields @@ -642,6 +697,16 @@ fn expand( .push(parse_quote!(#ty: #field_lt)); } + for borrow in field.captures.iter().rev() { + let lt = &borrow.lifetime; + if exist_lts.contains(lt) { + field_lts_split_variance_outlive_chain + .make_where_clause() + .predicates + .push(parse_quote!(#lt: #field_lt)); + } + } + // If a field is invariant, then the invariance closure rule will make all borrowed fields // to be invariant, so they're already captured in `field_lts_split_variance_outlive_chain`. if borrowed.lt_variance != Variance::Covariant { @@ -681,8 +746,10 @@ fn expand( fields, field_idx_map, is_tuple_struct, + exist_lts, field_lts_outlive_chain, field_lts_split_variance_outlive_chain, + exist_lts_generics: exist_lt_generics, }; for field in &info.fields { @@ -745,6 +812,129 @@ fn is_phantom_pinned(ty: &Type) -> bool { } } +struct ExistLt { + lifetime: Lifetime, + bounds: Vec, +} + +impl std::borrow::Borrow for ExistLt { + fn borrow(&self) -> &Lifetime { + &self.lifetime + } +} + +impl PartialEq for ExistLt { + fn eq(&self, other: &Self) -> bool { + self.lifetime == other.lifetime + } +} + +impl Eq for ExistLt {} + +impl PartialOrd for ExistLt { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ExistLt { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lifetime.cmp(&other.lifetime) + } +} + +impl ExistLt { + fn parse(dcx: &mut DiagCtxt, whr: &mut WhereClause) -> BTreeSet { + fn parse_one( + dcx: &mut DiagCtxt, + pred: &WherePredicate, + result: &mut BTreeSet, + ) -> bool { + let WherePredicate::Type(pred) = &pred else { + return false; + }; + let Type::Path(path) = &pred.bounded_ty else { + return false; + }; + + if path.qself.is_some() + || path.path.leading_colon.is_some() + || path.path.segments.len() != 1 + { + return false; + } + let path_seg = &path.path.segments[0]; + + if path_seg.ident != "exists" { + return false; + }; + + let PathArguments::AngleBracketed(p) = &path_seg.arguments else { + dcx.error( + path_seg, + "existential clauses require a single lifetime, e.g. `exists<'a>`", + ); + return true; + }; + + if p.args.len() != 1 { + dcx.error( + path_seg, + "existential clauses require a single lifetime, e.g. `exists<'a>`", + ); + return true; + } + + let GenericArgument::Lifetime(lt) = &p.args[0] else { + dcx.error( + path_seg, + "existential clauses require a single lifetime, e.g. `exists<'a>`", + ); + return true; + }; + + if result.contains(lt) { + dcx.error( + lt, + format!("an existential clause for `{lt}` already exists"), + ); + return true; + } + + let mut bounds = Vec::new(); + for bound in pred.bounds.iter() { + let TypeParamBound::Lifetime(lt) = bound else { + dcx.error(bound, "only lifetime can appear in existential clauses"); + return true; + }; + bounds.push(lt.clone()); + } + if bounds.is_empty() { + dcx.error( + pred.colon_token, + "existential clauses require at least one outlive bounds", + ); + return true; + } + + result.insert(ExistLt { + lifetime: lt.clone(), + bounds: bounds.clone(), + }); + + true + } + + let mut result = BTreeSet::new(); + whr.predicates = std::mem::take(&mut whr.predicates) + .into_pairs() + .filter(|p| !parse_one(dcx, p.value(), &mut result)) + .collect(); + + result + } +} + fn generate_struct_def(info: &StructInfo) -> TokenStream { let ItemStruct { attrs, @@ -783,12 +973,23 @@ fn generate_struct_def(info: &StructInfo) -> TokenStream { // implementation and thus may be used inside `Erase`. ty = quote!((#ty,)); + let mut exist_lt_phantoms = Vec::new(); for borrow in field.captures.iter().rev() { let lt = &borrow.lifetime; ty = quote!(for<#lt> fn(&#lt()) -> #ty); + + if let Some(exist_lt) = info.exist_lts.get(lt) { + for bound in &exist_lt.bounds { + exist_lt_phantoms.push(quote!(::pin_init::__internal::ExistLt<#bound>)); + } + } } ty = quote!(::pin_init::__internal::Erase<#ty>); + + if !exist_lt_phantoms.is_empty() { + ty = quote!(::pin_init::__internal::WithPhantom<#ty, (#(#exist_lt_phantoms,)*)>); + } }; if let Some(borrowed) = &field.borrowed { @@ -998,8 +1199,11 @@ fn generate_drop_order_check(dcx: &mut DiagCtxt, info: &StructInfo) -> TokenStre // with known lifetime bounds as bounds on the function, and asks Rust to *prove* that the types // are wellformed, given the bounds that we understand. - let generics_with_field_lt = - CombinedGenerics(vec![&info.field_lts_split_variance_outlive_chain, generics]); + let generics_with_field_lt = CombinedGenerics(vec![ + &info.exist_lts_generics, + &info.field_lts_split_variance_outlive_chain, + generics, + ]); let (_, ty_generics, _) = generics.split_for_impl(); let (impl_generics_with_field_lt, _, whr_with_field_lt) = @@ -1138,20 +1342,25 @@ fn generate_projections(info: &StructInfo) -> TokenStream { // Wrap in `CombinedGenerics` because it's ty generics will always output `<>`, so it can be // used with `for`. - let field_lts = CombinedGenerics(vec![&info.field_lts_outlive_chain]); + let field_lts = CombinedGenerics(vec![ + &info.field_lts_outlive_chain, + &info.exist_lts_generics, + ]); let generics_with_this_lt = CombinedGenerics(vec![&this_lt_generics, generics]); let generics_with_this_field_lt = CombinedGenerics(vec![ &this_lt_generics, + &info.exist_lts_generics, &info.field_lts_outlive_chain, generics, ]); let generics_with_this_field_ref_lt = CombinedGenerics(vec![ &this_lt_generics, + &info.exist_lts_generics, &info.field_lts_split_variance_outlive_chain, generics, ]); - let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, ty_generics_with_this_lt, _) = generics_with_this_lt.split_for_impl(); let (_, ty_generics_with_this_field_lt, _) = generics_with_this_field_lt.split_for_impl(); let (_, ty_generics_with_this_field_ref_lt, _) = @@ -1665,10 +1874,19 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { // Wrap in `CombinedGenerics` because it's ty generics will always output `<>`, so it can be // used with `for`. let field_lts = CombinedGenerics(vec![&info.field_lts_outlive_chain]); - let generics_with_field_lt = CombinedGenerics(vec![&info.field_lts_outlive_chain, generics]); + let field_exist_lts = CombinedGenerics(vec![ + &info.field_lts_outlive_chain, + &info.exist_lts_generics, + ]); + let generics_with_field_lt = CombinedGenerics(vec![ + &info.field_lts_outlive_chain, + &info.exist_lts_generics, + generics, + ]); let (impl_generics, ty_generics, whr) = generics.split_for_impl(); let (_, field_lt_ty_generics, _) = field_lts.split_for_impl(); + let (_, field_exist_lt_ty_generics, _) = field_exist_lts.split_for_impl(); let (impl_generics_with_lt, ty_generics_with_field_lt, whr_with_field_lt) = generics_with_field_lt.split_for_impl(); @@ -1778,6 +1996,7 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { }) .collect::(); + let exist_lt_generics_params = info.exist_lts_generics.params.iter(); quote! { // We declare this struct which will host all of the projection function for our type. #[doc(hidden)] @@ -1833,7 +2052,7 @@ impl #impl_generics __ThePinData #ty_generics { /// Type inference helper function. #[inline(always)] - #vis fn __make_closure<__F, __E>(self, f: __F) -> __F + #vis fn __make_closure<#(#exist_lt_generics_params,)* __F, __E>(self, f: __F) -> __F where __F: for #field_lt_ty_generics ::core::ops::FnOnce( *mut #struct_name #ty_generics, @@ -1844,7 +2063,9 @@ impl #impl_generics __ThePinData #ty_generics } #[inline(always)] - #vis fn __with_lt #field_lts(self) -> __PinDataLt #ty_generics_with_field_lt { + #vis fn __with_lt #field_exist_lt_ty_generics(self) + -> __PinDataLt #ty_generics_with_field_lt + { // Generate a zeroed to avoid naming all fields. // SAFETY: `__PinDataLt` only contains phantom fields. unsafe { ::core::mem::zeroed() } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 9f0f5a34e630..04abb7590aa5 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -725,3 +725,18 @@ pub fn new() -> Self { Self(PhantomData) } } + +/// Marker type to capture outlive bounds coming from existential lifetimes. +pub struct ExistLt<'a>(PhantomData<&'a ()>); + +// Dummy `Drop`, same reason as `Borrowed`. +impl Drop for ExistLt<'_> { + #[inline(always)] + fn drop(&mut self) {} +} + +/// Type that allows additional `PhantomData` to be attached. +/// +/// This allows changing variance of types without introducing additional fields. +#[repr(transparent)] +pub struct WithPhantom(PhantomData

, T); From 4608d8d060bff286b62fdf64f708e20d5b1f7576 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sun, 4 Oct 2026 21:42:11 +0900 Subject: [PATCH 74/91] MAINTAINERS: add entry for the Rust `bug` module Add a MAINTAINERS entry for the Rust `bug` module, which currently provides the `warn_on!` macro. Suggested-by: Miguel Ojeda Signed-off-by: FUJITA Tomonori Link: https://patch.msgid.link/20261004124211.381401-1-tomo@flapping.org Signed-off-by: Miguel Ojeda --- MAINTAINERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index db263cefb46b..ffdf8aa430be 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23990,6 +23990,15 @@ L: rust-for-linux@vger.kernel.org S: Maintained F: rust/kernel/bitfield.rs +RUST [BUG] +M: FUJITA Tomonori +L: rust-for-linux@vger.kernel.org +S: Maintained +F: rust/helpers/bug.c +F: rust/kernel/bug.rs +F: rust/kernel/generated_arch_reachable_asm.rs.S +F: rust/kernel/generated_arch_warn_asm.rs.S + RUST [INTEROP] M: Joel Fernandes M: Alexandre Courbot From b7218f1996f8491dbe28632b70f444cb73f554df Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sun, 11 Oct 2026 14:32:18 +0100 Subject: [PATCH 75/91] rust: pin-init: internal: suppress `explicit_outlives_requirements` `#[pin_data]` inserts outlive bounds between lifetimes to allow closure callbacks to rely on their relationship. In some cases, these relations are implied by fields, and Rust generates `explicit_outlives_requirements` allow-by-default lint, which is turned out by kernel as part of rust-2018-idioms lint group. As `#[pin_data]` is a proc macro and it cannot know if the bounds are already implied, suppress such lints for generated code. Link: https://patch.msgid.link/20261011133904.600785-1-gary@kernel.org Signed-off-by: Gary Guo --- rust/pin-init/internal/src/pin_data.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index dbf9084bfd22..08e11128d353 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1792,13 +1792,13 @@ fn generate_projections(info: &StructInfo) -> TokenStream { #[doc = #docs] // Allow `non_snake_case` since the same warning will be emitted on // the struct definition. - #[allow(dead_code, non_snake_case)] + #[allow(dead_code, non_snake_case, explicit_outlives_requirements)] #[doc(hidden)] #projection_lt_def // Allow `non_snake_case` since the same warning will be emitted on // the struct definition. - #[allow(dead_code, non_snake_case)] + #[allow(dead_code, non_snake_case, explicit_outlives_requirements)] #[doc(hidden)] #projection_ref_lt_def @@ -2000,7 +2000,7 @@ fn generate_the_pin_data(info: &StructInfo) -> TokenStream { quote! { // We declare this struct which will host all of the projection function for our type. #[doc(hidden)] - #[allow(non_snake_case)] + #[allow(non_snake_case, explicit_outlives_requirements)] #vis struct __PinDataLt #generics_with_field_lt #whr_with_field_lt { From 3a4e0ee57c5563aa997ea6d6b887a915e035e8e0 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Oct 2026 14:30:01 +0000 Subject: [PATCH 76/91] rust: sync: move lockdep types to rust/kernel/sync/lockdep.rs The lockdep types are currently stored directly in rust/kernel/sync.rs, but there are starting to be too many of them to keep them in that file. Thus, move them to a submodule. The new module is listed under LOCKING PRIMITIVES in the MAINTAINERS file. For commonly used lockdep logic it's useful to keep re-exports in kernel::sync, and this also avoids the need to update any users. Reviewed-by: Gary Guo Reviewed-by: Boqun Feng Reviewed-by: Carlos Llamas Acked-by: Miguel Ojeda Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20261007-pr-ratelimited-v6-1-f14accd4ea3b@google.com [ Used the "kernel vertical" imports style. - Miguel ] Signed-off-by: Miguel Ojeda --- MAINTAINERS | 1 + rust/kernel/sync.rs | 138 ++--------------------------------- rust/kernel/sync/lockdep.rs | 139 ++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 133 deletions(-) create mode 100644 rust/kernel/sync/lockdep.rs diff --git a/MAINTAINERS b/MAINTAINERS index ffdf8aa430be..1e7f1329c348 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15284,6 +15284,7 @@ F: rust/helpers/mutex.c F: rust/helpers/spinlock.c F: rust/kernel/sync/lock.rs F: rust/kernel/sync/lock/ +F: rust/kernel/sync/lockdep.rs F: rust/kernel/sync/locked_by.rs X: kernel/locking/locktorture.c diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 3e55de3b1636..ee372c048a8a 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -5,10 +5,6 @@ //! This module contains the kernel APIs related to synchronisation that have been ported or //! wrapped for usage by Rust code in the kernel. -use crate::prelude::*; -use crate::types::Opaque; -use pin_init; - mod arc; pub mod aref; pub mod atomic; @@ -16,6 +12,7 @@ pub mod completion; mod condvar; pub mod lock; +pub mod lockdep; mod locked_by; pub mod poll; pub mod rcu; @@ -36,136 +33,11 @@ SpinLockIrq, SpinLockIrqGuard, // }; +pub use lockdep::{ + static_lock_class, + LockClassKey, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; pub use srcu::Srcu; - -/// Represents a lockdep class. -/// -/// Wraps the kernel's `struct lock_class_key`. -#[repr(transparent)] -#[pin_data(PinnedDrop)] -pub struct LockClassKey { - #[pin] - inner: Opaque, -} - -// SAFETY: Unregistering a lock class key from a different thread than where it was registered is -// allowed. -unsafe impl Send for LockClassKey {} - -// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and -// provides its own synchronization. -unsafe impl Sync for LockClassKey {} - -impl LockClassKey { - /// Initializes a statically allocated lock class key. - /// - /// This is usually used indirectly through the [`static_lock_class!`] macro. See its - /// documentation for more information. - /// - /// # Safety - /// - /// * Before using the returned value, it must be pinned in a static memory location. - /// * The destructor must never run on the returned `LockClassKey`. - pub const unsafe fn new_static() -> Self { - LockClassKey { - inner: Opaque::uninit(), - } - } - - /// Initializes a dynamically allocated lock class key. - /// - /// In the common case of using a statically allocated lock class key, the - /// [`static_lock_class!`] macro should be used instead. - /// - /// # Examples - /// - /// ``` - /// use kernel::alloc::KBox; - /// use kernel::types::ForeignOwnable; - /// use kernel::sync::{LockClassKey, SpinLock}; - /// use pin_init::stack_pin_init; - /// - /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; - /// let key_ptr = key.into_foreign(); - /// - /// { - /// stack_pin_init!(let num: SpinLock = SpinLock::new( - /// 0, - /// c"my_spinlock", - /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose - /// // `from_foreign()` has not yet been called. - /// unsafe { > as ForeignOwnable>::borrow(key_ptr) } - /// )); - /// } - /// - /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous - /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. - /// unsafe { drop(> as ForeignOwnable>::from_foreign(key_ptr)) }; - /// # Ok::<(), Error>(()) - /// ``` - pub fn new_dynamic() -> impl PinInit { - pin_init!(Self { - // SAFETY: lockdep_register_key expects an uninitialized block of memory - inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) - }) - } - - /// Returns a raw pointer to the inner C struct. - /// - /// It is up to the caller to use the raw pointer correctly. - pub fn as_ptr(&self) -> *mut bindings::lock_class_key { - self.inner.get() - } -} - -#[pinned_drop] -impl PinnedDrop for LockClassKey { - fn drop(self: Pin<&mut Self>) { - // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address - // hasn't changed. Thus, it's safe to pass it to unregister. - unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } - } -} - -/// Defines a new static lock class and returns a pointer to it. -/// -/// # Examples -/// -/// ``` -/// use kernel::sync::{static_lock_class, Arc, SpinLock}; -/// -/// fn new_locked_int() -> Result>> { -/// Arc::pin_init(SpinLock::new( -/// 42, -/// c"new_locked_int", -/// static_lock_class!(), -/// ), GFP_KERNEL) -/// } -/// ``` -#[macro_export] -macro_rules! static_lock_class { - () => {{ - static CLASS: $crate::sync::LockClassKey = - // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop - // never runs on a static global. - unsafe { $crate::sync::LockClassKey::new_static() }; - $crate::prelude::Pin::static_ref(&CLASS) - }}; -} -pub use static_lock_class; - -/// Returns the given string, if one is provided, otherwise generates one based on the source code -/// location. -#[doc(hidden)] -#[macro_export] -macro_rules! optional_name { - () => { - $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) - }; - ($name:literal) => { - $crate::c_str!($name) - }; -} diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs new file mode 100644 index 000000000000..784821cc2a39 --- /dev/null +++ b/rust/kernel/sync/lockdep.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Utilities related to lockdep. +//! +//! C headers: [`include/linux/lockdep.h`](srctree/include/linux/lockdep.h) + +use crate::{ + prelude::*, + types::Opaque, // +}; + +/// Represents a lockdep class. +/// +/// Wraps the kernel's `struct lock_class_key`. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct LockClassKey { + #[pin] + inner: Opaque, +} + +// SAFETY: Unregistering a lock class key from a different thread than where it was registered is +// allowed. +unsafe impl Send for LockClassKey {} + +// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and +// provides its own synchronization. +unsafe impl Sync for LockClassKey {} + +impl LockClassKey { + /// Initializes a statically allocated lock class key. + /// + /// This is usually used indirectly through the [`static_lock_class!`] macro. See its + /// documentation for more information. + /// + /// # Safety + /// + /// * Before using the returned value, it must be pinned in a static memory location. + /// * The destructor must never run on the returned `LockClassKey`. + pub const unsafe fn new_static() -> Self { + LockClassKey { + inner: Opaque::uninit(), + } + } + + /// Initializes a dynamically allocated lock class key. + /// + /// In the common case of using a statically allocated lock class key, the + /// [`static_lock_class!`] macro should be used instead. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::KBox; + /// use kernel::types::ForeignOwnable; + /// use kernel::sync::{LockClassKey, SpinLock}; + /// use pin_init::stack_pin_init; + /// + /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; + /// let key_ptr = key.into_foreign(); + /// + /// { + /// stack_pin_init!(let num: SpinLock = SpinLock::new( + /// 0, + /// c"my_spinlock", + /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose + /// // `from_foreign()` has not yet been called. + /// unsafe { > as ForeignOwnable>::borrow(key_ptr) } + /// )); + /// } + /// + /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous + /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. + /// unsafe { drop(> as ForeignOwnable>::from_foreign(key_ptr)) }; + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_dynamic() -> impl PinInit { + pin_init!(Self { + // SAFETY: lockdep_register_key expects an uninitialized block of memory + inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) + }) + } + + /// Returns a raw pointer to the inner C struct. + /// + /// It is up to the caller to use the raw pointer correctly. + pub fn as_ptr(&self) -> *mut bindings::lock_class_key { + self.inner.get() + } +} + +#[pinned_drop] +impl PinnedDrop for LockClassKey { + fn drop(self: Pin<&mut Self>) { + // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address + // hasn't changed. Thus, it's safe to pass it to unregister. + unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } + } +} + +/// Defines a new static lock class and returns a pointer to it. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{static_lock_class, Arc, SpinLock}; +/// +/// fn new_locked_int() -> Result>> { +/// Arc::pin_init(SpinLock::new( +/// 42, +/// c"new_locked_int", +/// static_lock_class!(), +/// ), GFP_KERNEL) +/// } +/// ``` +#[macro_export] +macro_rules! static_lock_class { + () => {{ + static CLASS: $crate::sync::LockClassKey = + // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop + // never runs on a static global. + unsafe { $crate::sync::LockClassKey::new_static() }; + $crate::prelude::Pin::static_ref(&CLASS) + }}; +} +pub use static_lock_class; + +/// Returns the given string, if one is provided, otherwise generates one based on the source code +/// location. +#[doc(hidden)] +#[macro_export] +macro_rules! optional_name { + () => { + $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) + }; + ($name:literal) => { + $crate::c_str!($name) + }; +} From 107a25d62c40ef47b2284ee37fdefc1d3f5d013b Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Oct 2026 14:30:02 +0000 Subject: [PATCH 77/91] rust: sync: add const constructor for raw_spinlock_t The abstractions for pr_*_ratelimited! need to construct a global `struct ratelimit_state`, which contains a `raw_spinlock_t` field. Thus, add a const constructor for the `raw_spinlock_t` type. The SPINLOCK_OWNER_INIT constant isn't mirrored via a const helper because bindgen generates a 'static mut' instead of a constant from the pointer constant., The __ARCH_SPIN_LOCK_UNLOCKED constant cannot be translated by bindgen because it's a define for a struct without type annotations, so it's explicitly declared in Rust. Reviewed-by: Boqun Feng Reviewed-by: Carlos Llamas Acked-by: Miguel Ojeda Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20261007-pr-ratelimited-v6-2-f14accd4ea3b@google.com Signed-off-by: Miguel Ojeda --- include/linux/spinlock_types_raw.h | 4 ++++ rust/bindings/lib.rs | 25 +++++++++++++++++++++++++ rust/kernel/sync/lock/spinlock.rs | 29 +++++++++++++++++++++++++++++ rust/kernel/sync/lockdep.rs | 22 ++++++++++++++++++++++ 4 files changed, 80 insertions(+) diff --git a/include/linux/spinlock_types_raw.h b/include/linux/spinlock_types_raw.h index e5644ab2161f..942c229c90bb 100644 --- a/include/linux/spinlock_types_raw.h +++ b/include/linux/spinlock_types_raw.h @@ -11,6 +11,10 @@ #include +/* + * Keep in sync with rust/kernel/sync/lock/spinlock.rs + */ + context_lock_struct(raw_spinlock) { arch_spinlock_t raw_lock; #ifdef CONFIG_DEBUG_SPINLOCK diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 98313fc31024..f95295f1b4ab 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -88,3 +88,28 @@ mod bindings_helper { // The generated `HZ` is therefore `__USER_HZ` from // `include/uapi/asm-generic/param.h`, not `CONFIG_HZ`. pub const HZ: u32 = CONFIG_HZ; + +// Explicitly list architectures where this logic is checked correct. +#[cfg(any( + CONFIG_ARM, + CONFIG_ARM64, + CONFIG_LOONGARCH, + CONFIG_PPC, + CONFIG_RISCV, + CONFIG_S390, + CONFIG_UML_X86, + CONFIG_X86, +))] +pub const __ARCH_SPIN_LOCK_UNLOCKED: arch_spinlock_t = { + // SAFETY: The `arch_spinlock_t` type can be zeroed. + #[allow(unused_mut)] + let mut lock: arch_spinlock_t = unsafe { core::mem::zeroed() }; + + #[cfg(not(CONFIG_SMP))] + #[cfg(CONFIG_DEBUG_SPINLOCK)] + { + lock.slock = 1; + } + + lock +}; diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index aafc80125f59..037fa63ebcac 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -463,3 +463,32 @@ fn spinlock_irq_condvar() -> Result { Ok(()) } } + +/// Helper for creating a raw unlocked `bindings::raw_spinlock_t`. +/// +/// For use in statics containing raw spinlocks. +#[doc(alias("__SPIN_LOCK_UNLOCKED", "DEFINE_SPINLOCK"))] +#[expect(dead_code)] +pub(crate) const fn raw_spin_lock_unlocked(name: &'static CStr) -> bindings::raw_spinlock_t { + // Silence unused variable warnings. + #[cfg(not(CONFIG_DEBUG_LOCK_ALLOC))] + let _ = name; + + bindings::raw_spinlock_t { + raw_lock: bindings::__ARCH_SPIN_LOCK_UNLOCKED, + + #[cfg(CONFIG_DEBUG_SPINLOCK)] + magic: bindings::SPINLOCK_MAGIC, + #[cfg(CONFIG_DEBUG_SPINLOCK)] + owner_cpu: u32::MAX, + #[cfg(CONFIG_DEBUG_SPINLOCK)] + owner: usize::MAX as *mut c_void, + + #[cfg(CONFIG_DEBUG_LOCK_ALLOC)] + dep_map: kernel::sync::lockdep::raw_lockdep_map( + name, + kernel::sync::lockdep::LD_WAIT_SPIN, + kernel::sync::lockdep::LD_WAIT_INV, + ), + } +} diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs index 784821cc2a39..d9222be0fb29 100644 --- a/rust/kernel/sync/lockdep.rs +++ b/rust/kernel/sync/lockdep.rs @@ -137,3 +137,25 @@ macro_rules! optional_name { $crate::c_str!($name) }; } + +/// Not checked, catch all. +pub const LD_WAIT_INV: u8 = bindings::lockdep_wait_type_LD_WAIT_INV as u8; +/// Spin loops, `raw_spinlock_t` etc +pub const LD_WAIT_SPIN: u8 = bindings::lockdep_wait_type_LD_WAIT_SPIN as u8; + +/// Helper for declaring a raw `struct lockdep_map` for locks in statics. +/// +/// It's up to the caller to use the returned `struct lockdep_map` correctly. +#[cfg(CONFIG_DEBUG_LOCK_ALLOC)] +pub(crate) const fn raw_lockdep_map( + name: &'static CStr, + wait_type_inner: u8, + wait_type_outer: u8, +) -> bindings::lockdep_map { + // SAFETY: All zeros is valid for this type. + let mut map: bindings::lockdep_map = unsafe { core::mem::zeroed() }; + map.name = kernel::str::as_char_ptr_in_const_context(name); + map.wait_type_inner = wait_type_inner; + map.wait_type_outer = wait_type_outer; + map +} From ecdfcc7f2974c368fb932b16dd3c2deb4a8cbdb0 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Oct 2026 14:30:03 +0000 Subject: [PATCH 78/91] rust: add pr_*_ratelimited! macros for printing Printing can be very expensive if it occurs often, so printing that can be triggered by userspace should be rate limited. For this purpose, add a Rust wrapper around `struct ratelimit_state` and use it in the new macros. The new files will fall under the catch-all RUST maintainers entry. Tested-by: Alvin Sun Reviewed-by: Carlos Llamas Link: https://github.com/Rust-for-Linux/linux/issues/122 Acked-by: Miguel Ojeda Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20261007-pr-ratelimited-v6-3-f14accd4ea3b@google.com [ Removed unused link reference definitions. On the `static`, allowed the `non_upper_case_globals` lint so that `kernel` users (and users providing their own name) do not warn. Reworded title for typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/helpers/helpers.c | 1 + rust/helpers/ratelimit.c | 14 +++ rust/kernel/lib.rs | 1 + rust/kernel/prelude.rs | 8 ++ rust/kernel/ratelimit.rs | 201 ++++++++++++++++++++++++++++++ rust/kernel/sync/lock/spinlock.rs | 1 - 6 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 rust/helpers/ratelimit.c create mode 100644 rust/kernel/ratelimit.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 08374b163774..9bb850294a2f 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -86,6 +86,7 @@ #include "processor.c" #include "property.c" #include "pwm.c" +#include "ratelimit.c" #include "rbtree.c" #include "rcu.c" #include "refcount.c" diff --git a/rust/helpers/ratelimit.c b/rust/helpers/ratelimit.c new file mode 100644 index 000000000000..e5052f568b81 --- /dev/null +++ b/rust/helpers/ratelimit.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_ratelimit_state_init(struct ratelimit_state *rs, + int interval, int burst) +{ + ratelimit_state_init(rs, interval, burst); +} + +__rust_helper void rust_helper_ratelimit_state_exit(struct ratelimit_state *rs) +{ + ratelimit_state_exit(rs); +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 7225abc64084..a4747f75b163 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -120,6 +120,7 @@ pub mod ptr; #[cfg(CONFIG_RUST_PWM_ABSTRACTIONS)] pub mod pwm; +pub mod ratelimit; pub mod rbtree; pub mod regulator; pub mod revocable; diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index ca396f1f78a6..bcaa232205be 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -107,13 +107,21 @@ }, init::InPlaceInit, pr_alert, + pr_alert_ratelimited, pr_crit, + pr_crit_ratelimited, pr_debug, + pr_debug_ratelimited, pr_emerg, + pr_emerg_ratelimited, pr_err, + pr_err_ratelimited, pr_info, + pr_info_ratelimited, pr_notice, + pr_notice_ratelimited, pr_warn, + pr_warn_ratelimited, str::CStrExt as _, try_init, try_pin_init, diff --git a/rust/kernel/ratelimit.rs b/rust/kernel/ratelimit.rs new file mode 100644 index 000000000000..2c54099cf11e --- /dev/null +++ b/rust/kernel/ratelimit.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rate limiting support. +//! +//! C header: [`include/linux/ratelimit.h`](srctree/include/linux/ratelimit.h) + +use crate::{ + bindings, + prelude::*, + types::Opaque, // +}; + +/// Defines a `static` containing a [`Ratelimit`]. +#[macro_export] +macro_rules! ratelimit_state_init { + ($name:ident, $interval:expr, $burst:expr $(,)?) => { + #[allow(non_upper_case_globals)] + static $name: $crate::ratelimit::Ratelimit = { + let name = $crate::c_str!(::core::stringify!($name)); + let interval = $interval; + let burst = $burst; + // SAFETY: This will be stored in static memory. + unsafe { $crate::ratelimit::Ratelimit::new_static(name, interval, burst) } + }; + }; +} +pub use ratelimit_state_init; + +/// Rate limiter state. +/// +/// # Invariants +/// +/// The `inner` field contains an initialized `struct ratelimit_state`. +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct Ratelimit { + #[pin] + inner: Opaque, +} + +// SAFETY: `Ratelimit` is safe to be sent to any task. +unsafe impl Send for Ratelimit {} + +// SAFETY: `Ratelimit` is safe to be accessed concurrently as it is protected by an internal +// spinlock. +unsafe impl Sync for Ratelimit {} + +impl Ratelimit { + /// Constructs a [`Ratelimit`] with the specified configuration. + /// + /// If `interval` is zero, then no rate limit is applied. + #[inline] + pub fn new(interval: i32, burst: i32) -> impl PinInit { + // INVARIANT: This creates a `Ratelimit` containing an initialized `struct ratelimit_state` + pin_init!(Self { + inner <- Opaque::ffi_init(|slot: *mut bindings::ratelimit_state| { + // SAFETY: `slot` is a valid pointer to an uninitialized `struct ratelimit_state`. + // The memory is pinned so it remains valid until `ratelimit_state_exit` is called. + unsafe { bindings::ratelimit_state_init(slot, interval, burst) }; + }), + }) + } + + /// Constructs a [`Ratelimit`] with the default configuration. + #[inline] + pub fn new_default() -> impl PinInit { + Ratelimit::new(Ratelimit::DEFAULT_INTERVAL, Ratelimit::DEFAULT_BURST) + } + + /// Constructs a [`Ratelimit`] with the specified configuration. + /// + /// The name will be used for the lockdep name of the internal spinlock. See [`Self::new`] for + /// the meaning of `interval` and `burst`. + /// + /// # Safety + /// + /// The resulting value must be stored in static memory. + #[inline] + pub const unsafe fn new_static(name: &'static CStr, interval: i32, burst: i32) -> Self { + // INVARIANT: This creates a `Ratelimit` containing an initialized `struct ratelimit_state`. + Self { + inner: Opaque::new(bindings::ratelimit_state { + lock: kernel::sync::lock::spinlock::raw_spin_lock_unlocked(name), + interval, + burst, + ..pin_init::zeroed() + }), + } + } + + /// The default interval used for rate limiting. + pub const DEFAULT_INTERVAL: i32 = bindings::DEFAULT_RATELIMIT_INTERVAL as i32; + + /// The default burst size. + pub const DEFAULT_BURST: i32 = bindings::DEFAULT_RATELIMIT_BURST as i32; + + /// Check if an action should be rate-limited. + /// + /// Returns [`true`] if the action is allowed, and [`false`] if it should be suppressed. + #[inline] + pub fn ratelimit(&self, func: &CStr) -> bool { + // SAFETY: `self.inner.get()` is a valid pointer to a `struct ratelimit_state`. + // The lifetime of `func` ensures the pointer remains valid for the duration of the call. + // The C function `___ratelimit` handles its own internal locking, so it is safe to call + // concurrently. + unsafe { bindings::___ratelimit(self.inner.get(), func.as_char_ptr()) != 0 } + } +} + +#[pinned_drop] +impl PinnedDrop for Ratelimit { + #[inline] + fn drop(self: Pin<&mut Self>) { + // SAFETY: By the type invariants, this struct contains an initialized `struct + // ratelimit_state`. + unsafe { bindings::ratelimit_state_exit(self.inner.get()) }; + } +} + +/// Helper macro to implement ratelimited printing. +#[macro_export] +#[doc(hidden)] +macro_rules! print_ratelimited { + ($print_macro:ident, $($arg:tt)*) => {{ + $crate::ratelimit::ratelimit_state_init!( + _rs, + $crate::ratelimit::Ratelimit::DEFAULT_INTERVAL, + $crate::ratelimit::Ratelimit::DEFAULT_BURST, + ); + let loc = $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())); + if $crate::ratelimit::Ratelimit::ratelimit(&_rs, loc) { + $crate::$print_macro!($($arg)*); + } + }}; +} + +/// Prints an emergency-level message (level 0) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_emerg_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_emerg, $($arg)*) + ) +); + +/// Prints an alert-level message (level 1) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_alert_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_alert, $($arg)*) + ) +); + +/// Prints a critical-level message (level 2) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_crit_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_crit, $($arg)*) + ) +); + +/// Prints an error-level message (level 3) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_err_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_err, $($arg)*) + ) +); + +/// Prints a warning-level message (level 4) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_warn_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_warn, $($arg)*) + ) +); + +/// Prints a notice-level message (level 5) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_notice_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_notice, $($arg)*) + ) +); + +/// Prints an info-level message (level 6) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_info_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_info, $($arg)*) + ) +); + +/// Prints a debug-level message (level 7) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_debug_ratelimited ( + ($($arg:tt)*) => ( + if cfg!(debug_assertions) { + $crate::print_ratelimited!(pr_debug, $($arg)*) + } + ) +); diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 037fa63ebcac..6a71bb3a5c45 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -468,7 +468,6 @@ fn spinlock_irq_condvar() -> Result { /// /// For use in statics containing raw spinlocks. #[doc(alias("__SPIN_LOCK_UNLOCKED", "DEFINE_SPINLOCK"))] -#[expect(dead_code)] pub(crate) const fn raw_spin_lock_unlocked(name: &'static CStr) -> bindings::raw_spinlock_t { // Silence unused variable warnings. #[cfg(not(CONFIG_DEBUG_LOCK_ALLOC))] From a12f22b2c9398541d75d86d18be0a9ec92909fe0 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Oct 2026 14:30:04 +0000 Subject: [PATCH 79/91] rust_binder: use pr_*_ratelimited! for printing To avoid DoS from printing too much, make printing in Binder rate limited. A big portion of these print statements have been updated to use binder_debug!, but some still remain to be converted. For now, just update them to use pr_*_ratelimited! until we get around to converting them to use binder_debug! too. While we're at it, fix the missing newlines at the end of some of those println statements. Four transaction failure pr_warn! calls in thread.rs and transaction.rs are not converted because they're being deleted by another patch series. Acked-by: Carlos Llamas Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20261007-pr-ratelimited-v6-4-f14accd4ea3b@google.com Signed-off-by: Miguel Ojeda --- drivers/android/binder/allocation.rs | 4 +- drivers/android/binder/context.rs | 6 +-- drivers/android/binder/debug.rs | 4 +- drivers/android/binder/freeze.rs | 2 +- drivers/android/binder/node.rs | 4 +- drivers/android/binder/page_range.rs | 12 +++--- drivers/android/binder/process.rs | 22 +++++----- drivers/android/binder/range_alloc/tree.rs | 26 ++++++------ drivers/android/binder/thread.rs | 48 ++++++++++++---------- drivers/android/binder/transaction.rs | 6 +-- 10 files changed, 71 insertions(+), 63 deletions(-) diff --git a/drivers/android/binder/allocation.rs b/drivers/android/binder/allocation.rs index 165cb797eb1e..8151ba3ea7f4 100644 --- a/drivers/android/binder/allocation.rs +++ b/drivers/android/binder/allocation.rs @@ -261,7 +261,7 @@ fn drop(&mut self) { let view = AllocationView::new(self, offsets.start); for i in offsets.step_by(size_of::()) { if view.cleanup_object(i).is_err() { - pr_warn!("Error cleaning up object at offset {}\n", i) + pr_warn_ratelimited!("Error cleaning up object at offset {}\n", i) } } } @@ -286,7 +286,7 @@ fn drop(&mut self) { if info.clear_on_free { if let Err(e) = self.fill_zero() { - pr_warn!("Failed to clear data on free: {:?}", e); + pr_warn_ratelimited!("Failed to clear data on free: {:?}\n", e); } } } diff --git a/drivers/android/binder/context.rs b/drivers/android/binder/context.rs index ddddb66b3557..431d6007a9b1 100644 --- a/drivers/android/binder/context.rs +++ b/drivers/android/binder/context.rs @@ -80,7 +80,7 @@ pub(crate) fn deregister(self: &Arc) { pub(crate) fn register_process(self: &Arc, proc: Arc) -> Result { if !Arc::ptr_eq(self, &proc.ctx) { - pr_err!("Context::register_process called on the wrong context."); + pr_err_ratelimited!("Context::register_process called on the wrong context.\n"); return Err(EINVAL); } self.manager.lock().all_procs.push(proc, GFP_KERNEL)?; @@ -89,7 +89,7 @@ pub(crate) fn register_process(self: &Arc, proc: Arc) -> Result { pub(crate) fn deregister_process(self: &Arc, proc: &Arc) { if !Arc::ptr_eq(self, &proc.ctx) { - pr_err!("Context::deregister_process called on the wrong context."); + pr_err_ratelimited!("Context::deregister_process called on the wrong context.\n"); return; } let mut manager = self.manager.lock(); @@ -110,7 +110,7 @@ pub(crate) fn deregister_process(self: &Arc, proc: &Arc) { pub(crate) fn set_manager_node(&self, node_ref: NodeRef) -> Result { let mut manager = self.manager.lock(); if manager.node.is_some() { - pr_warn!("BINDER_SET_CONTEXT_MGR already set"); + pr_warn_ratelimited!("BINDER_SET_CONTEXT_MGR already set\n"); return Err(EBUSY); } security::binder_set_context_mgr(&node_ref.node.owner.cred)?; diff --git a/drivers/android/binder/debug.rs b/drivers/android/binder/debug.rs index 824b10c004c3..6d8dcddf4619 100644 --- a/drivers/android/binder/debug.rs +++ b/drivers/android/binder/debug.rs @@ -53,7 +53,7 @@ macro_rules! binder_debug { // Rule to explicitly specify a PID (used in kworkers). (pid=$pid:expr, $mask:ident, $($arg:tt)*) => { if $crate::debug::debug_mask_enabled($crate::debug::DebugMask::$mask) { - kernel::pr_info!( + kernel::pr_info_ratelimited!( "{}: {}\n", $pid, kernel::prelude::fmt!($($arg)*) @@ -65,7 +65,7 @@ macro_rules! binder_debug { ($mask:ident, $($arg:tt)*) => { if $crate::debug::debug_mask_enabled($crate::debug::DebugMask::$mask) { let thread = kernel::current!(); - kernel::pr_info!( + kernel::pr_info_ratelimited!( "{}:{} {}\n", thread.tgid(), thread.pid(), diff --git a/drivers/android/binder/freeze.rs b/drivers/android/binder/freeze.rs index 66912b4cb527..7fbde3345ced 100644 --- a/drivers/android/binder/freeze.rs +++ b/drivers/android/binder/freeze.rs @@ -412,7 +412,7 @@ fn find_freeze_recipients(&self) -> Result, Arc)>, Al recipients .push_within_capacity(node_proc_pair) .map_err(|_| { - pr_err!( + pr_err_ratelimited!( "push_within_capacity failed even though we checked the capacity\n" ); AllocError diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index 0a82af14cda3..69b36951dbbe 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -405,7 +405,7 @@ pub(crate) fn update_refcount_locked( !is_dead && !state.has_count } else { if state.count < count { - pr_err!("Failure: refcount underflow!"); + pr_err_ratelimited!("Failure: refcount underflow!\n"); return None; } state.count -= count; @@ -693,7 +693,7 @@ pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec> .freeze_list .retain(|proc| !core::ptr::eq::(&**proc, p)); if len == inner.freeze_list.len() { - pr_warn!( + pr_warn_ratelimited!( "Could not remove freeze listener for {}\n", p.pid_in_current_ns() ); diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs index 52ffbf3504e7..c1f4f635c2c0 100644 --- a/drivers/android/binder/page_range.rs +++ b/drivers/android/binder/page_range.rs @@ -212,7 +212,7 @@ unsafe fn set_page(me: *mut PageInfo, page: Page) { // SAFETY: The pointer is valid for writing, so also valid for reading. if unsafe { (*ptr).is_some() } { - pr_err!("set_page called when there is already a page"); + pr_err_ratelimited!("set_page called when there is already a page\n"); // SAFETY: We will initialize the page again below. unsafe { ptr::drop_in_place(ptr) }; } @@ -300,11 +300,11 @@ pub(crate) fn register_with_vma(&self, vma: &virt::VmaNew) -> Result { let num_pages = num_bytes >> PAGE_SHIFT; if !ptr::eq::(&*self.mm, &**vma.mm()) { - pr_debug!("Failed to register with vma: invalid vma->vm_mm"); + pr_debug_ratelimited!("Failed to register with vma: invalid vma->vm_mm\n"); return Err(EINVAL); } if num_pages == 0 { - pr_debug!("Failed to register with vma: size zero"); + pr_debug_ratelimited!("Failed to register with vma: size zero\n"); return Err(EINVAL); } @@ -325,7 +325,7 @@ pub(crate) fn register_with_vma(&self, vma: &virt::VmaNew) -> Result { let mut inner = self.lock.lock(); if inner.size > 0 { - pr_debug!("Failed to register with vma: already registered"); + pr_debug_ratelimited!("Failed to register with vma: already registered\n"); drop(inner); return Err(EBUSY); } @@ -380,7 +380,7 @@ pub(crate) fn use_range(&self, start: usize, end: usize) -> Result<()> { match unsafe { self.use_page_slow(i) } { Ok(()) => {} Err(err) => { - pr_warn!("Error in use_page_slow: {:?}", err); + pr_warn_ratelimited!("Error in use_page_slow: {:?}\n", err); return Err(err); } } @@ -529,7 +529,7 @@ unsafe fn iterate(&self, mut offset: usize, mut size: usize, mut cb: T) -> Re // duration of this call to `iterate`, so nobody will change the page. let page = unsafe { PageInfo::get_page(page_info) }; if page.is_none() { - pr_warn!("Page is null!"); + pr_warn_ratelimited!("Page is null!\n"); } let page = page.ok_or(EFAULT)?; cb(page, offset, available)?; diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index 5372bfbd93b3..65f719e46c3e 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -324,7 +324,7 @@ pub(crate) fn death_delivered(&mut self, death: DArc) { if let Some(death) = ListArc::try_from_arc_or_drop(death) { self.delivered_deaths.push_back(death); } else { - pr_warn!("Notification added to `delivered_deaths` twice."); + pr_warn_ratelimited!("Notification added to `delivered_deaths` twice.\n"); } } @@ -701,7 +701,7 @@ fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result> { let id = { let current = kernel::current!(); if self.task != current.group_leader() { - pr_err!("get_current_thread was called from the wrong process."); + pr_err_ratelimited!("get_current_thread was called from the wrong process.\n"); return Err(EINVAL); } current.pid() @@ -725,7 +725,7 @@ fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result> { Ok(ta) } rbtree::Entry::Occupied(_entry) => { - pr_err!("Cannot create two threads with the same id."); + pr_err_ratelimited!("Cannot create two threads with the same id.\n"); Err(EINVAL) } } @@ -861,7 +861,9 @@ pub(crate) fn insert_or_update_handle( match refs.by_handle.entry(res.as_u32()) { rbtree::Entry::Vacant(entry) => break (res, entry), rbtree::Entry::Occupied(_) => { - pr_err!("Detected mismatch between handle_is_present and by_handle"); + pr_err_ratelimited!( + "Detected mismatch between handle_is_present and by_handle\n" + ); res.acquire(); kernel::warn_on!(true); return Err(EINVAL); @@ -1101,7 +1103,7 @@ pub(crate) fn buffer_alloc( ) { Ok(()) => {} Err(err) => { - pr_warn!("use_range failure {:?}", err); + pr_warn_ratelimited!("use_range failure {:?}\n", err); return Err(err.into()); } } @@ -1132,7 +1134,7 @@ pub(crate) fn buffer_raw_free(&self, ptr: usize) { let freed_range = match mapping.alloc.reservation_abort(offset) { Ok(freed_range) => freed_range, Err(_) => { - pr_warn!( + pr_warn_ratelimited!( "Pointer {:x} failed to free, base = {:x}\n", ptr, mapping.address @@ -1154,7 +1156,7 @@ pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option, guard: &mut Guard<'_, ProcessInner, SpinLockBack // It is an error to hit this branch, and it should not be reachable. We try to do // something reasonable when the failure path happens. Most likely, the thread in // question will sleep forever. - pr_err!("Same thread registered with `ready_threads` twice."); + pr_err_ratelimited!("Same thread registered with `ready_threads` twice.\n"); } Self { thread } } @@ -1861,7 +1863,7 @@ impl Drop for WithNodes<'_> { fn drop(&mut self) { core::mem::swap(&mut self.nodes, &mut self.inner.nodes); if self.nodes.iter().next().is_some() { - pr_err!("nodes array was modified while using lock_with_nodes\n"); + pr_err_ratelimited!("nodes array was modified while using lock_with_nodes\n"); } } } diff --git a/drivers/android/binder/range_alloc/tree.rs b/drivers/android/binder/range_alloc/tree.rs index 48796fcdb362..3e3b452909ed 100644 --- a/drivers/android/binder/range_alloc/tree.rs +++ b/drivers/android/binder/range_alloc/tree.rs @@ -166,7 +166,7 @@ pub(crate) fn reserve_new( let (found_size, found_off, tree_node, free_tree_node) = match self.find_best_match(size) { None => { - pr_warn!("ENOSPC from range_alloc.reserve_new - size: {}", size); + pr_warn_ratelimited!("ENOSPC from range_alloc.reserve_new - size: {}\n", size); return Err(ENOSPC); } Some(desc) => { @@ -208,8 +208,8 @@ pub(crate) fn reserve_new( pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result { let mut cursor = self.tree.cursor_lower_bound_mut(&offset).ok_or_else(|| { - pr_warn!( - "EINVAL from range_alloc.reservation_abort - offset: {}", + pr_warn_ratelimited!( + "EINVAL from range_alloc.reservation_abort - offset: {}\n", offset ); EINVAL @@ -218,8 +218,8 @@ pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result let (_, desc) = cursor.current_mut(); if desc.offset != offset { - pr_warn!( - "EINVAL from range_alloc.reservation_abort - offset: {}", + pr_warn_ratelimited!( + "EINVAL from range_alloc.reservation_abort - offset: {}\n", offset ); return Err(EINVAL); @@ -230,15 +230,15 @@ pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result (None, Ok((reservation, free_node_res))) } None => { - pr_warn!( - "EINVAL from range_alloc.reservation_abort - offset: {}", + pr_warn_ratelimited!( + "EINVAL from range_alloc.reservation_abort - offset: {}\n", offset ); (None, Err(EINVAL)) } allocated => { - pr_warn!( - "EPERM from range_alloc.reservation_abort - offset: {}", + pr_warn_ratelimited!( + "EPERM from range_alloc.reservation_abort - offset: {}\n", offset ); (allocated, Err(EPERM)) @@ -327,8 +327,8 @@ pub(crate) fn reservation_commit(&mut self, offset: usize, data: &mut Option) /// Returns the size of the existing entry and the data associated with it. pub(crate) fn reserve_existing(&mut self, offset: usize) -> Result<(usize, usize, Option)> { let desc = self.tree.get_mut(&offset).ok_or_else(|| { - pr_warn!( - "ENOENT from range_alloc.reserve_existing - offset: {}", + pr_warn_ratelimited!( + "ENOENT from range_alloc.reserve_existing - offset: {}\n", offset ); ENOENT @@ -344,8 +344,8 @@ pub(crate) fn reserve_existing(&mut self, offset: usize) -> Result<(usize, usize ) } other => { - pr_warn!( - "ENOENT from range_alloc.reserve_existing - offset: {}", + pr_warn_ratelimited!( + "ENOENT from range_alloc.reserve_existing - offset: {}\n", offset ); (other, Err(ENOENT)) diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index 18a14aa8a835..897c86188c96 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -160,8 +160,8 @@ fn validate_parent_fixup( let sg_entry = match self.sg_entries.get(sg_idx) { Some(sg_entry) => sg_entry, None => { - pr_err!( - "self.ancestors[{}] is {}, but self.sg_entries.len() is {}", + pr_err_ratelimited!( + "self.ancestors[{}] is {}, but self.sg_entries.len() is {}\n", ancestors_i, sg_idx, self.sg_entries.len() @@ -170,8 +170,8 @@ fn validate_parent_fixup( } }; if sg_entry.fixup_min_offset > parent_offset { - pr_warn!( - "validate_parent_fixup: fixup_min_offset={}, parent_offset={}", + pr_warn_ratelimited!( + "validate_parent_fixup: fixup_min_offset={}, parent_offset={}\n", sg_entry.fixup_min_offset, parent_offset ); @@ -179,8 +179,8 @@ fn validate_parent_fixup( } let new_min_offset = parent_offset.checked_add(length).ok_or(EINVAL)?; if new_min_offset > sg_entry.length { - pr_warn!( - "validate_parent_fixup: new_min_offset={}, sg_entry.length={}", + pr_warn_ratelimited!( + "validate_parent_fixup: new_min_offset={}, sg_entry.length={}\n", new_min_offset, sg_entry.length ); @@ -339,7 +339,7 @@ fn push_reply_work(&mut self, code: u32) -> PushWorkRes { work.set_error_code(code); self.push_work(work) } else { - pr_warn!("Thread reply work is already in use."); + pr_warn_ratelimited!("Thread reply work is already in use.\n"); PushWorkRes::Ok } } @@ -350,7 +350,7 @@ fn push_return_work(&mut self, reply: u32) { // Not notifying: Reply to current thread. let _ = self.push_work(work); } else { - pr_warn!("Thread return work is already in use."); + pr_warn_ratelimited!("Thread return work is already in use.\n"); } } @@ -803,8 +803,8 @@ fn translate_object( let parent_entry = match sg_state.sg_entries.get_mut(info.parent_sg_index) { Some(parent_entry) => parent_entry, None => { - pr_err!( - "validate_parent_fixup returned index out of bounds for sg.entries" + pr_err_ratelimited!( + "validate_parent_fixup returned index out of bounds for sg.entries\n" ); return Err(EINVAL.into()); } @@ -850,8 +850,8 @@ fn translate_object( let parent_entry = match sg_state.sg_entries.get_mut(info.parent_sg_index) { Some(parent_entry) => parent_entry, None => { - pr_err!( - "validate_parent_fixup returned index out of bounds for sg.entries" + pr_err_ratelimited!( + "validate_parent_fixup returned index out of bounds for sg.entries\n" ); return Err(EINVAL.into()); } @@ -884,7 +884,9 @@ fn translate_object( .read_all(&mut fda_bytes, GFP_KERNEL)?; if fds_len != fda_bytes.len() { - pr_err!("UserSlice::read_all returned wrong length in BINDER_TYPE_FDA"); + pr_err_ratelimited!( + "UserSlice::read_all returned wrong length in BINDER_TYPE_FDA\n" + ); return Err(EINVAL.into()); } @@ -999,7 +1001,11 @@ pub(crate) fn copy_transaction_data( let ctx = match security::SecurityCtx::from_secid(secid) { Ok(ctx) => ctx, Err(err) => { - pr_warn!("Failed to get security ctx for id {}: {:?}", secid, err); + pr_warn_ratelimited!( + "Failed to get security ctx for id {}: {:?}\n", + secid, + err + ); return Err(err.into()); } }; @@ -1233,7 +1239,7 @@ fn top_of_transaction_stack(&self) -> Result>> { let inner = self.inner.lock(); if let Some(cur) = &inner.current_transaction { if core::ptr::eq(self, cur.from.as_ref()) { - pr_warn!("got new transaction with bad transaction stack"); + pr_warn_ratelimited!("got new transaction with bad transaction stack\n"); return Err(EINVAL); } Ok(Some(cur.clone())) @@ -1562,7 +1568,7 @@ fn read(self: &Arc, req: &mut BinderWriteRead, wait: bool) -> Result { let mut has_noop_placeholder = false; if req.read_consumed == 0 { if let Err(err) = writer.write_code(BR_NOOP) { - pr_warn!("Failure when writing BR_NOOP at beginning of buffer."); + pr_warn_ratelimited!("Failure when writing BR_NOOP at beginning of buffer.\n"); return Err(err); } has_noop_placeholder = true; @@ -1585,7 +1591,7 @@ fn read(self: &Arc, req: &mut BinderWriteRead, wait: bool) -> Result { Err(err) => { // Propagate the error if we haven't written anything else. if err != EINTR && err != EAGAIN { - pr_warn!("Failure in work getter: {:?}", err); + pr_warn_ratelimited!("Failure in work getter: {:?}\n", err); } if initial_len == writer.len() { return Err(err); @@ -1620,8 +1626,8 @@ pub(crate) fn write_read(self: &Arc, data: UserSlice, wait: bool) -> Resul ret = self.write(&mut req); crate::trace::trace_write_done(ret); if let Err(err) = ret { - pr_warn!( - "Write failure {:?} in pid:{}", + pr_warn_ratelimited!( + "Write failure {:?} in pid:{}\n", err, self.process.pid_in_current_ns() ); @@ -1637,8 +1643,8 @@ pub(crate) fn write_read(self: &Arc, data: UserSlice, wait: bool) -> Resul ret = self.read(&mut req, wait); crate::trace::trace_read_done(ret); if ret.is_err() && ret != Err(EINTR) { - pr_warn!( - "Read failure {:?} in pid:{}", + pr_warn_ratelimited!( + "Read failure {:?} in pid:{}\n", ret, self.process.pid_in_current_ns() ); diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs index 245f1556b5db..66045547c23a 100644 --- a/drivers/android/binder/transaction.rs +++ b/drivers/android/binder/transaction.rs @@ -83,7 +83,7 @@ pub(crate) fn is_oneway(&self) -> bool { pub(crate) fn report_netlink(&self, reply: u32, ctx: &crate::Context) { if let Err(err) = self.report_netlink_inner(reply, ctx) { - pr_warn!( + pr_warn_ratelimited!( "{}:{} netlink report failed: {err:?}\n", self.from_pid, self.from_tid @@ -183,7 +183,7 @@ pub(crate) fn new( }; if info.is_oneway() { if from_parent.is_some() { - pr_warn!("Oneway transaction should not be in a transaction stack."); + pr_warn_ratelimited!("Oneway transaction should not be in a transaction stack.\n"); return Err(EINVAL.into()); } alloc.set_info_oneway_node(node_ref.node.clone()); @@ -384,7 +384,7 @@ pub(crate) fn submit(self: DLArc, info: &mut TransactionInfo) -> BinderRes return Ok(()); } } else { - pr_err!("Failed to submit oneway transaction to node."); + pr_err_ratelimited!("Failed to submit oneway transaction to node.\n"); } } From cad3f4daef8a8793a953c632c2fcb0a6560da46a Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Fri, 2 Oct 2026 14:04:25 +0900 Subject: [PATCH 80/91] rust: time: make Delta::ZERO generic over the time unit Delta::ZERO is defined only for Delta, so code that needs a zero Delta has to write Delta::from_jiffies(0). Define ZERO for every Delta instead. An integer literal cannot be used for a generic U::Repr, so add ZERO to TimeUnit. Suggested-by: Markus Probst Suggested-by: Gary Guo Signed-off-by: FUJITA Tomonori Acked-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://patch.msgid.link/20261002050428.2746277-2-tomo@flapping.org Signed-off-by: Miguel Ojeda --- rust/kernel/time.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 5ae377e70dd8..9f231bf25ead 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -306,6 +306,9 @@ impl Sealed for super::Jiffy {} pub trait TimeUnit: private::Sealed { /// The underlying representation of the time unit. type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; + + /// The zero value of [`Self::Repr`]. + const ZERO: Self::Repr; } /// A time unit of nanoseconds. @@ -317,6 +320,7 @@ pub enum Nsec {} impl TimeUnit for Nsec { type Repr = i64; + const ZERO: Self::Repr = 0; } /// A time unit of jiffies. @@ -328,6 +332,7 @@ pub enum Jiffy {} impl TimeUnit for Jiffy { type Repr = isize; + const ZERO: Self::Repr = 0; } /// A span of time. @@ -341,6 +346,11 @@ pub struct Delta { value: U::Repr, } +impl Delta { + /// A span of time equal to zero. + pub const ZERO: Self = Self { value: U::ZERO }; +} + impl Delta { /// Create a new [`Delta`] from a number of jiffies. #[inline] @@ -428,9 +438,6 @@ fn div(self, rhs: Self) -> Self::Output { } impl Delta { - /// A span of time equal to zero. - pub const ZERO: Self = Self { value: 0 }; - /// Create a new [`Delta`] from a number of nanoseconds. #[inline] pub const fn from_nanos(nanos: i64) -> Self { From daff0977b2e36c777a00b4ca9fabc019834e60aa Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Fri, 2 Oct 2026 14:04:26 +0900 Subject: [PATCH 81/91] rust: time: add Delta::as_jiffies_unsigned() Delta is signed, but some C functions, such as queue_delayed_work_on(), take a span in jiffies as unsigned long. So each caller has to clamp a negative value to zero and cast it before it passes the value to C. Add a helper that does both. Suggested-by: Gary Guo Signed-off-by: FUJITA Tomonori Acked-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://patch.msgid.link/20261002050428.2746277-3-tomo@flapping.org Signed-off-by: Miguel Ojeda --- rust/kernel/time.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 9f231bf25ead..b6a2aab12a50 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -363,6 +363,27 @@ pub const fn from_jiffies(jiffies: isize) -> Self { pub const fn as_jiffies(self) -> isize { self.value } + + /// Return the number of jiffies in the [`Delta`] as an unsigned value. + /// + /// If the number of jiffies is negative, zero is returned. + /// + /// # Examples + /// + /// ``` + /// use kernel::time::Delta; + /// + /// // A negative span returns zero. + /// assert_eq!(Delta::from_jiffies(-1).as_jiffies_unsigned(), 0); + /// + /// // A positive span returns its value. + /// assert_eq!(Delta::from_jiffies(1).as_jiffies_unsigned(), 1); + /// ``` + #[inline] + pub fn as_jiffies_unsigned(self) -> usize { + // CAST: `isize::max()` makes the value non-negative, so the cast keeps it. + isize::max(self.value, 0) as usize + } } impl ops::Add for Delta { From 5c94956f2bfb87025c1eaded6c51d5841d38d333 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Fri, 2 Oct 2026 19:45:50 +0900 Subject: [PATCH 82/91] rust: bug: update the SAFETY comment for warn_slowpath_fmt() on UML The SAFETY comment says that it is always safe to call `warn_slowpath_fmt()` with a valid null-terminated string. This is not true. The taint argument must also be a valid taint number. Commit 951d09ca450c ("rust: bug: pass TAINT_WARN to warn_slowpath_fmt() on UML") fixed a bug where it was not. List each condition that the call meets: the file name, the taint number, and the null `fmt`. Suggested-by: Miguel Ojeda Signed-off-by: FUJITA Tomonori Link: https://patch.msgid.link/20261002104550.2877930-1-tomo@flapping.org Cc: stable@vger.kernel.org Fixes: dff64b072708 ("rust: Add warn_on macro") Signed-off-by: Miguel Ojeda --- rust/kernel/bug.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index 6abed2c055b1..3b40b0bd37b5 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -88,8 +88,10 @@ macro_rules! warn_flags { _ = $flags; } - // SAFETY: It is always safe to call `warn_slowpath_fmt()` - // with a valid null-terminated string. + // SAFETY: + // - The file name is a null-terminated string with `'static` lifetime. + // - `TAINT_WARN` is a valid taint number. + // - `fmt` can be NULL, and NULL is passed. unsafe { $crate::bindings::warn_slowpath_fmt( $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), From 1d09c9ee13be740ec0bbf976f14b6f942396cabd Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 13 Oct 2026 14:58:06 +0900 Subject: [PATCH 83/91] rust: num: add cv! macro to create values from constant expressions Currently, using NonZero/Bounded constants is quite verbose. It's unfortunate because it disincentivizes using it in interface boundaries. Introduce a macro to make it nicer to use. The macro `cv!` (for constant value) takes a const integer expression and widens it to i128 (at build time only) before passing it as a const generic value to a new trait `FromConst`. The value is then converted and appears in the associated constant `FromConst::VALUE`. The trait is implemented by NonZero, Bounded, and Alignment and lets values of each be constructed from constants without a verbose turbofish syntax. For example, `const { NonZero::new(1).unwrap() }` can be written as `cv!(1)`. Add the `cv!` macro to the prelude so it doesn't need to be imported explicitly to use it. Suggested-by: Gary Guo Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Eliot Courtney Acked-by: Alexandre Courbot Link: https://patch.msgid.link/20261013-cv-v5-1-113a538dfaef@nvidia.com [ Added a few intra-doc links. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/num.rs | 149 +++++++++++++++++++++++++++++++++++++ rust/kernel/num/bounded.rs | 16 ++++ rust/kernel/prelude.rs | 1 + rust/kernel/ptr.rs | 12 ++- 4 files changed, 177 insertions(+), 1 deletion(-) diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index de589792a77a..f81f0c1250d4 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -86,3 +86,152 @@ impl Integer for $type { i128: Signed, isize: Signed ); + +/// Creates a constant value ("cv") from an integer constant expression, with validity checked at +/// compile time. +/// +/// This works for any type that implements [`FromConst`], with the target type inferred from +/// the context, or named explicitly with `cv!(value => Type)`. It can also be used in const +/// contexts. +/// +/// # Pitfalls +/// +/// Only the target type is inferred from the context. The expression passed to [`cv!`] is typed by +/// itself. So, unsuffixed integer literal expressions default to [`i32`]. For example +/// `cv!(2 << 31)` does not evaluate to the same value as `cv!(2i64 << 31)`, since `2 << 31` is +/// treated as an [`i32`] before being widened to [`i128`] internally. Literals and arithmetic +/// expressions that overflow [`i32`] are a compile error, it's just left shifts that require care. +/// +/// # Examples +/// +/// ``` +/// use core::num::NonZero; +/// +/// use kernel::{ +/// num::Bounded, +/// ptr::Alignment, // +/// }; +/// +/// let v: NonZero = cv!(8); +/// assert_eq!(v.get(), 8); +/// +/// // Any integer constant expression works, not only literals. +/// let m: NonZero = cv!(usize::MAX); +/// assert_eq!(m.get(), usize::MAX); +/// +/// let b: Bounded = cv!(15); +/// assert_eq!(b.get(), 15); +/// +/// let a: Alignment = cv!(4096); +/// assert_eq!(a.as_usize(), 4096); +/// +/// // Checked narrowing of integer constants, including in `const` items. +/// const SMALL: u8 = cv!(200u32); +/// assert_eq!(SMALL, 200); +/// +/// const N: NonZero = cv!(5); +/// assert_eq!(N.get(), 5); +/// +/// // The target type can be given explicitly. +/// let e = cv!(200u32 => u8); +/// assert_eq!(e, 200); +/// +/// // With an explicit primitive target, the expression can use generic parameters. +/// const fn as_u64() -> u64 { +/// cv!(KEY => u64) +/// } +/// assert_eq!(as_u64::<0x40>(), 0x40); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! cv { + (@cast $v:expr => $t:ty) => { + const { + #[allow(unused_comparisons, unused_assignments, clippy::as_underscore)] + { + let v = $v; + let r = v as $t; + // Pin `back` to `v`'s type so `as _` casts back to the source type. + let mut back = v; + back = r as _; + + ::core::assert!( + back == v && (v < 0) == (r < 0), + "value does not fit into the target type" + ); + + r + } + } + }; + + // Using `FromConst::VALUE` means const generic expressions can't be used, since it requires + // the `generic_const_exprs` feature. Provide a special path for each primitive type that allows + // const generic expressions. + ($v:expr => u8) => { $crate::cv!(@cast $v => u8) }; + ($v:expr => u16) => { $crate::cv!(@cast $v => u16) }; + ($v:expr => u32) => { $crate::cv!(@cast $v => u32) }; + ($v:expr => u64) => { $crate::cv!(@cast $v => u64) }; + ($v:expr => u128) => { $crate::cv!(@cast $v => u128) }; + ($v:expr => usize) => { $crate::cv!(@cast $v => usize) }; + ($v:expr => i8) => { $crate::cv!(@cast $v => i8) }; + ($v:expr => i16) => { $crate::cv!(@cast $v => i16) }; + ($v:expr => i32) => { $crate::cv!(@cast $v => i32) }; + ($v:expr => i64) => { $crate::cv!(@cast $v => i64) }; + ($v:expr => i128) => { $crate::cv!(@cast $v => i128) }; + ($v:expr => isize) => { $crate::cv!(@cast $v => isize) }; + ($v:expr => $t:ty) => { + <$t as $crate::num::FromConst<{ $crate::cv!(@cast $v => i128) }>>::VALUE + }; + ($v:expr) => { + <_ as $crate::num::FromConst<{ $crate::cv!(@cast $v => i128) }>>::VALUE + }; +} +#[doc(inline)] +pub use cv; + +/// Types that can be created from an integer constant expression validated at compile time. +/// +/// Implement this trait to make a type usable with [`cv!`]. Use the [`cv!`] macro, not this trait +/// directly, for creating values. +#[diagnostic::on_unimplemented(message = "`{Self}` cannot be converted from a constant")] +pub trait FromConst: Sized { + /// The value that corresponds to the constant `V`. + /// + /// Fails the build if `V` is not a valid value for [`Self`]. + const VALUE: Self; +} + +/// Implements [`FromConst`] for primitive integer types and their [`NonZero`](core::num::NonZero) +/// versions. +macro_rules! impl_from_const { + ($($type:ty)*) => { + $( + impl FromConst for $type { + const VALUE: Self = { + // CAST: The macro is only used for types up to 64 bits wide, so `MIN` and `MAX` + // widen to `i128` losslessly. + assert!( + V >= <$type>::MIN as i128 && V <= <$type>::MAX as i128, + "constant cannot be represented by the target type" + ); + + // CAST: The assert above confirmed that `V` fits in `$type`. + V as $type + }; + } + + impl FromConst for core::num::NonZero<$type> { + const VALUE: Self = match core::num::NonZero::new(<$type as FromConst>::VALUE) { + Some(value) => value, + None => panic!("constant cannot be zero"), + }; + } + )* + }; +} + +impl_from_const!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index 2a2b0a4bca5e..0b32f4343922 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -14,6 +14,7 @@ use kernel::{ num::{ + FromConst, Integer, Unsigned, // }, @@ -272,6 +273,21 @@ pub const fn new() -> Self { unsafe { Self::__new(VALUE) } } } + + impl FromConst for Bounded<$type, N> { + const VALUE: Self = { + let value = <$type as FromConst>::VALUE; + // Statically assert that `value` fits within the set number of bits. + assert!( + fits_within!(value, $type, N), + "constant cannot be represented within the given number of bits" + ); + + // SAFETY: The assert above confirmed that `value` can be represented within `N` + // bits. + unsafe { Self::__new(value) } + }; + } )* }; } diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index bcaa232205be..edaf1e4c174b 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -106,6 +106,7 @@ Result, // }, init::InPlaceInit, + num::cv, pr_alert, pr_alert_ratelimited, pr_crit, diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 82acb531b17b..2c12e4fd4572 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -11,7 +11,10 @@ }; use core::num::NonZero; -use crate::const_assert; +use crate::{ + const_assert, + num::FromConst, // +}; /// Type representing an alignment, which is always a power of two. /// @@ -166,6 +169,13 @@ pub const fn mask(self) -> usize { } } +impl FromConst for Alignment { + const VALUE: Self = match Alignment::new_checked(>::VALUE) { + Some(alignment) => alignment, + None => panic!("constant is not a power of two"), + }; +} + /// Trait for items that can be aligned against an [`Alignment`]. pub trait Alignable: Sized { /// Aligns `self` down to `alignment`. From 8d26ced7b496f4cb9c5eafbad7ee21adf0fb6673 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 30 Sep 2026 11:42:50 +0900 Subject: [PATCH 84/91] rust: bitmap: use function-level cfg on kunit test Since commit c652dc44192d ("rust: kunit: allow `cfg` on `test`s"), we no longer need this workaround. Reviewed-by: Alice Ryhl Reviewed-by: Yury Norov Reviewed-by: Burak Emir Reviewed-by: Gary Guo Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260930-chid-v9-1-0d6cca376cff@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/bitmap.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index b27e0ec80d64..a43bfe0ec3dc 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -572,24 +572,21 @@ fn bitmap_set_clear_find() -> Result<(), AllocError> { } #[test] + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> { - // TODO: Kunit #[test]s do not support `cfg` yet, - // so we add it here in the body. - #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] - { - let mut b = BitmapVec::new(128, GFP_KERNEL)?; - b.set_bit(2048); - b.set_bit_atomic(2048); - b.clear_bit(2048); - b.clear_bit_atomic(2048); - assert_eq!(None, b.next_bit(2048)); - assert_eq!(None, b.next_zero_bit(2048)); - assert_eq!(None, b.last_bit()); - } + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + + b.set_bit(2048); + b.set_bit_atomic(2048); + b.clear_bit(2048); + b.clear_bit_atomic(2048); + assert_eq!(None, b.next_bit(2048)); + assert_eq!(None, b.next_zero_bit(2048)); + assert_eq!(None, b.last_bit()); Ok(()) } - // TODO: uncomment once kunit supports [should_panic] and `cfg`. + // TODO: uncomment once kunit supports `#[should_panic]`. // #[cfg(CONFIG_RUST_BITMAP_HARDENED)] // #[test] // #[should_panic] From 931790a91378f6513ba82730eb5607479c3184a0 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 30 Sep 2026 11:42:51 +0900 Subject: [PATCH 85/91] rust: bitmap: restrict bitmap length to at most i32::MAX It is currently possible to construct a non-`BitmapVec` backed `Bitmap` using `Bitmap::from_raw` that is larger than `i32::MAX`, and it is not part of the unsafe requirements. Restricting all bitmaps (even non-`BitmapVec` backed ones) to a maximum size of `i32::MAX` simplifies a few things and matches `BitmapVec::MAX_LEN`. For example, `copy_and_extend` truncates `len` to u32, which is wrong for > u32::MAX size. `__bitmap_set` and `__bitmap_clear` need i32 for `size` and u32 for `start` - so they can't be run on a `Bitmap` with a > i32::MAX size. Rather than adding runtime checks to account for the case of a non `BitmapVec` backed `Bitmap`, just include that in the requirements for `Bitmap`. Add that requirement to the unsafe requirements on `Bitmap::from_raw` and `Bitmap::from_raw_mut`, and to the invariants on `Bitmap`. This also fixes u32 casts truncating in `copy_and_extend`, which could otherwise lead to OOB writes. Fixes: 11eca92a2cae ("rust: add bitmap API.") Link: https://lore.kernel.org/DKG0U8RLO7LZ.2I1AIH0S38PAP@nvidia.com Reviewed-by: Yury Norov Reviewed-by: Burak Emir Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260930-chid-v9-2-0d6cca376cff@nvidia.com Cc: stable@vger.kernel.org [ Added a few intra-doc links. Added backticks to C types. Capitalize tagged comments. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/bitmap.rs | 70 ++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index a43bfe0ec3dc..9098ee062d80 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -17,24 +17,59 @@ /// # Invariants /// /// Must reference a `[c_ulong]` long enough to fit `data.len()` bits. +/// Must not be longer than [`i32::MAX`] bits, so offsets and lengths used with +/// [`Bitmap`] functions fit in the `int` and `unsigned int` arguments of the C bitmap API. +/// This also matches [`BitmapVec::MAX_LEN`]. #[cfg_attr(CONFIG_64BIT, repr(align(8)))] #[cfg_attr(not(CONFIG_64BIT), repr(align(4)))] pub struct Bitmap { data: [()], } +macro_rules! bitmap_assert { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + } +} + +macro_rules! bitmap_assert_return { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + if !($cond) { + pr_err!($($arg)*); + return + } + } +} + impl Bitmap { /// Borrows a C bitmap. /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `nbits` exceeds [`i32::MAX`]. + /// /// # Safety /// /// * `ptr` holds a non-null address of an initialized array of `unsigned long` /// that is large enough to hold `nbits` bits. + /// * `nbits` must not exceed [`i32::MAX`]. /// * the array must not be freed for the lifetime of this [`Bitmap`] /// * concurrent access only happens through atomic operations pub unsafe fn from_raw<'a>(ptr: *const usize, nbits: usize) -> &'a Bitmap { + bitmap_assert!( + nbits <= i32::MAX as usize, + "`nbits` must be <= {}, was {}", + i32::MAX, + nbits + ); let data: *const [()] = core::ptr::slice_from_raw_parts(ptr.cast(), nbits); // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // INVARIANT: The caller guarantees that `nbits` does not exceed `i32::MAX`. // SAFETY: // The caller guarantees that `data` (derived from `ptr` and `nbits`) // points to a valid, initialized, and appropriately sized memory region @@ -51,15 +86,27 @@ pub unsafe fn from_raw<'a>(ptr: *const usize, nbits: usize) -> &'a Bitmap { /// Borrows a C bitmap exclusively. /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `nbits` exceeds [`i32::MAX`]. + /// /// # Safety /// /// * `ptr` holds a non-null address of an initialized array of `unsigned long` /// that is large enough to hold `nbits` bits. + /// * `nbits` must not exceed [`i32::MAX`]. /// * the array must not be freed for the lifetime of this [`Bitmap`] /// * no concurrent access may happen. pub unsafe fn from_raw_mut<'a>(ptr: *mut usize, nbits: usize) -> &'a mut Bitmap { + bitmap_assert!( + nbits <= i32::MAX as usize, + "`nbits` must be <= {}, was {}", + i32::MAX, + nbits + ); let data: *mut [()] = core::ptr::slice_from_raw_parts_mut(ptr.cast(), nbits); // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // INVARIANT: The caller guarantees that `nbits` does not exceed `i32::MAX`. // SAFETY: // The caller guarantees that `data` (derived from `ptr` and `nbits`) // points to a valid, initialized, and appropriately sized memory region @@ -96,26 +143,6 @@ union BitmapRepr { ptr: NonNull, } -macro_rules! bitmap_assert { - ($cond:expr, $($arg:tt)+) => { - #[cfg(CONFIG_RUST_BITMAP_HARDENED)] - assert!($cond, $($arg)*); - } -} - -macro_rules! bitmap_assert_return { - ($cond:expr, $($arg:tt)+) => { - #[cfg(CONFIG_RUST_BITMAP_HARDENED)] - assert!($cond, $($arg)*); - - #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] - if !($cond) { - pr_err!($($arg)*); - return - } - } -} - /// Represents an owned bitmap. /// /// Wraps underlying C bitmap API. See [`Bitmap`] for available @@ -415,7 +442,8 @@ pub fn clear_bit_atomic(&self, index: usize) { #[inline] pub fn copy_and_extend(&mut self, src: &Bitmap) { let len = core::cmp::min(src.len(), self.len()); - // SAFETY: access to `self` and `src` is within bounds. + // SAFETY: Access to `self` and `src` is within bounds. Both lengths fit in `u32` + // because a `Bitmap` is at most `i32::MAX` bits, so the casts are lossless. unsafe { bindings::bitmap_copy_and_extend( self.as_mut_ptr(), From bf461edb197bd7c0462c1d13bb1c3b51d6892fd3 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 30 Sep 2026 11:42:52 +0900 Subject: [PATCH 86/91] rust: sizes: add sub-1K size constants Add some more size constants, mirroring include/linux/sizes.h. This is useful for making `Alignment` implement `SizeConstants` in a following patch, because these are more common alignment values. Reviewed-by: Alexandre Courbot Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260930-chid-v9-3-0d6cca376cff@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/sizes.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs index 521b2b38bfe7..7d03361eae3d 100644 --- a/rust/kernel/sizes.rs +++ b/rust/kernel/sizes.rs @@ -35,6 +35,26 @@ macro_rules! define_sizes { ($($type:ty),* $(,)?) => { define_sizes!(@internal [$($type),*] + /// `0x0000_0001`. + SZ_1, + /// `0x0000_0002`. + SZ_2, + /// `0x0000_0004`. + SZ_4, + /// `0x0000_0008`. + SZ_8, + /// `0x0000_0010`. + SZ_16, + /// `0x0000_0020`. + SZ_32, + /// `0x0000_0040`. + SZ_64, + /// `0x0000_0080`. + SZ_128, + /// `0x0000_0100`. + SZ_256, + /// `0x0000_0200`. + SZ_512, /// `0x0000_0400`. SZ_1K, /// `0x0000_0800`. From d21cfabdd8aee0a837d31d945844e5c177154389 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 30 Sep 2026 11:42:53 +0900 Subject: [PATCH 87/91] rust: sizes: implement SizeConstants for Alignment Currently, constructing an alignment is quite verbose [1]: Alignment::new::<8>() It's unfortunate because it disincentivizes using it at interface boundaries. Implement `SizeConstants` for `Alignment` so we can write e.g. `Alignment::SZ_8` instead. Link: https://lore.kernel.org/an4xDp29VX8Am0uR@yury [1] Reviewed-by: Alexandre Courbot Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260930-chid-v9-4-0d6cca376cff@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/sizes.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs index 7d03361eae3d..188e00c2b8b4 100644 --- a/rust/kernel/sizes.rs +++ b/rust/kernel/sizes.rs @@ -13,6 +13,11 @@ //! these constants as [`u64`] (or [`u32`]) rather than [`usize`], because //! device address spaces are sized independently of the CPU pointer width. //! +//! The trait is also implemented for [`Alignment`], providing each size as a +//! compile-time validated alignment. +//! +//! [`Alignment`]: crate::ptr::Alignment +//! //! # Examples //! //! ``` @@ -105,6 +110,7 @@ macro_rules! define_sizes { (@internal [$($type:ty),*] $($names_and_metas:tt)*) => { define_sizes!(@consts_and_trait $($names_and_metas)*); define_sizes!(@impls [$($type),*] $($names_and_metas)*); + define_sizes!(@impl_alignment $($names_and_metas)*); }; (@consts_and_trait $($(#[$meta:meta])* $name:ident,)*) => { @@ -119,13 +125,22 @@ macro_rules! define_sizes { /// choose the width that matches their hardware. All `SZ_*` values fit /// in a [`u32`], so all implementations are lossless. /// + /// Also implemented for [`Alignment`], providing each size as a + /// compile-time validated alignment. + /// + /// [`Alignment`]: crate::ptr::Alignment + /// /// # Examples /// /// ``` - /// use kernel::sizes::SizeConstants; + /// use kernel::{ + /// ptr::Alignment, + /// sizes::SizeConstants, // + /// }; /// /// let gpu_heap = 14 * u64::SZ_1M; /// let mmio_window = u32::SZ_16M; + /// let page_align = Alignment::SZ_4K; /// ``` pub trait SizeConstants { $( @@ -137,6 +152,16 @@ pub trait SizeConstants { (@impls [] $($(#[$meta:meta])* $name:ident,)*) => {}; + (@impl_alignment $($(#[$meta:meta])* $name:ident,)*) => { + impl SizeConstants for crate::ptr::Alignment { + $( + $(#[$meta])* + // A non-power-of-two constant will fail the build here if used. + const $name: Self = crate::ptr::Alignment::new::<{ self::$name }>(); + )* + } + }; + (@impls [$first:ty $(, $rest:ty)*] $($(#[$meta:meta])* $name:ident,)*) => { impl SizeConstants for $first { $( From b666bd22be8f211aceeeb8fbc827ebdb7abe98c0 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 30 Sep 2026 11:42:55 +0900 Subject: [PATCH 88/91] rust: bitmap: add contiguous area operations Add bindings for area operations on bitmaps. Each one is made safe by adding some extra checks compared to the underlying C code (for example, checking bounds) and with additional checks to catch likely erroneous usage if `CONFIG_RUST_BITMAP_HARDENED` is on. Add tests demonstrating the edge cases. Reviewed-by: Burak Emir Signed-off-by: Eliot Courtney Reviewed-by: Yury Norov Link: https://patch.msgid.link/20260930-chid-v9-6-0d6cca376cff@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/bitmap.rs | 241 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 239 insertions(+), 2 deletions(-) diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index 9098ee062d80..c676b0e44eb5 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -10,7 +10,11 @@ use crate::bindings; #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] use crate::pr_err; -use core::ptr::NonNull; +use crate::ptr::Alignment; +use core::{ + num::NonZero, + ptr::NonNull, // +}; /// Represents a C bitmap. Wraps underlying C bitmap API. /// @@ -525,13 +529,159 @@ pub fn next_zero_bit(&self, start: usize) -> Option { Some(index) } } + + /// Finds a contiguous area of `nbits` zero bits at or after `start`, where the area plus + /// `align_offset` is aligned to `align`. + /// + /// Returns the bit index of the start of the area, or [`None`] if no such area fitting in + /// the bitmap exists. + /// + /// The returned index plus `align_offset` is a multiple of `align`. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `start` is out of bounds. + #[inline] + pub fn next_zero_area_off( + &self, + start: usize, + nbits: NonZero, + align: Alignment, + align_offset: usize, + ) -> Option { + bitmap_assert!( + start < self.len(), + "`start` must be < {}, was {}", + self.len(), + start + ); + + let nr = u32::try_from(nbits.get()).ok()?; + let align_mask = align.as_usize() - 1; + + // The C alignment and end arithmetic must not overflow, or it can read out of bounds. + // Overflow is only possible on 32-bit. + #[cfg(not(CONFIG_64BIT))] + align_mask + .checked_add(self.len())? + .checked_add(nbits.get())?; + + // SAFETY: `bitmap_find_next_zero_area_off` is safe to use with an out of bounds `start` + // value and, given the overflow check above, never reads beyond `self.len()` bits. + let index = unsafe { + bindings::bitmap_find_next_zero_area_off( + self.as_ptr().cast_mut(), + self.len(), + start, + nr, + align_mask, + align_offset, + ) + }; + + (index < self.len()).then_some(index) + } + + /// Finds a contiguous area of `nbits` zero bits at or after `start`, aligned to `align`. + /// + /// Returns the bit index of the start of the area, or [`None`] if no such area fitting in + /// the bitmap exists. + /// + /// The returned index is a multiple of `align`. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `start` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// use kernel::{ + /// alloc::{AllocError, flags::GFP_KERNEL}, + /// bitmap::BitmapVec, + /// ptr::Alignment, + /// sizes::SizeConstants, // + /// }; + /// + /// let mut b = BitmapVec::new(64, GFP_KERNEL)?; + /// + /// assert_eq!(Some(0), b.next_zero_area(0, cv!(8), Alignment::SZ_1)); + /// b.set(0, cv!(5)); + /// assert_eq!(Some(5), b.next_zero_area(0, cv!(8), Alignment::SZ_1)); + /// assert_eq!(Some(8), b.next_zero_area(0, cv!(8), Alignment::SZ_8)); + /// assert_eq!(None, b.next_zero_area(0, cv!(65), Alignment::SZ_1)); + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn next_zero_area( + &self, + start: usize, + nbits: NonZero, + align: Alignment, + ) -> Option { + self.next_zero_area_off(start, nbits, align, 0) + } + + /// Sets a contiguous area of `nbits` bits starting at `start`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and the area `start..start + nbits` is out of + /// bounds, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and the area `start..start + nbits` is out + /// of bounds. + #[inline] + pub fn set(&mut self, start: usize, nbits: NonZero) { + bitmap_assert_return!( + start + .checked_add(nbits.get()) + .is_some_and(|end| end <= self.len()), + "Area `start..start + nbits` ({}..{}) must be within bounds {}", + start, + start.saturating_add(nbits.get()), + self.len() + ); + // SAFETY: The area `start..start + nbits` is within bounds and a `Bitmap` is at most + // `i32::MAX` bits, so the casts are lossless. + unsafe { bindings::__bitmap_set(self.as_mut_ptr(), start as u32, nbits.get() as i32) }; + } + + /// Clears a contiguous area of `nbits` bits starting at `start`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and the area `start..start + nbits` is out of + /// bounds, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and the area `start..start + nbits` is out + /// of bounds. + #[inline] + pub fn clear(&mut self, start: usize, nbits: NonZero) { + bitmap_assert_return!( + start + .checked_add(nbits.get()) + .is_some_and(|end| end <= self.len()), + "Area `start..start + nbits` ({}..{}) must be within bounds {}", + start, + start.saturating_add(nbits.get()), + self.len() + ); + // SAFETY: The area `start..start + nbits` is within bounds and a `Bitmap` is at most + // `i32::MAX` bits, so the casts are lossless. + unsafe { bindings::__bitmap_clear(self.as_mut_ptr(), start as u32, nbits.get() as i32) }; + } } #[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)] #[macros::kunit_tests(rust_kernel_bitmap)] mod tests { use super::*; - use kernel::alloc::flags::GFP_KERNEL; + use kernel::{ + alloc::flags::GFP_KERNEL, + num::cv, + sizes::SizeConstants, // + }; #[test] fn bitmap_borrow() { @@ -642,4 +792,91 @@ fn bitmap_copy_and_extend() -> Result<(), AllocError> { assert_eq!(Some(17), long_bitmap.last_bit()); Ok(()) } + + #[test] + fn bitmap_area_set_clear_find() -> Result<(), AllocError> { + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + + assert_eq!(Some(0), b.next_zero_area(0, cv!(5), Alignment::SZ_1)); + b.set(0, cv!(5)); // Now contains {[0, 5)}. + + assert_eq!(Some(0), b.next_bit(0)); + assert_eq!(Some(4), b.next_bit(4)); + assert_eq!(Some(5), b.next_zero_bit(0)); + assert_eq!(Some(5), b.next_zero_area(0, cv!(5), Alignment::SZ_1)); + assert_eq!(Some(8), b.next_zero_area(0, cv!(5), Alignment::SZ_8)); + + b.set(8, cv!(8)); // Now contains {[0, 5), [8, 16)}. + assert_eq!(Some(16), b.next_zero_area(0, cv!(4), Alignment::SZ_16)); + assert_eq!(Some(16), b.next_zero_area(0, cv!(4), Alignment::SZ_1)); + + b.clear(0, cv!(5)); // Now contains {[8, 16)}. + assert_eq!(Some(0), b.next_zero_area(0, cv!(5), Alignment::SZ_1)); + assert_eq!(Some(8), b.next_bit(0)); + assert_eq!(Some(15), b.last_bit()); + + b.set(60, cv!(10)); // Now contains {[8, 16), [60, 70)}. + assert_eq!(Some(60), b.next_bit(16)); + assert_eq!(Some(69), b.last_bit()); + assert_eq!(Some(16), b.next_zero_area(9, cv!(40), Alignment::SZ_1)); + assert_eq!(Some(70), b.next_zero_area(0, cv!(45), Alignment::SZ_1)); + + b.clear(62, cv!(6)); // Now contains {[8, 16), [60, 62), [68, 70)}. + assert_eq!(Some(62), b.next_zero_area(60, cv!(6), Alignment::SZ_1)); + assert_eq!(Some(61), b.next_bit(61)); + assert_eq!(Some(69), b.last_bit()); + Ok(()) + } + + #[test] + fn bitmap_area_exhaustion() -> Result<(), AllocError> { + let mut b = BitmapVec::new(64, GFP_KERNEL)?; + + assert_eq!(None, b.next_zero_area(0, cv!(65), Alignment::SZ_1)); + assert_eq!(None, b.next_zero_area(0, cv!(usize::MAX), Alignment::SZ_1)); + assert_eq!(None, b.next_zero_area(1, cv!(usize::MAX), Alignment::SZ_1)); + + b.set_bit(0); // Now contains {[0, 1)}. + assert_eq!(None, b.next_zero_area(0, cv!(usize::MAX), Alignment::SZ_1)); + + b.set(0, cv!(61)); // Now contains {[0, 61)}. + assert_eq!(None, b.next_zero_area(0, cv!(4), Alignment::SZ_1)); + assert_eq!(Some(61), b.next_zero_area(0, cv!(3), Alignment::SZ_1)); + assert_eq!(None, b.next_zero_area(0, cv!(1), Alignment::SZ_64)); + Ok(()) + } + + #[test] + fn bitmap_area_off() -> Result<(), AllocError> { + let mut b = BitmapVec::new(64, GFP_KERNEL)?; + + b.set(0, cv!(5)); // Now contains {[0, 5)}. + + // The area plus align_offset starts at a multiple of the alignment. + assert_eq!(Some(7), b.next_zero_area_off(0, cv!(8), Alignment::SZ_8, 1)); + assert_eq!(Some(5), b.next_zero_area_off(0, cv!(8), Alignment::SZ_8, 3)); + + // A zero offset behaves like next_zero_area(). + assert_eq!( + b.next_zero_area(0, cv!(8), Alignment::SZ_8), + b.next_zero_area_off(0, cv!(8), Alignment::SZ_8, 0) + ); + Ok(()) + } + + #[test] + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + fn owned_bitmap_area_out_of_bounds() -> Result<(), AllocError> { + let mut b = BitmapVec::new(64, GFP_KERNEL)?; + + // Should be ignored since out of bounds. + b.set(64, cv!(4)); + b.set(62, cv!(8)); + b.set(usize::MAX, cv!(1)); + b.clear(usize::MAX, cv!(1)); + b.clear(2048, cv!(8)); + assert_eq!(None, b.next_bit(0)); + assert_eq!(None, b.next_zero_area(64, cv!(1), Alignment::SZ_1)); + Ok(()) + } } From 3132a951a7e4ec2e18bc314877137deee96ab83e Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 30 Sep 2026 11:42:56 +0900 Subject: [PATCH 89/91] rust: id_pool: add contiguous ID reservation Add `IdPool::reserve_ids` which allocates a contiguous range with the given offset, count, and alignment. Reviewed-by: Burak Emir Reviewed-by: Alexandre Courbot Signed-off-by: Eliot Courtney Reviewed-by: Yury Norov Link: https://patch.msgid.link/20260930-chid-v9-7-0d6cca376cff@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/id_pool.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rust/kernel/id_pool.rs b/rust/kernel/id_pool.rs index 384753fe0e44..06a4c71c4c6c 100644 --- a/rust/kernel/id_pool.rs +++ b/rust/kernel/id_pool.rs @@ -4,8 +4,14 @@ //! Rust API for an ID pool backed by a [`BitmapVec`]. +use core::{ + num::NonZero, + ops::Range, // +}; + use crate::alloc::{AllocError, Flags}; use crate::bitmap::BitmapVec; +use crate::ptr::Alignment; /// Represents a dynamic ID pool backed by a [`BitmapVec`]. /// @@ -240,6 +246,32 @@ pub fn find_unused_id(&mut self, offset: usize) -> Option> { pub fn release_id(&mut self, id: usize) { self.map.clear_bit(id); } + + /// Reserves a contiguous area of `count` IDs at or after `offset`. + /// + /// The start of the returned area is a multiple of `align`. + /// + /// Returns the reserved range upon success, or [`None`] if no such area could be found. + #[inline] + #[must_use] + pub fn reserve_ids( + &mut self, + offset: usize, + count: NonZero, + align: Alignment, + ) -> Option> { + let start = self.map.next_zero_area(offset, count, align)?; + self.map.set(start, count); + Some(start..start + count.get()) + } + + /// Releases a contiguous area of IDs. + #[inline] + pub fn release_ids(&mut self, range: &Range) { + if let Some(nbits) = NonZero::new(range.len()) { + self.map.clear(range.start, nbits); + } + } } /// Represents an unused id in an [`IdPool`]. From d61be5c960e4b89b2e6fa8cedc1d9aa64401ae13 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Tue, 13 Oct 2026 20:48:07 +0200 Subject: [PATCH 90/91] rust: zerocopy: fix link to upstream `README.md` Upstream moved the `zerocopy` crate into a `zerocopy/` subdirectory of their repository in v0.8.51 [1]. However, in our v0.8.52 update, the link was not updated. Fix it by adding the subdirectory. Fixes: 2808a3963988 ("rust: zerocopy: update to v0.8.52") Link: https://github.com/google/zerocopy/pull/3434 [1] Assisted-by: LLM Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20261013184809.35143-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/zerocopy-derive/README.md | 2 +- rust/zerocopy/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/zerocopy-derive/README.md b/rust/zerocopy-derive/README.md index d62c79804342..49de31b8e082 100644 --- a/rust/zerocopy-derive/README.md +++ b/rust/zerocopy-derive/README.md @@ -8,7 +8,7 @@ non-ASCII identifiers. For copyright details, please see: - https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.54/zerocopy/README.md?plain=1 https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT diff --git a/rust/zerocopy/README.md b/rust/zerocopy/README.md index 3d11a6502cf0..8443f6166008 100644 --- a/rust/zerocopy/README.md +++ b/rust/zerocopy/README.md @@ -7,7 +7,7 @@ modified to tweak the SPDX license identifiers. For copyright details, please see: - https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.54/zerocopy/README.md?plain=1 https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT From 7c306f00835a237a70024a8f968dbc3bc8db8fcf Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Tue, 13 Oct 2026 20:48:08 +0200 Subject: [PATCH 91/91] rust: zerocopy: update to v0.8.62 Update our vendored copy of `zerocopy` (and `zerocopy-derive`) to v0.8.62. This update advances 8 releases, with 3 of them carrying most of the changes. It contains new features as well as fixes [1]. However, almost nothing has changed that requires updates on our side: we only need to move our non-ASCII change to the recently added `project.rs` file, and to add an SPDX license identifier to the also new `invariant.rs` file (I reported it to upstream [2]). The following script may be used to check for the remaining differences: for path in $(cd rust/zerocopy-derive/ && find . -type f ! -name README.md); do curl --silent --show-error --location \ https://github.com/google/zerocopy/raw/v0.8.62/zerocopy/zerocopy-derive/src/$path | git diff --no-index - rust/zerocopy-derive/$path && echo $path: OK done for path in $(cd rust/zerocopy/ && find . -type f ! -name README.md); do curl --silent --show-error --location \ https://github.com/google/zerocopy/raw/v0.8.62/zerocopy/$path | git diff --no-index - rust/zerocopy/$path && echo $path: OK done Comparing the output above with the one for the latest update (v0.8.54) shows no new differences on our side apart from the two mentioned above. Cc: Joshua Liebow-Feeser Cc: Jack Wrenn Link: https://github.com/google/zerocopy/releases [1] Link: https://github.com/google/zerocopy/issues/3858 [2] Assisted-by: LLM Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20261013184809.35143-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 2 +- rust/zerocopy-derive/README.md | 12 +- rust/zerocopy-derive/derive/from_bytes.rs | 12 + rust/zerocopy-derive/derive/into_bytes.rs | 147 +- rust/zerocopy-derive/derive/known_layout.rs | 4 +- rust/zerocopy-derive/derive/mod.rs | 33 +- rust/zerocopy-derive/derive/project.rs | 569 +++ rust/zerocopy-derive/derive/try_from_bytes.rs | 881 ++-- rust/zerocopy-derive/invariant.rs | 116 + rust/zerocopy-derive/lib.rs | 118 +- rust/zerocopy-derive/repr.rs | 46 +- rust/zerocopy-derive/util.rs | 414 +- rust/zerocopy/README.md | 12 +- .../benches/as_bytes_dynamic_size.x86-64 | 3 +- .../benches/as_bytes_dynamic_size.x86-64.mca | 20 +- .../zerocopy/benches/extend_vec_zeroed.x86-64 | 4 +- .../benches/extend_vec_zeroed.x86-64.mca | 8 +- .../zerocopy/benches/insert_vec_zeroed.x86-64 | 4 +- .../benches/insert_vec_zeroed.x86-64.mca | 8 +- rust/zerocopy/benches/new_vec_zeroed.x86-64 | 3 +- .../benches/new_vec_zeroed.x86-64.mca | 46 +- .../ref_from_bytes_dynamic_padding.x86-64 | 35 +- .../ref_from_bytes_dynamic_padding.x86-64.mca | 70 +- .../ref_from_bytes_dynamic_size.x86-64 | 21 +- .../ref_from_bytes_dynamic_size.x86-64.mca | 58 +- .../benches/ref_from_bytes_static_size.x86-64 | 7 +- .../ref_from_bytes_static_size.x86-64.mca | 26 +- .../ref_from_prefix_dynamic_padding.x86-64 | 26 +- ...ref_from_prefix_dynamic_padding.x86-64.mca | 64 +- .../ref_from_prefix_dynamic_size.x86-64 | 21 +- .../ref_from_prefix_dynamic_size.x86-64.mca | 52 +- .../ref_from_suffix_dynamic_padding.x86-64 | 29 +- ...ref_from_suffix_dynamic_padding.x86-64.mca | 68 +- ...m_suffix_with_elems_dynamic_padding.x86-64 | 36 +- ...ffix_with_elems_dynamic_padding.x86-64.mca | 86 +- ...from_suffix_with_elems_dynamic_size.x86-64 | 31 +- ..._suffix_with_elems_dynamic_size.x86-64.mca | 72 +- .../benches/ref_type_bytes_dynamic_padding.rs | 19 + .../ref_type_bytes_dynamic_padding.x86-64 | 4 + .../ref_type_bytes_dynamic_padding.x86-64.mca | 45 + .../benches/ref_type_bytes_dynamic_size.rs | 17 + .../ref_type_bytes_dynamic_size.x86-64 | 4 + .../ref_type_bytes_dynamic_size.x86-64.mca | 45 + .../ref_type_bytes_mut_dynamic_padding.rs | 19 + .../ref_type_bytes_mut_dynamic_padding.x86-64 | 4 + ..._type_bytes_mut_dynamic_padding.x86-64.mca | 45 + .../ref_type_bytes_mut_dynamic_size.rs | 19 + .../ref_type_bytes_mut_dynamic_size.x86-64 | 4 + ...ref_type_bytes_mut_dynamic_size.x86-64.mca | 45 + .../benches/ref_type_bytes_mut_static_size.rs | 19 + .../ref_type_bytes_mut_static_size.x86-64 | 4 + .../ref_type_bytes_mut_static_size.x86-64.mca | 45 + .../benches/ref_type_bytes_static_size.rs | 17 + .../benches/ref_type_bytes_static_size.x86-64 | 4 + .../ref_type_bytes_static_size.x86-64.mca | 45 + .../benches/ref_type_deref_dynamic_padding.rs | 19 + .../ref_type_deref_dynamic_padding.x86-64 | 29 + .../ref_type_deref_dynamic_padding.x86-64.mca | 91 + .../benches/ref_type_deref_dynamic_size.rs | 19 + .../ref_type_deref_dynamic_size.x86-64 | 27 + .../ref_type_deref_dynamic_size.x86-64.mca | 87 + .../ref_type_deref_mut_dynamic_size.rs | 19 + .../ref_type_deref_mut_dynamic_size.x86-64 | 27 + ...ref_type_deref_mut_dynamic_size.x86-64.mca | 87 + .../benches/ref_type_deref_mut_static_size.rs | 19 + .../ref_type_deref_mut_static_size.x86-64 | 3 + .../ref_type_deref_mut_static_size.x86-64.mca | 43 + .../benches/ref_type_deref_static_size.rs | 19 + .../benches/ref_type_deref_static_size.x86-64 | 3 + .../ref_type_deref_static_size.x86-64.mca | 43 + .../ref_type_from_bytes_dynamic_padding.rs | 19 + ...ref_type_from_bytes_dynamic_padding.x86-64 | 25 + ...type_from_bytes_dynamic_padding.x86-64.mca | 85 + .../ref_type_from_bytes_dynamic_size.rs | 17 + .../ref_type_from_bytes_dynamic_size.x86-64 | 21 + ...ef_type_from_bytes_dynamic_size.x86-64.mca | 77 + .../ref_type_from_bytes_static_size.rs | 17 + .../ref_type_from_bytes_static_size.x86-64 | 15 + ...ref_type_from_bytes_static_size.x86-64.mca | 65 + ...e_from_bytes_with_elems_dynamic_padding.rs | 20 + ...om_bytes_with_elems_dynamic_padding.x86-64 | 26 + ...ytes_with_elems_dynamic_padding.x86-64.mca | 85 + ...type_from_bytes_with_elems_dynamic_size.rs | 20 + ..._from_bytes_with_elems_dynamic_size.x86-64 | 28 + ...m_bytes_with_elems_dynamic_size.x86-64.mca | 89 + .../ref_type_from_prefix_dynamic_padding.rs | 22 + ...ef_type_from_prefix_dynamic_padding.x86-64 | 26 + ...ype_from_prefix_dynamic_padding.x86-64.mca | 87 + .../ref_type_from_prefix_dynamic_size.rs | 22 + .../ref_type_from_prefix_dynamic_size.x86-64 | 19 + ...f_type_from_prefix_dynamic_size.x86-64.mca | 73 + .../ref_type_from_prefix_static_size.rs | 20 + .../ref_type_from_prefix_static_size.x86-64 | 15 + ...ef_type_from_prefix_static_size.x86-64.mca | 65 + ..._from_prefix_with_elems_dynamic_padding.rs | 23 + ...m_prefix_with_elems_dynamic_padding.x86-64 | 26 + ...efix_with_elems_dynamic_padding.x86-64.mca | 85 + ...ype_from_prefix_with_elems_dynamic_size.rs | 23 + ...from_prefix_with_elems_dynamic_size.x86-64 | 27 + ..._prefix_with_elems_dynamic_size.x86-64.mca | 87 + .../ref_type_from_suffix_dynamic_padding.rs | 22 + ...ef_type_from_suffix_dynamic_padding.x86-64 | 31 + ...ype_from_suffix_dynamic_padding.x86-64.mca | 97 + .../ref_type_from_suffix_dynamic_size.rs | 22 + .../ref_type_from_suffix_dynamic_size.x86-64 | 23 + ...f_type_from_suffix_dynamic_size.x86-64.mca | 81 + .../ref_type_from_suffix_static_size.rs | 20 + .../ref_type_from_suffix_static_size.x86-64 | 18 + ...ef_type_from_suffix_static_size.x86-64.mca | 71 + ..._from_suffix_with_elems_dynamic_padding.rs | 23 + ...m_suffix_with_elems_dynamic_padding.x86-64 | 27 + ...ffix_with_elems_dynamic_padding.x86-64.mca | 87 + ...ype_from_suffix_with_elems_dynamic_size.rs | 23 + ...from_suffix_with_elems_dynamic_size.x86-64 | 27 + ..._suffix_with_elems_dynamic_size.x86-64.mca | 87 + .../benches/ref_type_into_mut_dynamic_size.rs | 19 + .../ref_type_into_mut_dynamic_size.x86-64 | 27 + .../ref_type_into_mut_dynamic_size.x86-64.mca | 87 + .../benches/ref_type_into_mut_static_size.rs | 19 + .../ref_type_into_mut_static_size.x86-64 | 3 + .../ref_type_into_mut_static_size.x86-64.mca | 43 + .../ref_type_into_ref_dynamic_padding.rs | 19 + .../ref_type_into_ref_dynamic_padding.x86-64 | 29 + ...f_type_into_ref_dynamic_padding.x86-64.mca | 91 + .../benches/ref_type_into_ref_dynamic_size.rs | 19 + .../ref_type_into_ref_dynamic_size.x86-64 | 27 + .../ref_type_into_ref_dynamic_size.x86-64.mca | 87 + .../benches/ref_type_into_ref_static_size.rs | 19 + .../ref_type_into_ref_static_size.x86-64 | 3 + .../ref_type_into_ref_static_size.x86-64.mca | 43 + .../benches/ref_type_read_static_size.rs | 17 + .../benches/ref_type_read_static_size.x86-64 | 7 + .../ref_type_read_static_size.x86-64.mca | 51 + .../benches/ref_type_write_static_size.rs | 20 + .../benches/ref_type_write_static_size.x86-64 | 6 + .../ref_type_write_static_size.x86-64.mca | 49 + ...t_via_runtime_check_dynamic_padding.x86-64 | 5 +- ...a_runtime_check_dynamic_padding.x86-64.mca | 30 +- .../benches/try_read_from_bytes.x86-64 | 28 +- .../benches/try_read_from_bytes.x86-64.mca | 66 +- .../benches/try_read_from_prefix.x86-64 | 19 +- .../benches/try_read_from_prefix.x86-64.mca | 52 +- .../benches/try_read_from_suffix.x86-64 | 21 +- .../benches/try_read_from_suffix.x86-64.mca | 56 +- .../try_ref_from_bytes_dynamic_padding.x86-64 | 34 +- ..._ref_from_bytes_dynamic_padding.x86-64.mca | 78 +- .../try_ref_from_bytes_dynamic_size.x86-64 | 33 +- ...try_ref_from_bytes_dynamic_size.x86-64.mca | 74 +- ...try_ref_from_prefix_dynamic_padding.x86-64 | 23 +- ...ref_from_prefix_dynamic_padding.x86-64.mca | 56 +- .../try_ref_from_prefix_dynamic_size.x86-64 | 16 +- ...ry_ref_from_prefix_dynamic_size.x86-64.mca | 56 +- ...try_ref_from_suffix_dynamic_padding.x86-64 | 29 +- ...ref_from_suffix_dynamic_padding.x86-64.mca | 74 +- ...m_suffix_with_elems_dynamic_padding.x86-64 | 31 +- ...ffix_with_elems_dynamic_padding.x86-64.mca | 80 +- ...from_suffix_with_elems_dynamic_size.x86-64 | 28 +- ..._suffix_with_elems_dynamic_size.x86-64.mca | 72 +- .../benches/write_to_dynamic_size.x86-64 | 3 +- .../benches/write_to_dynamic_size.x86-64.mca | 34 +- .../write_to_prefix_dynamic_size.x86-64 | 3 +- .../write_to_prefix_dynamic_size.x86-64.mca | 36 +- .../write_to_suffix_dynamic_size.x86-64 | 3 +- .../write_to_suffix_dynamic_size.x86-64.mca | 38 +- .../zerocopy/benches/zero_dynamic_size.x86-64 | 3 +- .../benches/zero_dynamic_size.x86-64.mca | 20 +- rust/zerocopy/src/byte_slice.rs | 6 +- rust/zerocopy/src/byteorder.rs | 41 +- rust/zerocopy/src/error.rs | 14 +- rust/zerocopy/src/impls.rs | 117 +- rust/zerocopy/src/layout.rs | 3707 +++++++++++++++-- rust/zerocopy/src/lib.rs | 1083 ++++- rust/zerocopy/src/macros.rs | 53 +- rust/zerocopy/src/pointer/inner.rs | 35 +- rust/zerocopy/src/pointer/invariant.rs | 61 +- rust/zerocopy/src/pointer/mod.rs | 50 +- rust/zerocopy/src/pointer/ptr.rs | 177 +- rust/zerocopy/src/pointer/transmute.rs | 100 +- rust/zerocopy/src/ref.rs | 760 +++- rust/zerocopy/src/split_at.rs | 796 +++- rust/zerocopy/src/util/macro_util.rs | 112 +- rust/zerocopy/src/util/macros.rs | 127 +- rust/zerocopy/src/util/mod.rs | 40 +- rust/zerocopy/src/wrappers.rs | 262 +- 184 files changed, 12266 insertions(+), 3052 deletions(-) create mode 100644 rust/zerocopy-derive/derive/project.rs create mode 100644 rust/zerocopy-derive/invariant.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_deref_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_deref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_deref_mut_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_deref_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_deref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_deref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_into_mut_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_into_mut_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_into_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_into_ref_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_read_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_read_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_read_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_type_write_static_size.rs create mode 100644 rust/zerocopy/benches/ref_type_write_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_type_write_static_size.x86-64.mca diff --git a/rust/Makefile b/rust/Makefile index ddca144cb084..0ac9ab28cc2b 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -94,7 +94,7 @@ zerocopy-flags := \ $(call cfgs-to-flags,$(zerocopy-cfgs)) zerocopy-envs := \ - CARGO_PKG_VERSION=0.8.54 + CARGO_PKG_VERSION=0.8.62 proc_macro2-cfgs := \ feature="proc-macro" \ diff --git a/rust/zerocopy-derive/README.md b/rust/zerocopy-derive/README.md index 49de31b8e082..fab3779b9ed2 100644 --- a/rust/zerocopy-derive/README.md +++ b/rust/zerocopy-derive/README.md @@ -1,14 +1,14 @@ # `zerocopy-derive` -These source files come from the Rust `zerocopy-derive` crate, version v0.8.54 -(released 2026-07-08), hosted in the +These source files come from the Rust `zerocopy-derive` crate, version v0.8.62 +(released 2026-10-08), hosted in the repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only modified to tweak the SPDX license identifiers and to remove the generation of non-ASCII identifiers. For copyright details, please see: - https://github.com/google/zerocopy/blob/v0.8.54/zerocopy/README.md?plain=1 - https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD - https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE - https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT + https://github.com/google/zerocopy/blob/v0.8.62/zerocopy/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.62/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.62/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.62/LICENSE-MIT diff --git a/rust/zerocopy-derive/derive/from_bytes.rs b/rust/zerocopy-derive/derive/from_bytes.rs index 66d820f6ad4c..7fe505a86457 100644 --- a/rust/zerocopy-derive/derive/from_bytes.rs +++ b/rust/zerocopy-derive/derive/from_bytes.rs @@ -89,6 +89,12 @@ pub(crate) fn find_zero_variant(enm: &DataEnum) -> Result { Err(has_unknown_discriminants) } pub(crate) fn derive_from_zeros(ctx: &Ctx, top_level: Trait) -> Result { + if let Some(span) = ctx.invariant_span { + return ctx.error_or_skip(Error::new( + span, + "cannot derive `FromZeros` for a type with invariants", + )); + } let try_from_bytes = derive_try_from_bytes(ctx, top_level)?; let from_zeros = match &ctx.ast.data { Data::Struct(strct) => derive_from_zeros_struct(ctx, strct), @@ -98,6 +104,12 @@ pub(crate) fn derive_from_zeros(ctx: &Ctx, top_level: Trait) -> Result Result { + if let Some(span) = ctx.invariant_span { + return ctx.error_or_skip(Error::new( + span, + "cannot derive `FromBytes` for a type with invariants", + )); + } let from_zeros = derive_from_zeros(ctx, top_level)?; let from_bytes = match &ctx.ast.data { Data::Struct(strct) => derive_from_bytes_struct(ctx, strct), diff --git a/rust/zerocopy-derive/derive/into_bytes.rs b/rust/zerocopy-derive/derive/into_bytes.rs index 0103a78d087f..55c53de6b8c4 100644 --- a/rust/zerocopy-derive/derive/into_bytes.rs +++ b/rust/zerocopy-derive/derive/into_bytes.rs @@ -2,7 +2,7 @@ // use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{Data, DataEnum, DataStruct, DataUnion, Error, Type}; +use syn::{parse_quote, Data, DataEnum, DataStruct, DataUnion, Error, Ident, Type, WherePredicate}; use crate::{ repr::{EnumRepr, StructUnionRepr}, @@ -18,6 +18,74 @@ pub(crate) fn derive_into_bytes(ctx: &Ctx, _top_level: Trait) -> Result derive_into_bytes_union(ctx, unn), } } + +/// If every field is exactly `T`, `[T; _]`, or a final `[T]` for the same type +/// parameter `T`, returns the bounds required to prove this to rustc. +fn homogeneous_field_bounds(ctx: &Ctx, strct: &DataStruct) -> Option> { + fn strip_parens_and_groups(mut ty: &Type) -> &Type { + loop { + ty = match ty { + Type::Group(group) => &group.elem, + Type::Paren(paren) => &paren.elem, + ty => return ty, + }; + } + } + + fn type_is_parameter(ty: &Type, parameter: &Ident) -> bool { + match strip_parens_and_groups(ty) { + Type::Path(path) => path.qself.is_none() && path.path.is_ident(parameter), + _ => false, + } + } + + let fields = strct.fields(); + ctx.ast.generics.type_params().find_map(|parameter| { + let parameter = ¶meter.ident; + let elements = fields + .iter() + .enumerate() + .map(|(index, (_, _, ty))| match strip_parens_and_groups(ty) { + Type::Array(array) => Some(&*array.elem), + Type::Slice(slice) if index + 1 == fields.len() => Some(&*slice.elem), + Type::Slice(_) => None, + ty => Some(ty), + }) + .collect::>>()?; + + if !elements.iter().all(|element| type_is_parameter(element, parameter)) { + return None; + } + + let zerocopy_crate = &ctx.zerocopy_crate; + let mut bounds = fields + .iter() + .map(|(_, _, ty)| parse_quote!(#ty: #zerocopy_crate::IntoBytes)) + .collect::>(); + + // `Ident` equality is only a syntactic prefilter above. In + // macro-generated input, two identifiers can both print as `T` and + // compare equal here while having different hygiene contexts. For + // example, a call-site `T` could resolve to a `u8` type alias while a + // def-site `T` resolves to this struct's type parameter. Instantiating + // that parameter with `u16` would make the fields actually be `u8` and + // `u16`; treating them as homogeneous could overlook padding and cause + // us to emit an unsound `IntoBytes` impl. + // + // Quote every element type from its original field AST so that its + // hygiene context is retained, then use `Identity` to have rustc verify + // that it resolves to `parameter`. Do not deduplicate these predicates: + // identically-printed types may have different hygiene contexts and + // must each be checked by rustc. + bounds.extend(elements.into_iter().map(|element| { + parse_quote! { + #element: #zerocopy_crate::util::macro_util::Identity + } + })); + Some(bounds) + }) +} + fn derive_into_bytes_struct(ctx: &Ctx, strct: &DataStruct) -> Result { let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; @@ -25,8 +93,12 @@ fn derive_into_bytes_struct(ctx: &Ctx, strct: &DataStruct) -> Result Result Result Result Result Result TokenStream { + let ctx = Ctx::try_from_derive_input(input).unwrap(); + derive_into_bytes(&ctx, Trait::IntoBytes).unwrap() + } + + #[test] + fn raw_repr_matches_ordinary_repr() { + let ordinary = derive(parse_quote! { + #[repr(C)] + #[repr(align(8))] + struct Response { + status: u8, + } + }); + let raw = derive(parse_quote! { + #[repr(C)] + #[r#repr(r#align(8))] + struct Response { + status: u8, + } + }); + + let ordinary = ordinary.to_string(); + let raw = raw.to_string(); + + // In particular, both expansions contain the padding check which + // prevents the resulting type from implementing `IntoBytes`. + assert!(ordinary.contains("PaddingFree")); + assert_eq!(ordinary, raw); + } +} diff --git a/rust/zerocopy-derive/derive/known_layout.rs b/rust/zerocopy-derive/derive/known_layout.rs index d0c4cecfff15..d4f6ecff5730 100644 --- a/rust/zerocopy-derive/derive/known_layout.rs +++ b/rust/zerocopy-derive/derive/known_layout.rs @@ -81,7 +81,7 @@ fn raw_from_ptr_len( #[inline(always)] fn pointer_to_metadata(ptr: *mut Self) -> ::PointerMetadata { - <#trailing_field_ty>::pointer_to_metadata(ptr as *mut _) + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::pointer_to_metadata(ptr as *mut _) } } }; @@ -144,6 +144,7 @@ fn pointer_to_metadata(ptr: *mut Self) -> // Define the collection of type-level field handles. let field_defs = field_indices.iter().zip(fields).map(|(idx, (vis, _, _))| { quote! { + #[allow(missing_debug_implementations)] #vis struct #idx; } }); @@ -200,6 +201,7 @@ unsafe impl #impl_generics #zerocopy_crate::util::macro_util::Field<#idx> for #i // `#ty`, not `__ZerocopyKnownLayoutMaybeUninit` (see #2116). #repr #[doc(hidden)] + #[allow(missing_debug_implementations)] #vis struct __ZerocopyKnownLayoutMaybeUninit<#params> ( #(#core::mem::MaybeUninit< <#ident #ty_generics as diff --git a/rust/zerocopy-derive/derive/mod.rs b/rust/zerocopy-derive/derive/mod.rs index b3839fcf73c9..4ccb71807ee7 100644 --- a/rust/zerocopy-derive/derive/mod.rs +++ b/rust/zerocopy-derive/derive/mod.rs @@ -3,6 +3,7 @@ pub mod from_bytes; pub mod into_bytes; pub mod known_layout; +pub mod project; pub mod try_from_bytes; pub mod unaligned; @@ -102,13 +103,6 @@ pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result Result::Elem); + let elem = + if repr.get_packed().is_some() { quote!(#zerocopy_crate::Unalign<#elem>) } else { elem }; + + // SAFETY: The checks above require `repr(C)` or `repr(transparent)`, and + // `FieldBounds::TRAILING_SELF` requires the trailing field to be `SplitAt`. + // Its `Elem` therefore has the size, bit validity, and `UnsafeCell` + // coverage of the actual trailing element. For an unpacked struct, the + // trailing field is aligned, so its `SplitAt` alignment guarantee applies. + // For a packed struct, wrapping its `Elem` in `Unalign` preserves those + // representation properties and gives alignment 1. Thus the trailing + // slice can be accessed as `[#elem]` even if packing misaligns its field. + // Packing does not change the field's internal layout [1], so its element + // size and byte offsets are preserved in either case. + // + // [1] Per https://doc.rust-lang.org/1.93.0/reference/type-layout.html#layout.repr.alignment.intro: + // + // `packed` may also alter the padding between fields (although it will + // not alter the padding inside of any field). Ok(ImplBlockBuilder::new(ctx, &ctx.ast.data, Trait::SplitAt, FieldBounds::TRAILING_SELF) .inner_extras(quote! { - type Elem = <#trailing_field as #zerocopy_crate::SplitAt>::Elem; + type Elem = #elem; }) .build()) } diff --git a/rust/zerocopy-derive/derive/project.rs b/rust/zerocopy-derive/derive/project.rs new file mode 100644 index 000000000000..1cfde70624a7 --- /dev/null +++ b/rust/zerocopy-derive/derive/project.rs @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{ + parse_quote, spanned::Spanned as _, Data, DataEnum, DeriveInput, Error, Expr, Fields, Ident, + Index, Type, +}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{ + const_block, generate_tag_enum, Client, Ctx, DataExt, FieldBounds, FieldProjection, + ImplBlockBuilder, Trait, + }, +}; + +pub(crate) fn tag_ident(variant: &Ident) -> Ident { + ident!(("___ZEROCOPY_TAG_{}", variant), variant.span()) +} + +pub(crate) fn variant_struct_ident(variant: &Ident) -> Ident { + ident!(("___ZerocopyVariantStruct_{}", variant), variant.span()) +} + +pub(crate) fn variants_union_field_ident(variant: &Ident) -> Ident { + ident!(("__field_{}", variant), variant.span()) +} + +/// Generates a constant for the tag associated with each variant of the enum. +/// When we match on the enum's tag, each arm matches one of these constants. We +/// have to use constants here because: +/// +/// - The type that we're matching on is not the type of the tag, it's an +/// integer of the same size as the tag type and with the same bit patterns. +/// - We can't read the enum tag as an enum because the bytes may not represent +/// a valid variant. +/// - Patterns do not currently support const expressions, so we have to assign +/// these constants to names rather than use them inline in the `match` +/// statement. +pub(crate) fn generate_tag_consts(data: &DataEnum) -> TokenStream { + let tags = data.variants.iter().map(|variant| { + let variant_ident = &variant.ident; + let tag = tag_ident(variant_ident); + + quote! { + // This casts the enum variant to its discriminant, and then + // converts the discriminant to the target integral type via a + // numeric cast [1]. + // + // Because these are the same size, this is defined to be a no-op + // and therefore is a lossless conversion [2]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#enum-cast: + // + // Casts an enum to its discriminant. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#numeric-cast: + // + // Casting between two integers of the same size (e.g. i32 -> u32) + // is a no-op. + const #tag: ___ZerocopyTagPrimitive = + ___ZerocopyTag::#variant_ident as ___ZerocopyTagPrimitive; + } + }); + + quote! { + #(#tags)* + } +} + +#[derive(Copy, Clone, Eq, PartialEq)] +enum Validity { + Uninit, + Initialized, + Safe, +} + +impl Validity { + fn as_type(self, ctx: &Ctx) -> Type { + let zerocopy_crate = &ctx.zerocopy_crate; + match self { + Validity::Uninit => parse_quote!(#zerocopy_crate::invariant::Uninit), + Validity::Initialized => { + parse_quote!(#zerocopy_crate::invariant::Initialized) + } + Validity::Safe => parse_quote!(#zerocopy_crate::invariant::Safe), + } + } + + /// Computes the output validity of projecting a struct or union field, or + /// returns `None` if that projection must not be generated. + /// + /// Struct fields preserve all three validity invariants. Union fields + /// preserve `Uninit` and `Initialized`. No projection is generated from a + /// `Safe` union. + fn output_validity_for_struct_or_union(self, data: &Data) -> Option { + match (data, self) { + (Data::Struct(_), validity) => Some(validity), + (Data::Union(_), validity @ (Validity::Uninit | Validity::Initialized)) => { + Some(validity) + } + (Data::Union(_), Validity::Safe) => None, + (Data::Enum(_), _) => unreachable!(), + } + } +} + +/// Returns the projection variant ID for the struct or union described by +/// `ctx`. +/// +/// This intentionally does not handle enums: an enum field's variant ID +/// identifies its source variant rather than the representation of its +/// containing type. +pub(crate) fn struct_union_variant_id(ctx: &Ctx) -> Box { + let zerocopy_crate = &ctx.zerocopy_crate; + match &ctx.ast.data { + Data::Struct(_) => parse_quote!({ #zerocopy_crate::STRUCT_VARIANT_ID }), + Data::Union(_) => { + if StructUnionRepr::from_attrs(&ctx.ast.attrs).map(|repr| repr.is_c()).unwrap() { + parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) + } else { + parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) + } + } + Data::Enum(_) => unreachable!("struct_union_variant_id called for enum"), + } +} + +/// Generates field projection implementations for a struct or union. +/// +/// `repr(C)` unions use `REPR_C_UNION_VARIANT_ID`, so their projections +/// additionally implement `pointer::cast::Cast`. Other unions use +/// `UNION_VARIANT_ID`. +pub(crate) fn derive_projection_struct_union( + ctx: &Ctx, + data: &dyn DataExt, + client: Client, +) -> TokenStream { + let fields = ctx.ast.data.fields(); + if fields.is_empty() { + return quote! {}; + } + + let field_tokens = fields.iter().map(|(vis, ident, _)| { + let ident = ident!(("__z{}", ident), ident.span()); + quote! { + #vis enum #ident {} + } + }); + + let zerocopy_crate = &ctx.zerocopy_crate; + let variant_id = struct_union_variant_id(ctx); + + let core = ctx.core_path(); + let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag { client }, FieldBounds::None) + .inner_extras(quote! { + type Tag = (); + type ProjectToTag = #zerocopy_crate::pointer::cast::CastToUnit; + }) + .build(); + + let has_fields = fields.iter().map(move |(_, ident, ty)| { + let field_token = ident!(("__z{}", ident), ident.span()); + let projection = FieldProjection { + variant_id: variant_id.clone(), + field: parse_quote!(#field_token), + field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), + }; + let has_field_trait = Trait::HasField { client, projection: projection.clone() }; + let has_field_path = has_field_trait.crate_path(ctx); + let has_field = ImplBlockBuilder::new(ctx, data, has_field_trait, FieldBounds::None) + .inner_extras(quote! { + type Type = #ty; + + #[inline(always)] + fn project( + slf: #zerocopy_crate::pointer::PtrInner<'_, Self>, + ) -> *mut ::Type { + let slf = slf.as_ptr(); + // SAFETY: By invariant on `PtrInner`, `slf` is a non-null + // pointer whose referent is zero-sized or lives in a valid + // allocation. Since `#ident` is a struct or union field of + // `Self`, this projection preserves or shrinks the referent + // size, and so the resulting referent also fits in the same + // allocation. + unsafe { #core::ptr::addr_of_mut!((*slf).#ident) } + } + }) + .build(); + + let derive_project_field = |input_validity: Validity| { + let output_validity = + match input_validity.output_validity_for_struct_or_union(&ctx.ast.data) { + Some(output_validity) => output_validity.as_type(ctx), + None => return TokenStream::new(), + }; + let input_validity = input_validity.as_type(ctx); + let projection_preserves_alignment = StructUnionRepr::from_attrs(&ctx.ast.attrs) + .map(|repr| repr.get_packed().is_none()) + .unwrap(); + let output_alignment = if projection_preserves_alignment { + quote! { ___ZcAlignment } + } else { + quote! { #zerocopy_crate::invariant::Unaligned } + }; + ImplBlockBuilder::new( + ctx, + data, + Trait::ProjectField { + client, + projection: projection.clone(), + invariants: parse_quote!((___ZcAliasing, ___ZcAlignment, #input_validity)), + }, + FieldBounds::None, + ) + .param_extras(vec![ + parse_quote!(___ZcAliasing: #zerocopy_crate::invariant::Aliasing), + parse_quote!(___ZcAlignment: #zerocopy_crate::invariant::Alignment), + ]) + .inner_extras(quote! { + // SAFETY: Struct and union projections do not depend on the value of + // the referent, and are therefore infallible. + type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; + + // SAFETY: Projection preserves aliasing. It also preserves + // alignment unless the containing type is packed. `Validity` + // determines whether a projection is emitted and, if so, + // computes its output validity: struct fields preserve validity; + // union fields preserve `Uninit` and `Initialized`, and no + // projection is emitted for `Safe` union input. + type Invariants = (___ZcAliasing, #output_alignment, #output_validity); + }) + .build() + }; + + let project_uninit = derive_project_field(Validity::Uninit); + let project_initialized = derive_project_field(Validity::Initialized); + let project_safe = derive_project_field(Validity::Safe); + + quote! { + #has_field + #project_uninit + #project_initialized + #project_safe + } + }); + + const_block(field_tokens.into_iter().chain(Some(has_tag)).chain(has_fields).map(Some)) +} + +pub(crate) fn derive_enum( + ctx: &Ctx, + data: &DataEnum, + client: Client, +) -> Result { + // With no fields, there are no `HasField` or `ProjectField` impls to emit, + // and therefore no representation requirement to enforce. + if data.fields().is_empty() { + return Ok(TokenStream::new()); + } + + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + let (outer_tag_type, inner_tag_type) = if repr.is_c() { + (quote! { ___ZerocopyTag }, quote! { () }) + } else if repr.is_primitive() { + (quote! { () }, quote! { ___ZerocopyTag }) + } else { + return Err(Error::new( + ctx.ast.span(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + )); + }; + + let zerocopy_crate = &ctx.zerocopy_crate; + let core = &ctx.core_path(); + let client_path = &client.crate_path(ctx); + let has_tag_trait = Trait::HasTag { client }; + let has_tag_path = &has_tag_trait.crate_path(ctx); + let infallible = + quote! { #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible }; + let tag_enum = generate_tag_enum(ctx, &repr, data); + let tag_consts = generate_tag_consts(data); + + let generics = &ctx.ast.generics; + let (impl_generics, ref ty_generics, where_clause) = generics.split_for_impl(); + + // Synthesize a `repr(C)` struct for each fieldful variant. We must do this + // because Rust does not provide an `addr_of!` that works on enums. + let variant_structs = data.variants.iter().filter_map(move |variant| { + if matches!(variant.fields, Fields::Unit) { + return None; + } + + let ident = variant_struct_ident(&variant.ident); + let field_types = variant.fields.iter().map(|field| &field.ty); + let enum_name = &ctx.ast.ident; + let variant_struct: DeriveInput = parse_quote! { + #[repr(C)] + struct #ident #impl_generics ( + #core::mem::MaybeUninit<___ZerocopyInnerTag>, + #(#field_types,)* + #core::marker::PhantomData<#enum_name #ty_generics>, + ) #where_clause; + }; + + let projections = derive_projection_struct_union( + &ctx.with_input(&variant_struct), + &variant_struct.data, + client, + ); + + Some(quote! { + #variant_struct + #projections + }) + }); + + // Synthesize a `repr(C)` union encompassing all synthesized variants. + let variants_union = { + let fields = data.variants.iter().filter_map(|variant| { + if matches!(variant.fields, Fields::Unit) { + return None; + } + let field_name = variants_union_field_ident(&variant.ident); + let variant_struct = variant_struct_ident(&variant.ident); + Some(quote! { + #field_name: #core::mem::ManuallyDrop<#variant_struct #ty_generics>, + }) + }); + + let variants_union: DeriveInput = parse_quote! { + #[repr(C)] + union ___ZerocopyVariants #generics { + #(#fields)* + // A fieldless enum produces no variant structs, but a union must + // have at least one field. This unit does not affect `repr(C)` + // layout. + __nonempty: (), + } + }; + let projections = derive_projection_struct_union( + &ctx.with_input(&variants_union), + &variants_union.data, + client, + ); + + quote! { + #variants_union + #projections + } + }; + + // Synthesize a `repr(C)` struct that pairs the the tag with the synthesized + // variants union. In practice, the actual tag storage may be here, or + // inside the synthesized variant structs, depending on the `repr` of the + // enum. + let raw_enum: DeriveInput = parse_quote! { + #[repr(C)] + struct ___ZerocopyRawEnum #generics { + tag: ___ZerocopyOuterTag, + variants: ___ZerocopyVariants #ty_generics, + } + }; + + let raw_projections = + derive_projection_struct_union(&ctx.with_input(&raw_enum), &raw_enum.data, client); + + let has_tag = ImplBlockBuilder::new(ctx, data, has_tag_trait, FieldBounds::None) + .inner_extras(quote! { + type Tag = ___ZerocopyTag; + type ProjectToTag = #zerocopy_crate::pointer::cast::CastSized; + }) + .build(); + + let aliasing_param: &syn::GenericParam = + &parse_quote!(___ZcAliasing: #zerocopy_crate::invariant::Aliasing); + let reference_param: &syn::GenericParam = + &parse_quote!(___ZcAliasing: #zerocopy_crate::invariant::Reference); + let alignment_param: &syn::GenericParam = + &parse_quote!(___ZcAlignment: #zerocopy_crate::invariant::Alignment); + + let has_fields = data.variants().into_iter().flat_map(|(variant, fields)| { + let variant_ident = &variant.unwrap().ident; + let variants_union_field = variants_union_field_ident(variant_ident); + let infallible = &infallible; + + fields.into_iter().enumerate().map(move |(idx, (vis, ident, ty))| { + // Rust does not presently support explicit visibility modifiers on + // enum fields. Keep this assertion so that a future language + // change cannot silently invalidate the visibility invariant. + assert!(matches!(vis, syn::Visibility::Inherited)); + let projection = FieldProjection { + variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), + field: parse_quote!(()), + field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), + }; + // The generated variant struct's first field is its inner tag. + let variant_struct_field_index = Index::from(idx + 1); + let has_field_trait = Trait::HasField { + client, + projection: projection.clone(), + }; + let has_field_path = has_field_trait.crate_path(ctx); + let has_field = ImplBlockBuilder::new(ctx, data, has_field_trait, FieldBounds::None) + .inner_extras(quote! { + type Type = #ty; + + #[inline(always)] + fn project( + slf: #zerocopy_crate::pointer::PtrInner<'_, Self>, + ) -> *mut ::Type { + use #zerocopy_crate::pointer::cast::{CastSized, Projection}; + + slf.project::<___ZerocopyRawEnum #ty_generics, CastSized>() + .project::<_, Projection<#client_path, _, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(variants) }>>() + .project::<_, Projection<#client_path, _, { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variants_union_field) }>>() + .project::<_, Projection<#client_path, _, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(value) }>>() + .project::<_, Projection<#client_path, _, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variant_struct_field_index) }>>() + .as_ptr() + } + }) + .build(); + + // SAFETY: `projection` is constructed from the current field and + // `variant_ident`, and that same `variant_ident` selects the + // variants union field through which `has_field` projects. Thus the + // `HasField` impl, `ProjectField` impls, and valid projection's tag + // check all identify the same enum field. `idx` is the current + // field's zero-based position, and the generated variant struct + // prepends its inner tag before reproducing the enum fields in + // source order, so `idx + 1` selects that same field. The assertion + // above enforces that `repr` is `repr(C)` or a primitive + // representation; `___ZerocopyRawEnum` and its nested types model + // exactly the representations specified by [1] and [2]. Those + // representations use `repr(C)` structs for variant fields, so field + // projection preserves alignment. Each `ProjectField` impl preserves + // the input validity. The `Uninit` and `Initialized` projections are + // infallible because those validity invariants do not depend on the + // enum's tag. The `Safe` projection first checks that the tag + // selects `variant_ident`; its `Reference` aliasing bound prevents + // mutation of the tag between that check and field projection. The + // corresponding `HasTag` impl projects to the raw representation's + // tag using `CastSized`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/reference/type-layout.html#reprc-enums-with-fields: + // + // The representation of a `repr(C)` enum with fields is a + // `repr(C)` struct with two fields [...]: a `repr(C)` version of + // the enum with all fields removed ("the tag") [and] a `repr(C)` + // union of `repr(C)` structs for the fields of each variant [...]. + // + // [2] Per https://doc.rust-lang.org/1.56.0/reference/type-layout.html#primitive-representation-of-enums-with-fields: + // + // The representation of a primitive representation enum is a + // `repr(C)` union of `repr(C)` structs for each variant with a + // field. The first field of each struct in the union is [...] the + // tag and the remaining fields are the fields of that variant. + let derive_enum_project_field = |validity| { + let (error, is_projectable, aliasing_bound) = if validity == Validity::Safe { + assert!(!matches!(validity, Validity::Uninit | Validity::Initialized)); + ( + quote! { () }, + quote! { + #[inline(always)] + fn is_projectable( + tag: #zerocopy_crate::pointer::Ptr< + '_, + ::Tag, + ( + #zerocopy_crate::invariant::Shared, + ___ZcAlignment, + #zerocopy_crate::invariant::Safe, + ), + >, + ) -> #core::result::Result<(), ()> { + let tag = tag.read::<#zerocopy_crate::BecauseImmutable>(); + if tag == ___ZerocopyTag::#variant_ident { + #core::result::Result::Ok(()) + } else { + #core::result::Result::Err(()) + } + } + }, + reference_param.clone(), + ) + } else { + (infallible.clone(), quote! {}, aliasing_param.clone()) + }; + + let validity = validity.as_type(ctx); + let invariants: Box = + parse_quote!((___ZcAliasing, ___ZcAlignment, #validity)); + let project_field_trait = Trait::ProjectField { + client, + projection: projection.clone(), + invariants: invariants.clone(), + }; + + ImplBlockBuilder::new(ctx, data, project_field_trait, FieldBounds::None) + .param_extras(vec![aliasing_bound.clone(), alignment_param.clone()]) + .inner_extras(quote! { + type Error = #error; + type Invariants = #invariants; + #is_projectable + }) + .build() + }; + + let project_uninit = derive_enum_project_field(Validity::Uninit); + let project_initialized = derive_enum_project_field(Validity::Initialized); + let project_valid = derive_enum_project_field(Validity::Safe); + + quote! { + #has_field + #project_uninit + #project_initialized + #project_valid + } + }) + }); + + Ok(quote! { + #tag_enum + + type ___ZerocopyTagPrimitive = #zerocopy_crate::util::macro_util::SizeToTag< + { #core::mem::size_of::<___ZerocopyTag>() }, + >; + + #tag_consts + + type ___ZerocopyOuterTag = #outer_tag_type; + type ___ZerocopyInnerTag = #inner_tag_type; + + #(#variant_structs)* + #variants_union + + #raw_enum + #raw_projections + + #has_tag + #(#has_fields)* + }) +} + +pub(crate) fn derive(ctx: &Ctx, _top_level: Trait) -> Result { + match &ctx.ast.data { + Data::Struct(strct) => { + Ok(derive_projection_struct_union(ctx, strct, Client::ProjectDerive)) + } + Data::Union(unn) => Ok(derive_projection_struct_union(ctx, unn, Client::ProjectDerive)), + Data::Enum(enm) => { + let projections = derive_enum(ctx, enm, Client::ProjectDerive)?; + if projections.is_empty() { + Ok(projections) + } else { + Ok(const_block([Some(projections)])) + } + } + } +} diff --git a/rust/zerocopy-derive/derive/try_from_bytes.rs b/rust/zerocopy-derive/derive/try_from_bytes.rs index 44f083328786..ae0c796b91b1 100644 --- a/rust/zerocopy-derive/derive/try_from_bytes.rs +++ b/rust/zerocopy-derive/derive/try_from_bytes.rs @@ -1,383 +1,144 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT // -use proc_macro2::TokenStream; -use quote::quote; -use syn::{ - parse_quote, spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, DeriveInput, Error, - Expr, Fields, Ident, Index, Type, -}; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{quote, ToTokens as _}; +use syn::{spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, Error, Field}; use crate::{ - repr::{EnumRepr, StructUnionRepr}, + derive::project::{ + derive_enum, derive_projection_struct_union, generate_tag_consts, struct_union_variant_id, + tag_ident, + }, + repr::EnumRepr, util::{ - const_block, enum_size_from_repr, generate_tag_enum, Ctx, DataExt, FieldBounds, + enum_size_from_repr, generate_tag_enum, Client, Ctx, DataExt, FieldBounds, ImplBlockBuilder, Trait, TraitBound, }, }; -fn tag_ident(variant_ident: &Ident) -> Ident { - ident!(("___ZEROCOPY_TAG_{}", variant_ident), variant_ident.span()) -} -/// Generates a constant for the tag associated with each variant of the enum. -/// When we match on the enum's tag, each arm matches one of these constants. We -/// have to use constants here because: -/// -/// - The type that we're matching on is not the type of the tag, it's an -/// integer of the same size as the tag type and with the same bit patterns. -/// - We can't read the enum tag as an enum because the bytes may not represent -/// a valid variant. -/// - Patterns do not currently support const expressions, so we have to assign -/// these constants to names rather than use them inline in the `match` -/// statement. -fn generate_tag_consts(data: &DataEnum) -> TokenStream { - let tags = data.variants.iter().map(|v| { - let variant_ident = &v.ident; - let tag_ident = tag_ident(variant_ident); - - quote! { - // This casts the enum variant to its discriminant, and then - // converts the discriminant to the target integral type via a - // numeric cast [1]. - // - // Because these are the same size, this is defined to be a no-op - // and therefore is a lossless conversion [2]. - // - // [1] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#enum-cast: - // - // Casts an enum to its discriminant. - // - // [2] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#numeric-cast: - // - // Casting between two integers of the same size (e.g. i32 -> u32) - // is a no-op. - const #tag_ident: ___ZerocopyTagPrimitive = - ___ZerocopyTag::#variant_ident as ___ZerocopyTagPrimitive; - } - }); - - quote! { - #(#tags)* +fn candidate_ident(ctx: &Ctx) -> Ident { + // Choose a source-pointer name distinct from the field names. Mixed-site + // hygiene keeps it out of the user's invariant expressions. + let fields = ctx.ast.data.fields(); + let mut name = "candidate".to_owned(); + while fields.iter().any(|(_, field, _)| field.to_string().trim_start_matches("r#") == name) { + name.push('_'); } + Ident::new(&name, Span::mixed_site()) } -fn variant_struct_ident(variant_ident: &Ident) -> Ident { - ident!(("___ZerocopyVariantStruct_{}", variant_ident), variant_ident.span()) -} +/// Generates validation of every field of a struct or enum variant, leaving +/// the validated pointers in scope for subsequent field invariants. +/// A union is checked by invoking this once per field. +fn derive_variant_is_safe<'a>( + ctx: &Ctx, + variant_id: &TokenStream, + fields: impl IntoIterator, +) -> Result { + let fields = fields.into_iter().collect::>(); + if fields.is_empty() { + return Ok(quote!(true)); + } -/// Generates variant structs for the given enum variant. -/// -/// These are structs associated with each variant of an enum. They are -/// `repr(C)` tuple structs with the same fields as the variant after a -/// `MaybeUninit<___ZerocopyInnerTag>`. -/// -/// In order to unify the generated types for `repr(C)` and `repr(int)` enums, -/// we use a "fused" representation with fields for both an inner tag and an -/// outer tag. Depending on the repr, we will set one of these tags to the tag -/// type and the other to `()`. This lets us generate the same code but put the -/// tags in different locations. -fn generate_variant_structs(ctx: &Ctx, data: &DataEnum) -> TokenStream { - let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); - - let enum_name = &ctx.ast.ident; - - // All variant structs have a `PhantomData>` field because we - // don't know which generic parameters each variant will use, and unused - // generic parameters are a compile error. + let zerocopy_crate = &ctx.zerocopy_crate; let core = ctx.core_path(); - let phantom_ty = quote! { - #core::marker::PhantomData<#enum_name #ty_generics> - }; - - let variant_structs = data.variants.iter().filter_map(|variant| { - // We don't generate variant structs for unit variants because we only - // need to check the tag. This helps cut down our generated code a bit. - if matches!(variant.fields, Fields::Unit) { - return None; - } - - let variant_struct_ident = variant_struct_ident(&variant.ident); - let field_types = variant.fields.iter().map(|f| &f.ty); - - let variant_struct = parse_quote! { - #[repr(C)] - struct #variant_struct_ident #impl_generics ( - #core::mem::MaybeUninit<___ZerocopyInnerTag>, - #(#field_types,)* - #phantom_ty, - ) #where_clause; - }; - - // We do this rather than emitting `#[derive(::zerocopy::TryFromBytes)]` - // because that is not hygienic, and this is also more performant. - let try_from_bytes_impl = - derive_try_from_bytes(&ctx.with_input(&variant_struct), Trait::TryFromBytes) - .expect("derive_try_from_bytes should not fail on synthesized type"); - - Some(quote! { - #variant_struct - #try_from_bytes_impl - }) + let candidate = candidate_ident(ctx); + let field_names = fields.iter().enumerate().map(|(idx, field)| { + field + .ident + .as_ref() + .map(|name| name.to_token_stream()) + .unwrap_or_else(|| syn::Index::from(idx).to_token_stream()) }); - - quote! { - #(#variant_structs)* - } -} - -fn variants_union_field_ident(ident: &Ident) -> Ident { - // Field names are prefixed with `__field_` to prevent name collision - // with the `__nonempty` field. - ident!(("__field_{}", ident), ident.span()) -} - -fn generate_variants_union(ctx: &Ctx, data: &DataEnum) -> TokenStream { - let generics = &ctx.ast.generics; - let (_, ty_generics, _) = generics.split_for_impl(); - - let fields = data.variants.iter().filter_map(|variant| { - // We don't generate variant structs for unit variants because we only - // need to check the tag. This helps cut down our generated code a bit. - if matches!(variant.fields, Fields::Unit) { - return None; - } - - let field_name = variants_union_field_ident(&variant.ident); - let variant_struct_ident = variant_struct_ident(&variant.ident); - - let core = ctx.core_path(); - Some(quote! { - #field_name: #core::mem::ManuallyDrop<#variant_struct_ident #ty_generics>, + let field_checks = field_names + .map(|name| { + quote! { + #zerocopy_crate::into_inner!( + #candidate.project::< + #zerocopy_crate::project_clients::TryFromBytesDerive, + _, + { #variant_id }, + { #zerocopy_crate::ident_id!(#name) }, + >() + ).try_into_safe::<_, #zerocopy_crate::BecauseImmutable>() + } }) - }); - - let variants_union = parse_quote! { - #[repr(C)] - union ___ZerocopyVariants #generics { - #(#fields)* - // Enums can have variants with no fields, but unions must - // have at least one field. So we just add a trailing unit - // to ensure that this union always has at least one field. - // Because this union is `repr(C)`, this unit type does not - // affect the layout. - __nonempty: (), - } - }; - - let has_field = - derive_has_field_struct_union(&ctx.with_input(&variants_union), &variants_union.data); - - quote! { - #variants_union - #has_field + .collect::>(); + // Without invariants, no expression needs the validated pointers. Avoid + // introducing bindings that could collide with caller-defined items. + if ctx.invariant_span.is_none() { + return Ok(quote! { true #(&& #field_checks.is_ok())* }); } + + let field_bindings = fields + .iter() + .enumerate() + .map(|(idx, _)| ctx.fresh_ident(&format!("field_{}", idx))) + .collect::>(); + let bound_fields = + fields.iter().copied().zip(field_bindings.iter().cloned()).collect::>(); + let field_validations = fields + .iter() + .enumerate() + .map(|(idx, field)| { + let invariants = crate::invariant::parse(&field.attrs)?; + let invariants = invariants + .iter() + .map(|expression| crate::invariant::bind_fields(&bound_fields[..=idx], expression)); + Ok(quote! { + #( + // Keep `return` in an invariant from bypassing later fields. + if !{ #[inline(always)] || -> #core::primitive::bool { #invariants } }() { + return false; + } + )* + }) + }) + .collect::, Error>>()?; + Ok(quote! { + { + #( + // Retain each shared pointer for subsequent field invariants. + #[allow(unused_variables)] + let #field_bindings = match #field_checks { + #core::result::Result::Ok(#field_bindings) => #field_bindings, + #core::result::Result::Err(_) => return false, + }; + #field_validations + )* + true + } + }) } -/// Generates an implementation of `is_bit_valid` for an arbitrary enum. +/// Generates an implementation of `is_safe` for an arbitrary enum. /// -/// The general process is: -/// -/// 1. Generate a tag enum. This is an enum with the same repr, variants, and -/// corresponding discriminants as the original enum, but without any fields -/// on the variants. This gives us access to an enum where the variants have -/// the same discriminants as the one we're writing `is_bit_valid` for. -/// 2. Make constants from the variants of the tag enum. We need these because -/// we can't put const exprs in match arms. -/// 3. Generate variant structs. These are structs which have the same fields as -/// each variant of the enum, and are `#[repr(C)]` with an optional "inner -/// tag". -/// 4. Generate a variants union, with one field for each variant struct type. -/// 5. And finally, our raw enum is a `#[repr(C)]` struct of an "outer tag" and -/// the variants union. -/// -/// See these reference links for fully-worked example decompositions. -/// -/// - `repr(C)`: -/// - `repr(int)`: -/// - `repr(C, int)`: -pub(crate) fn derive_is_bit_valid( +/// For an enum with fields, [`derive_enum`] generates the representation model +/// and projection impls. This function reads the tag, matches it against the +/// enum's discriminants, and validates each field of the selected variant +/// through those projections. A fieldless enum needs only the generated tag +/// enum and discriminant constants. +pub(crate) fn derive_is_safe( ctx: &Ctx, data: &DataEnum, repr: &EnumRepr, ) -> Result { - let trait_path = Trait::TryFromBytes.crate_path(ctx); - let tag_enum = generate_tag_enum(ctx, repr, data); - let tag_consts = generate_tag_consts(data); - - let (outer_tag_type, inner_tag_type) = if repr.is_c() { - (quote! { ___ZerocopyTag }, quote! { () }) - } else if repr.is_primitive() { - (quote! { () }, quote! { ___ZerocopyTag }) - } else { + if !(repr.is_c() || repr.is_primitive()) { return Err(Error::new( ctx.ast.span(), "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", )); - }; - - let variant_structs = generate_variant_structs(ctx, data); - let variants_union = generate_variants_union(ctx, data); - - let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + } let zerocopy_crate = &ctx.zerocopy_crate; - let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag, FieldBounds::None) - .inner_extras(quote! { - type Tag = ___ZerocopyTag; - type ProjectToTag = #zerocopy_crate::pointer::cast::CastSized; - }) - .build(); - let has_fields = data.variants().into_iter().flat_map(|(variant, fields)| { - let variant_ident = &variant.unwrap().ident; - let variants_union_field_ident = variants_union_field_ident(variant_ident); - let field: Box = parse_quote!(()); - fields.into_iter().enumerate().map(move |(idx, (vis, ident, ty))| { - // Rust does not presently support explicit visibility modifiers on - // enum fields, but we guard against the possibility to ensure this - // derive remains sound. - assert!(matches!(vis, syn::Visibility::Inherited)); - let variant_struct_field_index = Index::from(idx + 1); - let (_, ty_generics, _) = ctx.ast.generics.split_for_impl(); - let has_field_trait = Trait::HasField { - variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), - // Since Rust does not presently support explicit visibility - // modifiers on enum fields, any public type is suitable here; - // we use `()`. - field: field.clone(), - field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), - }; - let has_field_path = has_field_trait.crate_path(ctx); - let has_field = ImplBlockBuilder::new( - ctx, - data, - has_field_trait, - FieldBounds::None, - ) - .inner_extras(quote! { - type Type = #ty; - - #[inline(always)] - fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut ::Type { - use #zerocopy_crate::pointer::cast::{CastSized, Projection}; - - slf.project::<___ZerocopyRawEnum #ty_generics, CastSized>() - .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(variants) }>>() - .project::<_, Projection<_, { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variants_union_field_ident) }>>() - .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(value) }>>() - .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variant_struct_field_index) }>>() - .as_ptr() - } - }) - .build(); - - let project = ImplBlockBuilder::new( - ctx, - data, - Trait::ProjectField { - variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), - // Since Rust does not presently support explicit visibility - // modifiers on enum fields, any public type is suitable - // here; we use `()`. - field: field.clone(), - field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), - invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), - }, - FieldBounds::None, - ) - .param_extras(vec![ - parse_quote!(Aliasing: #zerocopy_crate::invariant::Aliasing), - parse_quote!(Alignment: #zerocopy_crate::invariant::Alignment), - ]) - .inner_extras(quote! { - type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; - type Invariants = (Aliasing, Alignment, #zerocopy_crate::invariant::Initialized); - }) - .build(); - - quote! { - #has_field - #project - } - }) - }); - let core = ctx.core_path(); - let match_arms = data.variants.iter().map(|variant| { - let tag_ident = tag_ident(&variant.ident); - let variant_struct_ident = variant_struct_ident(&variant.ident); - let variants_union_field_ident = variants_union_field_ident(&variant.ident); - - if matches!(variant.fields, Fields::Unit) { - // Unit variants don't need any further validation beyond checking - // the tag. - quote! { - #tag_ident => true - } - } else { - quote! { - #tag_ident => { - // SAFETY: Since we know that the tag is `#tag_ident`, we - // know that no other `&`s exist which refer to this enum - // as any other variant. - let variant_md = variants.cast::< - _, - #zerocopy_crate::pointer::cast::Projection< - // #zerocopy_crate::ReadOnly<_>, - _, - { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, - { #zerocopy_crate::ident_id!(#variants_union_field_ident) } - >, - _ - >(); - let variant = variant_md.cast::< - #zerocopy_crate::ReadOnly<#variant_struct_ident #ty_generics>, - #zerocopy_crate::pointer::cast::CastSized, - (#zerocopy_crate::pointer::BecauseRead, _) - >(); - < - #variant_struct_ident #ty_generics as #trait_path - >::is_bit_valid(variant) - } - } - } - }); - - let generics = &ctx.ast.generics; - let raw_enum: DeriveInput = parse_quote! { - #[repr(C)] - struct ___ZerocopyRawEnum #generics { - tag: ___ZerocopyOuterTag, - variants: ___ZerocopyVariants #ty_generics, - } - }; - - let self_ident = &ctx.ast.ident; - let invariants_eq_impl = quote! { - // SAFETY: `___ZerocopyRawEnum` is designed to have the same layout, - // validity, and invariants as `Self`. - unsafe impl #impl_generics #zerocopy_crate::pointer::InvariantsEq<___ZerocopyRawEnum #ty_generics> for #self_ident #ty_generics #where_clause {} - }; - - let raw_enum_projections = - derive_has_field_struct_union(&ctx.with_input(&raw_enum), &raw_enum.data); - - let raw_enum = quote! { - #raw_enum - #invariants_eq_impl - #raw_enum_projections - }; - - Ok(quote! { - // SAFETY: We use `is_bit_valid` to validate that the bit pattern of the - // enum's tag corresponds to one of the enum's discriminants. Then, we - // check the bit validity of each field of the corresponding variant. - // Thus, this is a sound implementation of `is_bit_valid`. - #[inline] - fn is_bit_valid<___ZcAlignment>( - mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, - ) -> #core::primitive::bool - where - ___ZcAlignment: #zerocopy_crate::invariant::Alignment, - { + let candidate = candidate_ident(ctx); + let alignment = ctx.fresh_ident("___ZcAlignment"); + let tag = Ident::new("tag", Span::mixed_site()); + let projections = if data.fields().is_empty() { + let tag_enum = generate_tag_enum(ctx, repr, data); + let tag_consts = generate_tag_consts(data); + quote! { #tag_enum type ___ZerocopyTagPrimitive = #zerocopy_crate::util::macro_util::SizeToTag< @@ -385,58 +146,79 @@ fn is_bit_valid<___ZcAlignment>( >; #tag_consts + } + } else { + derive_enum(ctx, data, Client::TryFromBytesDerive)? + }; - type ___ZerocopyOuterTag = #outer_tag_type; - type ___ZerocopyInnerTag = #inner_tag_type; - - #variant_structs - - #variants_union - - #raw_enum - - #has_tag - - #(#has_fields)* - - let tag = { - // SAFETY: - // - The provided cast addresses a subset of the bytes addressed - // by `candidate` because it addresses the starting tag of the - // enum. - // - Because the pointer is cast from `candidate`, it has the - // same provenance as it. - // - There are no `UnsafeCell`s in the tag because it is a - // primitive integer. - // - `tag_ptr` is casted from `candidate`, whose referent is - // `Initialized`. Since we have not written uninitialized - // bytes into the referent, `tag_ptr` is also `Initialized`. - // - // FIXME(#2874): Revise this to a `cast` once `candidate` - // references a `ReadOnly`. - let tag_ptr = unsafe { - candidate.reborrow().project_transmute_unchecked::< - _, - #zerocopy_crate::invariant::Initialized, - #zerocopy_crate::pointer::cast::CastSized - >() - }; - tag_ptr.recall_validity::<_, (_, (_, _))>().read::<#zerocopy_crate::BecauseImmutable>() + let match_arms = data + .variants + .iter() + .enumerate() + .map(|(idx, variant)| { + let name = &variant.ident; + let variant_id = quote! { #zerocopy_crate::ident_id!(#name) }; + let fields_is_safe = derive_variant_is_safe(ctx, &variant_id, &variant.fields)?; + let pattern = if ctx.invariant_span.is_some() { + quote! { #core::option::Option::Some(#idx) } + } else { + tag_ident(name).to_token_stream() }; + Ok(quote! { + #pattern => { #fields_is_safe } + }) + }) + .collect::, Error>>()?; - let mut raw_enum = candidate.cast::< - #zerocopy_crate::ReadOnly<___ZerocopyRawEnum #ty_generics>, + let read_tag = quote! { + let #tag = #candidate + .reborrow() + .cast::< + ___ZerocopyTagPrimitive, #zerocopy_crate::pointer::cast::CastSized, - (#zerocopy_crate::pointer::BecauseRead, _) - >(); + (#zerocopy_crate::pointer::BecauseRead, _), + >() + .recall_validity::<_, (_, (_, _))>() + .read::<#zerocopy_crate::BecauseImmutable>(); + }; + let tag_init = if ctx.invariant_span.is_some() { + let allow = crate::util::allow_generated_code(); + let tag_arms = data.variants.iter().enumerate().map(|(idx, variant)| { + let tag = tag_ident(&variant.ident); + quote! { #tag => #core::option::Option::Some(#idx) } + }); + quote! { + // Keep implementation-only items and their lint allowances out + // of the scopes containing caller-authored invariant expressions. + #allow + let #tag = { + #projections + #read_tag + match #tag { + #(#tag_arms,)* + _ => #core::option::Option::None, + } + }; + } + } else { + quote! { #projections #read_tag } + }; - let variants = #zerocopy_crate::into_inner!(raw_enum.project::< - _, - { #zerocopy_crate::STRUCT_VARIANT_ID }, - { #zerocopy_crate::ident_id!(variants) } - >()); + Ok(quote! { + // SAFETY: We use `is_safe` to validate that the bit pattern of the + // enum's tag corresponds to one of the enum's discriminants. Then, we + // check the bit validity of each field of the corresponding variant. + // Thus, this is a sound implementation of `is_safe`. + #[inline] + fn is_safe<#alignment>( + mut #candidate: #zerocopy_crate::Maybe<'_, Self, #alignment>, + ) -> #core::primitive::bool + where + #alignment: #zerocopy_crate::invariant::Alignment, + { + #tag_init - match tag { + match #tag { #(#match_arms,)* _ => false, } @@ -447,219 +229,88 @@ pub(crate) fn derive_try_from_bytes(ctx: &Ctx, top_level: Trait) -> Result derive_try_from_bytes_struct(ctx, strct, top_level), Data::Enum(enm) => derive_try_from_bytes_enum(ctx, enm, top_level), - Data::Union(unn) => Ok(derive_try_from_bytes_union(ctx, unn, top_level)), + Data::Union(unn) => derive_try_from_bytes_union(ctx, unn, top_level), } } -fn derive_has_field_struct_union(ctx: &Ctx, data: &dyn DataExt) -> TokenStream { - let fields = ctx.ast.data.fields(); - if fields.is_empty() { - return quote! {}; - } - - let field_tokens = fields.iter().map(|(vis, ident, _)| { - let ident = ident!(("__z{}", ident), ident.span()); - quote!( - #vis enum #ident {} - ) - }); - - let zerocopy_crate = &ctx.zerocopy_crate; - let variant_id: Box = match &ctx.ast.data { - Data::Struct(_) => parse_quote!({ #zerocopy_crate::STRUCT_VARIANT_ID }), - Data::Union(_) => { - let is_repr_c = StructUnionRepr::from_attrs(&ctx.ast.attrs) - .map(|repr| repr.is_c()) - .unwrap_or(false); - if is_repr_c { - parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) - } else { - parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) - } - } - _ => unreachable!(), - }; - - let core = ctx.core_path(); - let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag, FieldBounds::None) - .inner_extras(quote! { - type Tag = (); - type ProjectToTag = #zerocopy_crate::pointer::cast::CastToUnit; - }) - .build(); - let has_fields = fields.iter().map(move |(_, ident, ty)| { - let field_token = ident!(("__z{}", ident), ident.span()); - let field: Box = parse_quote!(#field_token); - let field_id: Box = parse_quote!({ #zerocopy_crate::ident_id!(#ident) }); - let has_field_trait = Trait::HasField { - variant_id: variant_id.clone(), - field: field.clone(), - field_id: field_id.clone(), - }; - let has_field_path = has_field_trait.crate_path(ctx); - ImplBlockBuilder::new( - ctx, - data, - has_field_trait, - FieldBounds::None, - ) - .inner_extras(quote! { - type Type = #ty; - - #[inline(always)] - fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut ::Type { - let slf = slf.as_ptr(); - // SAFETY: By invariant on `PtrInner`, `slf` is a non-null - // pointer whose referent is zero-sized or lives in a valid - // allocation. Since `#ident` is a struct or union field of - // `Self`, this projection preserves or shrinks the referent - // size, and so the resulting referent also fits in the same - // allocation. - unsafe { #core::ptr::addr_of_mut!((*slf).#ident) } - } - }).outer_extras(if matches!(&ctx.ast.data, Data::Struct(..)) { - let fields_preserve_alignment = StructUnionRepr::from_attrs(&ctx.ast.attrs) - .map(|repr| repr.get_packed().is_none()) - .unwrap(); - let alignment = if fields_preserve_alignment { - quote! { Alignment } - } else { - quote! { #zerocopy_crate::invariant::Unaligned } - }; - // SAFETY: See comments on items. - ImplBlockBuilder::new( - ctx, - data, - Trait::ProjectField { - variant_id: variant_id.clone(), - field, - field_id, - invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), - }, - FieldBounds::None, - ) - .param_extras(vec![ - parse_quote!(Aliasing: #zerocopy_crate::invariant::Aliasing), - parse_quote!(Alignment: #zerocopy_crate::invariant::Alignment), - ]) - .inner_extras(quote! { - // SAFETY: Projection into structs is always infallible. - type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; - // SAFETY: The alignment of the projected `Ptr` is `Unaligned` - // if the structure is packed; otherwise inherited from the - // outer `Ptr`. If the validity of the outer pointer is - // `Initialized`, so too is the validity of its fields. - type Invariants = (Aliasing, #alignment, #zerocopy_crate::invariant::Initialized); - }) - .build() - } else { - quote! {} - }) - .build() - }); - - const_block(field_tokens.into_iter().chain(Some(has_tag)).chain(has_fields).map(Some)) -} fn derive_try_from_bytes_struct( ctx: &Ctx, strct: &DataStruct, top_level: Trait, ) -> Result { - let extras = try_gen_trivial_is_bit_valid(ctx, top_level).unwrap_or_else(|| { + let extras = if let Some(extras) = try_gen_trivial_is_safe(ctx, top_level) { + extras + } else { let zerocopy_crate = &ctx.zerocopy_crate; - let fields = strct.fields(); - let field_names = fields.iter().map(|(_vis, name, _ty)| name); - let field_tys = fields.iter().map(|(_vis, _name, ty)| ty); + let variant_id = quote! { #zerocopy_crate::STRUCT_VARIANT_ID }; + let fields_is_safe = derive_variant_is_safe(ctx, &variant_id, &strct.fields)?; let core = ctx.core_path(); + let candidate = candidate_ident(ctx); + let alignment = ctx.fresh_ident("___ZcAlignment"); quote!( - // SAFETY: We use `is_bit_valid` to validate that each field is - // bit-valid, and only return `true` if all of them are. The bit - // validity of a struct is just the composition of the bit - // validities of its fields, so this is a sound implementation - // of `is_bit_valid`. + // SAFETY: We use `is_safe` to validate that each field is bit-valid, + // and only return `true` if all of them are. The bit validity of a + // struct is just the composition of the bit validities of its + // fields, so this is a sound implementation of `is_safe`. #[inline] - fn is_bit_valid<___ZcAlignment>( - mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + fn is_safe<#alignment>( + mut #candidate: #zerocopy_crate::Maybe<'_, Self, #alignment>, ) -> #core::primitive::bool where - ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + #alignment: #zerocopy_crate::invariant::Alignment, { - true #(&& { - let field_candidate = #zerocopy_crate::into_inner!(candidate.reborrow().project::< - _, - { #zerocopy_crate::STRUCT_VARIANT_ID }, - { #zerocopy_crate::ident_id!(#field_names) } - >()); - <#field_tys as #zerocopy_crate::TryFromBytes>::is_bit_valid(field_candidate) - })* + #fields_is_safe } ) - }); + }; Ok(ImplBlockBuilder::new(ctx, strct, Trait::TryFromBytes, FieldBounds::ALL_SELF) .inner_extras(extras) - .outer_extras(derive_has_field_struct_union(ctx, strct)) + .outer_extras(derive_projection_struct_union(ctx, strct, Client::TryFromBytesDerive)) .build()) } -fn derive_try_from_bytes_union(ctx: &Ctx, unn: &DataUnion, top_level: Trait) -> TokenStream { +fn derive_try_from_bytes_union( + ctx: &Ctx, + unn: &DataUnion, + top_level: Trait, +) -> Result { let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); let zerocopy_crate = &ctx.zerocopy_crate; - let variant_id: Box = { - let is_repr_c = - StructUnionRepr::from_attrs(&ctx.ast.attrs).map(|repr| repr.is_c()).unwrap_or(false); - if is_repr_c { - parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) - } else { - parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) - } - }; - - let extras = try_gen_trivial_is_bit_valid(ctx, top_level).unwrap_or_else(|| { - let fields = unn.fields(); - let field_names = fields.iter().map(|(_vis, name, _ty)| name); - let field_tys = fields.iter().map(|(_vis, _name, ty)| ty); + let union_variant_id = struct_union_variant_id(ctx).to_token_stream(); + let extras = if let Some(extras) = try_gen_trivial_is_safe(ctx, top_level) { + extras + } else { + let fields_is_safe = unn + .fields + .named + .iter() + .map(|field| derive_variant_is_safe(ctx, &union_variant_id, [field])) + .collect::, _>>()?; let core = ctx.core_path(); + let candidate = candidate_ident(ctx); + let alignment = ctx.fresh_ident("___ZcAlignment"); quote!( - // SAFETY: We use `is_bit_valid` to validate that any field is - // bit-valid; we only return `true` if at least one of them is. - // The bit validity of a union is not yet well defined in Rust, - // but it is guaranteed to be no more strict than this - // definition. See #696 for a more in-depth discussion. + // SAFETY: We use `is_safe` to validate that any field is bit-valid; + // we only return `true` if at least one of them is and its + // invariants also hold. The bit validity of a union is not yet + // well defined in Rust, but it is guaranteed to be no more strict + // than this definition. See #696 for a more in-depth discussion. #[inline] - fn is_bit_valid<___ZcAlignment>( - mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + fn is_safe<#alignment>( + mut #candidate: #zerocopy_crate::Maybe<'_, Self, #alignment>, ) -> #core::primitive::bool where - ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + #alignment: #zerocopy_crate::invariant::Alignment, { - false #(|| { - // SAFETY: - // - Since `ReadOnly: Immutable` unconditionally, - // neither `*slf` nor the returned pointer's referent - // permit interior mutation. - // - Both source and destination validity are - // `Initialized`, which is always a sound - // transmutation. - let field_candidate = unsafe { - candidate.reborrow().project_transmute_unchecked::< - _, - _, - #zerocopy_crate::pointer::cast::Projection< - _, - #variant_id, - { #zerocopy_crate::ident_id!(#field_names) } - > - >() - }; - - <#field_tys as #zerocopy_crate::TryFromBytes>::is_bit_valid(field_candidate) - })* + // Keep each field's bindings and early returns local to that + // field, so a failure lets validation try the next field. + false #(|| { #[inline(always)] || { #fields_is_safe } }())* } ) - }); - ImplBlockBuilder::new(ctx, unn, Trait::TryFromBytes, field_type_trait_bounds) + }; + Ok(ImplBlockBuilder::new(ctx, unn, Trait::TryFromBytes, field_type_trait_bounds) .inner_extras(extras) - .outer_extras(derive_has_field_struct_union(ctx, unn)) - .build() + .outer_extras(derive_projection_struct_union(ctx, unn, Client::TryFromBytesDerive)) + .build()) } fn derive_try_from_bytes_enum( ctx: &Ctx, @@ -677,16 +328,15 @@ fn derive_try_from_bytes_enum( .map(|size| enm.fields().is_empty() && enm.variants.len() == 1usize << size) .unwrap_or(false); - let trivial_is_bit_valid = try_gen_trivial_is_bit_valid(ctx, top_level); - let extra = match (trivial_is_bit_valid, could_be_from_bytes) { - (Some(is_bit_valid), _) => is_bit_valid, + let trivial_is_safe = try_gen_trivial_is_safe(ctx, top_level); + let extra = match (trivial_is_safe, could_be_from_bytes) { + (Some(is_safe), _) => is_safe, // SAFETY: It would be sound for the enum to implement `FromBytes`, as - // required by `gen_trivial_is_bit_valid_unchecked`. - (None, true) => unsafe { gen_trivial_is_bit_valid_unchecked(ctx) }, - (None, false) => match derive_is_bit_valid(ctx, enm, &repr) { + // required by `gen_trivial_is_safe_unchecked`. + (None, true) => unsafe { gen_trivial_is_safe_unchecked(ctx) }, + (None, false) => match derive_is_safe(ctx, enm, &repr) { Ok(extra) => extra, - Err(_) if ctx.skip_on_error => return Ok(TokenStream::new()), - Err(e) => return Err(e), + Err(e) => return ctx.error_or_skip(e), }, }; @@ -694,15 +344,14 @@ fn derive_try_from_bytes_enum( .inner_extras(extra) .build()) } -fn try_gen_trivial_is_bit_valid(ctx: &Ctx, top_level: Trait) -> Option { +fn try_gen_trivial_is_safe(ctx: &Ctx, top_level: Trait) -> Option { // If the top-level trait is `FromBytes` and `Self` has no type parameters, // then the `FromBytes` derive will fail compilation if `Self` is not - // actually soundly `FromBytes`, and so we can rely on that for our - // `is_bit_valid` impl. It's plausible that we could make changes - or Rust - // could make changes (such as the "trivial bounds" language feature) - that - // make this no longer true. To hedge against these, we include an explicit - // `Self: FromBytes` check in the generated `is_bit_valid`, which is - // bulletproof. + // actually soundly `FromBytes`, and so we can rely on that for our `is_safe` + // impl. It's plausible that we could make changes - or Rust could make + // changes (such as the "trivial bounds" language feature) - that make this + // no longer true. To hedge against these, we include an explicit `Self: + // FromBytes` check in the generated `is_safe`, which is bulletproof. // // If `ctx.skip_on_error` is true, we can't rely on the `FromBytes` derive // to fail compilation if `Self` is not actually soundly `FromBytes`. @@ -715,7 +364,7 @@ fn try_gen_trivial_is_bit_valid(ctx: &Ctx, top_level: Trait) -> Option( + fn is_safe<___ZcAlignment>( _candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, ) -> #core::primitive::bool where @@ -746,14 +395,14 @@ fn assert_is_from_bytes() /// # Safety /// /// All initialized bit patterns must be valid for `Self`. -unsafe fn gen_trivial_is_bit_valid_unchecked(ctx: &Ctx) -> proc_macro2::TokenStream { +unsafe fn gen_trivial_is_safe_unchecked(ctx: &Ctx) -> proc_macro2::TokenStream { let zerocopy_crate = &ctx.zerocopy_crate; let core = ctx.core_path(); quote!( - // SAFETY: The caller of `gen_trivial_is_bit_valid_unchecked` has - // promised that all initialized bit patterns are valid for `Self`. + // SAFETY: The caller of `gen_trivial_is_safe_unchecked` has promised + // that all initialized bit patterns are valid for `Self`. #[inline(always)] - fn is_bit_valid<___ZcAlignment>( + fn is_safe<___ZcAlignment>( _candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, ) -> #core::primitive::bool where @@ -763,3 +412,43 @@ fn is_bit_valid<___ZcAlignment>( } ) } + +#[cfg(test)] +mod tests { + use syn::{parse_quote, DeriveInput}; + + use super::*; + + fn derive_error(input: DeriveInput) -> String { + let ctx = Ctx::try_from_derive_input(input).unwrap(); + match derive_try_from_bytes(&ctx, Trait::TryFromBytes) { + Ok(_) => panic!("expected derive to reject conflicting representations"), + Err(error) => error.to_string(), + } + } + + #[test] + fn raw_repr_conflict_matches_ordinary_repr() { + let ordinary = derive_error(parse_quote! { + #[repr(C, align(8))] + #[repr(u8)] + enum Packet { + Value(core::num::NonZeroU8), + End, + } + }); + let raw = derive_error(parse_quote! { + #[repr(C, align(8))] + #[r#repr(r#u8)] + enum Packet { + Value(core::num::NonZeroU8), + End, + } + }); + + // Rejecting both spellings prevents generating a validator whose + // synthetic enum layout disagrees with the compiler's actual layout. + assert_eq!(ordinary, "this conflicts with another representation hint"); + assert_eq!(ordinary, raw); + } +} diff --git a/rust/zerocopy-derive/invariant.rs b/rust/zerocopy-derive/invariant.rs new file mode 100644 index 000000000000..f7a1a617e378 --- /dev/null +++ b/rust/zerocopy-derive/invariant.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use proc_macro2::{Ident, Span, TokenStream, TokenTree}; +use quote::{quote, ToTokens as _}; +use syn::{ + punctuated::Punctuated, spanned::Spanned as _, Attribute, Data, DeriveInput, Error, Expr, + Field, MacroDelimiter, Meta, Token, +}; + +use crate::util::path_is_ident; + +/// Parses field invariants in attribute order. +pub(crate) fn parse(attrs: &[Attribute]) -> Result, Error> { + let mut invariants = Vec::new(); + for attr in attrs.iter().filter(|attr| path_is_ident(attr.path(), "zerocopy")) { + let options = attr.parse_args_with(Punctuated::::parse_terminated)?; + for option in options { + if !path_is_ident(option.path(), "invariant") { + return Err(Error::new_spanned(option, "expected `invariant(...)`")); + } + match option { + Meta::List(list) if matches!(list.delimiter, MacroDelimiter::Paren(_)) => { + invariants.push(list.parse_args::()?) + } + other => return Err(Error::new_spanned(other, "expected `invariant(...)`")), + } + } + } + Ok(invariants) +} + +/// Validates invariant syntax and placement, returning the first invariant's +/// span so derives which cannot honor invariants can reject them. +pub(crate) fn validate(ast: &DeriveInput) -> Result, Error> { + let fields: Vec<&Field> = match &ast.data { + Data::Struct(data) => data.fields.iter().collect(), + Data::Enum(data) => { + for variant in &data.variants { + if let Some(invariant) = parse(&variant.attrs)?.first() { + return Err(Error::new( + invariant.span(), + "invariants are only supported on named fields", + )); + } + } + data.variants.iter().flat_map(|variant| &variant.fields).collect() + } + Data::Union(data) => data.fields.named.iter().collect(), + }; + let mut first = None; + for field in fields { + if let Some(invariant) = parse(&field.attrs)?.first() { + if field.ident.is_none() { + return Err(Error::new( + invariant.span(), + "invariants are only supported on named fields", + )); + } + first.get_or_insert(invariant.span()); + } + } + Ok(first) +} + +/// Visits identifiers, including those inside macro arguments. +pub(crate) fn idents(tokens: TokenStream) -> Vec { + tokens + .into_iter() + .flat_map(|token| match token { + TokenTree::Ident(ident) => vec![ident], + TokenTree::Group(group) => idents(group.stream()), + _ => Vec::new(), + }) + .collect() +} + +/// Binds semantic field names in every syntax context used by the expression. +/// Preserve the expression's tokens so its other names retain their hygiene. +pub(crate) fn bind_fields(fields: &[(&Field, Ident)], expression: &Expr) -> TokenStream { + let mut body = expression.to_token_stream(); + let references = idents(body.clone()); + for (field, binding) in fields { + let name = match &field.ident { + Some(name) => name, + None => continue, + }; + let spelling = name.to_string(); + let spelling = spelling.trim_start_matches("r#"); + // Also support references produced by a macro invoked by the expression. + let mut call_site = name.clone(); + call_site.set_span(Span::call_site()); + for alias in [name.clone(), call_site].into_iter().chain( + references + .iter() + .filter(|ident| ident.to_string().trim_start_matches("r#") == spelling) + .cloned(), + ) { + body = quote! {{ + // Require a binding; reject names Rust would resolve as constant + // patterns. Item aliases could capture helper calls in macros. + #[allow(unused_variables, non_snake_case, clippy::redundant_pattern)] + let #alias @ _ = #binding; + #body + }}; + } + } + body +} diff --git a/rust/zerocopy-derive/lib.rs b/rust/zerocopy-derive/lib.rs index d387de368367..c650b59eaa82 100644 --- a/rust/zerocopy-derive/lib.rs +++ b/rust/zerocopy-derive/lib.rs @@ -28,6 +28,10 @@ #![allow(clippy::type_complexity)] // Inlining format args isn't supported on our MSRV. #![allow(clippy::uninlined_format_args)] +// `cargo-zerocopy` supplies this cfg for pinned-nightly tests. During UI tests, +// `testutil::UiTestRunner` explicitly supplies it to the host-built proc macro; +// ordinary `RUSTFLAGS` are not sufficient when Cargo receives `--target`. +#![cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, feature(proc_macro_def_site))] #![deny( rustdoc::bare_urls, rustdoc::broken_intra_doc_links, @@ -46,6 +50,7 @@ macro_rules! ident { } mod derive; +mod invariant; #[cfg(test)] mod output_tests; mod repr; @@ -79,7 +84,8 @@ macro_rules! ident { /// are currently required to live at the crate root, and so the caller must /// specify the name in order to avoid name collisions. macro_rules! derive { - ($trait:ident => $outer:ident => $inner:path) => { + ($(#[$attr:meta])* $trait:ident => $outer:ident => $inner:path) => { + $(#[$attr])* #[proc_macro_derive($trait, attributes(zerocopy))] pub fn $outer(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { let ast = syn::parse_macro_input!(ts as DeriveInput); @@ -88,10 +94,13 @@ pub fn $outer(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { Err(e) => return e.into_compile_error().into(), }; let ts = $inner(&ctx, Trait::$trait).into_ts(); - // We wrap in `const_block` as a backstop in case any derive fails - // to wrap its output in `const_block` (and thus fails to annotate) - // with the full set of `#[allow(...)]` attributes). - let ts = const_block([Some(ts)]); + // Apply generated-code lint allowances as a backstop, except + // around caller-authored invariant expressions. + let ts = if matches!(Trait::$trait, Trait::TryFromBytes) { + ctx.const_block([Some(ts)]) + } else { + const_block([Some(ts)]) + }; #[cfg(test)] crate::util::testutil::check_hygiene(ts.clone()); ts.into() @@ -120,6 +129,7 @@ fn into_ts(self) -> proc_macro2::TokenStream { derive!(KnownLayout => derive_known_layout => crate::derive::known_layout::derive); derive!(Immutable => derive_immutable => crate::derive::derive_immutable); +derive!(#[doc(hidden)] Project => derive_project => crate::derive::project::derive); derive!(TryFromBytes => derive_try_from_bytes => crate::derive::try_from_bytes::derive_try_from_bytes); derive!(FromZeros => derive_from_zeros => crate::derive::from_bytes::derive_from_zeros); derive!(FromBytes => derive_from_bytes => crate::derive::from_bytes::derive_from_bytes); @@ -129,6 +139,89 @@ fn into_ts(self) -> proc_macro2::TokenStream { derive!(ByteEq => derive_eq => crate::derive::derive_eq); derive!(SplitAt => derive_split_at => crate::derive::derive_split_at); +/// Generates a struct whose two identically-printed field types resolve to +/// different types. This is used to test that derives preserve type identity +/// across macro hygiene contexts. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] +#[proc_macro] +pub fn __test_hygienically_mixed_into_bytes( + _input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + use proc_macro::{Group, Ident, Span, TokenStream, TokenTree}; + + fn rewrite(input: TokenStream) -> TokenStream { + input + .into_iter() + .map(|token| match token { + TokenTree::Ident(ident) if ident.to_string() == "CallT" => { + TokenTree::Ident(Ident::new("T", Span::call_site())) + } + TokenTree::Ident(ident) if ident.to_string() == "DefT" => { + TokenTree::Ident(Ident::new("T", Span::def_site())) + } + TokenTree::Group(group) => { + let mut rewritten = Group::new(group.delimiter(), rewrite(group.stream())); + rewritten.set_span(group.span()); + TokenTree::Group(rewritten) + } + token => token, + }) + .collect() + } + + rewrite( + "#[derive(zerocopy_renamed::IntoBytes)] \ + #[zerocopy(crate = \"zerocopy_renamed\")] \ + #[repr(C)] \ + struct IntoBytes15(CallT, DefT);" + .parse() + .expect("test input must parse"), + ) +} + +/// Constructs an invariant whose field declaration and reference have different +/// syntax contexts, while preserving all other caller tokens. +#[doc(hidden)] +#[proc_macro] +pub fn __test_hygienically_mixed_invariant( + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + // Cross-compilation does not necessarily pass the nightly test cfg to + // host proc macros. Omit the fixture and its test together in that case. + #[cfg(not(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] + { + let _ = input; + proc_macro::TokenStream::new() + } + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + { + use proc_macro::{Group, Ident, Span, TokenStream, TokenTree}; + + fn rewrite(input: TokenStream) -> TokenStream { + input + .into_iter() + .map(|token| match token { + TokenTree::Ident(ident) if ident.to_string() == "DefField" => { + TokenTree::Ident(Ident::new("field", Span::def_site())) + } + TokenTree::Ident(ident) if ident.to_string() == "CallField" => { + TokenTree::Ident(Ident::new("field", Span::call_site())) + } + TokenTree::Group(group) => { + let mut rewritten = Group::new(group.delimiter(), rewrite(group.stream())); + rewritten.set_span(group.span()); + TokenTree::Group(rewritten) + } + token => token, + }) + .collect() + } + + rewrite(input) + } +} + #[cfg_attr(not(zerocopy_unstable_linux), doc(hidden))] #[proc_macro_derive(most_traits, attributes(zerocopy))] pub fn most_traits(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -153,11 +246,18 @@ pub fn most_traits(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { for (derive, t) in derives { tokens.extend(derive(&ctx, t)) } + // Invariants prevent `FromBytes` from generating its usual supertrait + // impls, but still permit checked conversions through `TryFromBytes`. + if ctx.invariant_span.is_some() { + tokens.extend(crate::derive::try_from_bytes::derive_try_from_bytes( + &ctx, + Trait::TryFromBytes, + )); + } - // We wrap in `const_block` as a backstop in case any derive fails - // to wrap its output in `const_block` (and thus fails to annotate) - // with the full set of `#[allow(...)]` attributes). - let ts = const_block([Some(tokens)]); + // Apply generated-code lint allowances as a backstop, except around + // caller-authored invariant expressions. + let ts = ctx.const_block([Some(tokens)]); #[cfg(test)] crate::util::testutil::check_hygiene(ts.clone()); ts.into() diff --git a/rust/zerocopy-derive/repr.rs b/rust/zerocopy-derive/repr.rs index 1525e94302d1..a1a877c0f47c 100644 --- a/rust/zerocopy-derive/repr.rs +++ b/rust/zerocopy-derive/repr.rs @@ -20,6 +20,8 @@ MetaList, }; +use crate::util::{path_is_ident, to_ident_str}; + /// The computed representation of a type. /// /// This is the result of processing all `#[repr(...)]` attributes on a type, if @@ -524,11 +526,11 @@ fn from_attrs( let mut reprs = Vec::new(); for attr in attrs { // Ignore documentation attributes. - if attr.path().is_ident("doc") { + if path_is_ident(attr.path(), "doc") { continue; } if let Meta::List(ref meta_list) = attr.meta { - if meta_list.path.is_ident("repr") { + if path_is_ident(&meta_list.path, "repr") { let parsed: Punctuated = match meta_list.parse_args_with(Punctuated::parse_terminated) { Ok(parsed) => parsed, @@ -564,6 +566,8 @@ fn from_meta(meta: &Meta) -> Result { let ident = path.get_ident().ok_or(UnrecognizedReprError)?; // Only returns `Ok` for non-zero power-of-two values. + // `NonZeroU32::is_power_of_two` is unsupported on our MSRV. + #[allow(clippy::needless_nonzero_get)] let parse_nzu64 = |list: &MetaList| { list.parse_args::() .and_then(|int| int.base10_parse::()) @@ -578,7 +582,8 @@ fn from_meta(meta: &Meta) -> Result { }; use RawRepr::*; - Ok(match (ident.to_string().as_str(), list) { + let ident = to_ident_str(ident); + Ok(match (ident.as_str(), list) { ("u8", None) => U8, ("u16", None) => U16, ("u32", None) => U32, @@ -743,7 +748,40 @@ macro_rules! test { use PrimitiveRepr::*; let nz = |n: u32| NonZeroU32::new(n).unwrap(); + // Rust treats raw identifiers and their ordinary spellings as the + // same representation hints. Exercise every spelling recognized by + // `RawRepr::from_meta` so that adding a hint cannot accidentally + // reintroduce raw-identifier-sensitive parsing. + macro_rules! test_raw_repr { + ($ordinary:meta, $raw:meta => $expected:expr) => {{ + let ordinary: Meta = parse_quote!($ordinary); + let raw: Meta = parse_quote!($raw); + assert_eq!(RawRepr::from_meta(&ordinary).ok().unwrap(), $expected); + assert_eq!(RawRepr::from_meta(&raw).ok().unwrap(), $expected); + }}; + } + + test_raw_repr!(transparent, r#transparent => RawRepr::Transparent); + test_raw_repr!(C, r#C => RawRepr::C); + test_raw_repr!(Rust, r#Rust => RawRepr::Rust); + test_raw_repr!(u8, r#u8 => RawRepr::U8); + test_raw_repr!(u16, r#u16 => RawRepr::U16); + test_raw_repr!(u32, r#u32 => RawRepr::U32); + test_raw_repr!(u64, r#u64 => RawRepr::U64); + test_raw_repr!(u128, r#u128 => RawRepr::U128); + test_raw_repr!(usize, r#usize => RawRepr::Usize); + test_raw_repr!(i8, r#i8 => RawRepr::I8); + test_raw_repr!(i16, r#i16 => RawRepr::I16); + test_raw_repr!(i32, r#i32 => RawRepr::I32); + test_raw_repr!(i64, r#i64 => RawRepr::I64); + test_raw_repr!(i128, r#i128 => RawRepr::I128); + test_raw_repr!(isize, r#isize => RawRepr::Isize); + test_raw_repr!(align(2), r#align(2) => RawRepr::Align(nz(2))); + test_raw_repr!(packed, r#packed => RawRepr::Packed); + test_raw_repr!(packed(2), r#packed(2) => RawRepr::PackedN(nz(2))); + test!(#[repr(transparent)] => StructUnionRepr::Transparent(s())); + test!(#[r#repr(r#transparent)] => StructUnionRepr::Transparent(s())); test!(#[repr()] => StructUnionRepr::Compound(Rust.into(), None)); test!(#[repr(packed)] => StructUnionRepr::Compound(Rust.into(), Some(Packed(nz(1)).into()))); test!(#[repr(packed(2))] => StructUnionRepr::Compound(Rust.into(), Some(Packed(nz(2)).into()))); @@ -754,6 +792,8 @@ macro_rules! test { test!(#[repr(C, packed(2))] => StructUnionRepr::Compound(C.into(), Some(Packed(nz(2)).into()))); test!(#[repr(C, align(1))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(1)).into()))); test!(#[repr(C, align(2))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(2)).into()))); + test!(#[r#repr(r#C, r#align(2))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(2)).into()))); + test!(#[r#repr(r#C, r#packed(2))] => StructUnionRepr::Compound(C.into(), Some(Packed(nz(2)).into()))); test!(#[repr(transparent)] => EnumRepr::Transparent(s())); test!(#[repr()] => EnumRepr::Compound(Rust.into(), None)); diff --git a/rust/zerocopy-derive/util.rs b/rust/zerocopy-derive/util.rs index 5c5e9d3bdcb8..717e068dd81c 100644 --- a/rust/zerocopy-derive/util.rs +++ b/rust/zerocopy-derive/util.rs @@ -13,9 +13,9 @@ use proc_macro2::{Span, TokenStream}; use quote::{quote, quote_spanned, ToTokens}; use syn::{ - parse_quote, spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, DeriveInput, Error, - Expr, ExprLit, Field, GenericParam, Ident, Index, Lit, LitStr, Meta, Path, Type, Variant, - Visibility, WherePredicate, + parse::ParseBuffer, parse_quote, spanned::Spanned as _, token::PathSep, Data, DataEnum, + DataStruct, DataUnion, DeriveInput, Error, Expr, ExprLit, Field, GenericParam, Ident, Index, + Lit, LitStr, Meta, Path, Type, Variant, Visibility, WherePredicate, }; use crate::repr::{CompoundRepr, EnumRepr, PrimitiveRepr, Repr, Spanned}; @@ -30,6 +30,59 @@ pub(crate) struct Ctx { // The span of the last `#[zerocopy(on_error = ...)]` attribute, if any. pub(crate) on_error_span: Option, + + /// The first field invariant on the source type, if any. + pub(crate) invariant_span: Option, +} + +#[derive(Eq, PartialEq)] +enum CratePath { + External, + CrateRelative, + ModuleRelative, +} + +fn validate_crate_path(path: &Path) -> Result { + if path.segments.is_empty() { + return Err(()); + } + + enum ModuleRelative { + Yes, + No, + } + let first = path.segments[0].ident.to_string(); + let (mut prev_segment, path_type) = match first.as_str() { + "Self" => return Err(()), + "crate" => { + if path.leading_colon.is_some() { + return Err(()); + } + (ModuleRelative::No, CratePath::CrateRelative) + } + "self" | "super" => { + if path.leading_colon.is_some() { + return Err(()); + } + (ModuleRelative::Yes, CratePath::ModuleRelative) + } + _ => (ModuleRelative::No, CratePath::External), + }; + + for seg in path.segments.iter().skip(1) { + let ident = seg.ident.to_string(); + match ident.as_str() { + "Self" | "crate" | "self" => return Err(()), + "super" => match prev_segment { + ModuleRelative::Yes => {} + ModuleRelative::No => return Err(()), + }, + _ => { + prev_segment = ModuleRelative::No; + } + } + } + Ok(path_type) } impl Ctx { @@ -42,24 +95,40 @@ pub(crate) fn try_from_derive_input(ast: DeriveInput) -> Result { for attr in &ast.attrs { if let Meta::List(ref meta_list) = attr.meta { - if meta_list.path.is_ident("zerocopy") { + if path_is_ident(&meta_list.path, "zerocopy") { attr.parse_nested_meta(|meta| { - if meta.path.is_ident("crate") { - let expr = meta.value().and_then(|value| value.parse()); + if path_is_ident(&meta.path, "crate") { + let expr = meta.value().and_then(ParseBuffer::parse); if let Ok(Expr::Lit(ExprLit { lit: Lit::Str(lit), .. })) = expr { - if let Ok(path_lit) = lit.parse::() { - path = parse_quote!(::#path_lit); - return Ok(()); + if let Ok(mut path_lit) = lit.parse_with(Path::parse_mod_style) { + if let Ok(crate_path) = validate_crate_path(&path_lit) { + // If not expressly relative, absolutize. + if path_lit.leading_colon.is_none() && crate_path == CratePath::External { + path_lit.leading_colon = Some(PathSep::default()); + } + path = path_lit; + return Ok(()); + } + + return Err(Error::new( + lit.span(), + "`crate` attribute requires a valid module path", + )); } + + return Err(Error::new( + lit.span(), + "`crate` attribute requires a path as the value", + )); } return Err(Error::new( - Span::call_site(), + meta.path.span(), "`crate` attribute requires a path as the value", )); } - if meta.path.is_ident("on_error") { + if path_is_ident(&meta.path, "on_error") { on_error_span = Some(meta.path.span()); let value = meta.value()?; let s: LitStr = value.parse()?; @@ -86,7 +155,8 @@ pub(crate) fn try_from_derive_input(ast: DeriveInput) -> Result { } } - Ok(Self { ast, zerocopy_crate: path, skip_on_error, on_error_span }) + let invariant_span = crate::invariant::validate(&ast)?; + Ok(Self { ast, zerocopy_crate: path, skip_on_error, on_error_span, invariant_span }) } pub(crate) fn with_input(&self, input: &DeriveInput) -> Self { @@ -95,6 +165,7 @@ pub(crate) fn with_input(&self, input: &DeriveInput) -> Self { zerocopy_crate: self.zerocopy_crate.clone(), skip_on_error: self.skip_on_error, on_error_span: self.on_error_span, + invariant_span: self.invariant_span, } } @@ -108,15 +179,41 @@ pub(crate) fn core_path(&self) -> TokenStream { quote!(#zerocopy_crate::util::macro_util::core_reexport) } + /// Choose an implementation name absent from the caller's tokens. This is + /// best-effort; names introduced by nested macros are not considered. + /// Unlike local variables, type parameters are not hidden by mixed-site + /// hygiene. + pub(crate) fn fresh_ident(&self, name: &str) -> Ident { + let idents = crate::invariant::idents(self.ast.to_token_stream()); + let mut name = name.to_owned(); + while idents.iter().any(|ident| ident.to_string().trim_start_matches("r#") == name) { + name.push('_'); + } + Ident::new(&name, Span::mixed_site()) + } + + /// Caller-authored invariant expressions must inherit the caller's lint + /// policy, rather than the blanket allowances for generated glue. + pub(crate) fn const_block( + &self, + items: impl IntoIterator>, + ) -> TokenStream { + if self.invariant_span.is_none() { + const_block(items) + } else { + let items = items.into_iter().flatten(); + quote! { const _: () = { #(#items)* }; } + } + } + pub(crate) fn cfg_compile_error(&self) -> TokenStream { // By checking both during the compilation of the proc macro *and* in - // the generated code, we ensure that `--cfg - // zerocopy_unstable_linux` need only be passed *either* when + // the generated code, we ensure that each cfg need only be passed when // compiling this crate *or* when compiling the user's crate. The former // is preferable, but in some situations (such as when cross-compiling // using `cargo build --target`), it doesn't get propagated to this // crate's build by default. - if cfg!(zerocopy_unstable_linux) { + let on_error = if cfg!(zerocopy_unstable_linux) { quote!() } else if let Some(span) = self.on_error_span { let core = self.core_path(); @@ -131,7 +228,24 @@ pub(crate) fn cfg_compile_error(&self) -> TokenStream { } } else { quote!() - } + }; + let invariant = if cfg!(zerocopy_unstable_ptr) { + quote!() + } else if let Some(span) = self.invariant_span { + let core = self.core_path(); + let error_message = + "`invariant` is experimental; pass '--cfg zerocopy_unstable_ptr' to enable"; + quote::quote_spanned! {span=> + #[allow(unused_attributes, unexpected_cfgs)] + const _: () = { + #[cfg(not(zerocopy_unstable_ptr))] + #core::compile_error!(#error_message); + }; + } + } else { + quote!() + }; + quote!(#on_error #invariant) } pub(crate) fn error_or_skip(&self, error: E) -> Result { @@ -151,7 +265,7 @@ pub(crate) trait DataExt { /// FIXME: Extracting field names for enums doesn't really make sense. Types /// makes sense because we don't care about where they live - we just care /// about transitive ownership. But for field names, we'd only use them when - /// generating is_bit_valid, which cares about where they live. + /// generating is_safe, which cares about where they live. fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)>; fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)>; @@ -255,6 +369,18 @@ pub(crate) fn to_ident_str(t: &impl ToString) -> String { } } +/// Does `path` consist solely of the identifier `expected`? +/// +/// Unlike [`Path::is_ident`], this treats a raw identifier and its ordinary +/// spelling as the same identifier. Rust applies that same normalization when +/// interpreting attribute names and arguments. +pub(crate) fn path_is_ident(path: &Path, expected: &str) -> bool { + match path.get_ident() { + Some(ident) => to_ident_str(ident) == expected, + None => false, + } +} + /// This enum describes what kind of padding check needs to be generated for the /// associated impl. pub(crate) enum PaddingCheck { @@ -298,22 +424,46 @@ pub(crate) fn validator_macro_context(&self) -> Option<&TokenStream> { } } +#[derive(Copy, Clone)] +pub(crate) enum Client { + ProjectDerive, + TryFromBytesDerive, +} + +impl ToTokens for Client { + fn to_tokens(&self, tokens: &mut TokenStream) { + let s = match self { + Client::ProjectDerive => "ProjectDerive", + Client::TryFromBytesDerive => "TryFromBytesDerive", + }; + let ident = Ident::new(s, Span::call_site()); + tokens.extend(quote!(project_clients::#ident)); + } +} + +impl Client { + pub(crate) fn crate_path(&self, ctx: &Ctx) -> Path { + let zerocopy_crate = &ctx.zerocopy_crate; + parse_quote!(#zerocopy_crate::#self) + } +} + +/// Bundles the trait parameters for `HasField` and `ProjectField`. +#[derive(Clone)] +pub(crate) struct FieldProjection { + pub(crate) variant_id: Box, + pub(crate) field: Box, + pub(crate) field_id: Box, +} + #[derive(Clone)] pub(crate) enum Trait { KnownLayout, - HasTag, - HasField { - variant_id: Box, - field: Box, - field_id: Box, - }, - ProjectField { - variant_id: Box, - field: Box, - field_id: Box, - invariants: Box, - }, + HasTag { client: Client }, + HasField { client: Client, projection: FieldProjection }, + ProjectField { client: Client, projection: FieldProjection, invariants: Box }, Immutable, + Project, TryFromBytes, FromZeros, FromBytes, @@ -340,8 +490,9 @@ fn to_tokens(&self, tokens: &mut TokenStream) { Trait::HasField { .. } => "HasField", Trait::ProjectField { .. } => "ProjectField", Trait::KnownLayout => "KnownLayout", - Trait::HasTag => "HasTag", + Trait::HasTag { .. } => "HasTag", Trait::Immutable => "Immutable", + Trait::Project => "Project", Trait::TryFromBytes => "TryFromBytes", Trait::FromZeros => "FromZeros", Trait::FromBytes => "FromBytes", @@ -354,15 +505,18 @@ fn to_tokens(&self, tokens: &mut TokenStream) { }; let ident = Ident::new(s, Span::call_site()); let arguments: Option = match self { - Trait::HasField { variant_id, field, field_id } => { - Some(parse_quote!(<#field, #variant_id, #field_id>)) + Trait::HasTag { client } => Some(parse_quote!(<#client>)), + Trait::HasField { client, projection } => { + let FieldProjection { variant_id, field, field_id } = projection; + Some(parse_quote!(<#client, #field, #variant_id, #field_id>)) } - Trait::ProjectField { variant_id, field, field_id, invariants } => { - Some(parse_quote!(<#field, #invariants, #variant_id, #field_id>)) + Trait::ProjectField { client, projection, invariants } => { + let FieldProjection { variant_id, field, field_id } = projection; + Some(parse_quote!(<#client, #field, #invariants, #variant_id, #field_id>)) } Trait::KnownLayout - | Trait::HasTag | Trait::Immutable + | Trait::Project | Trait::TryFromBytes | Trait::FromZeros | Trait::FromBytes @@ -383,6 +537,28 @@ pub(crate) fn crate_path(&self, ctx: &Ctx) -> Path { let core = ctx.core_path(); match self { Self::Sized => parse_quote!(#core::marker::#self), + Self::HasTag { client } => { + let client = client.crate_path(ctx); + parse_quote!(#zerocopy_crate::HasTag<#client>) + } + Self::HasField { client, projection } => { + let FieldProjection { variant_id, field, field_id } = projection; + let client = client.crate_path(ctx); + parse_quote!(#zerocopy_crate::HasField<#client, #field, #variant_id, #field_id>) + } + Self::ProjectField { client, projection, invariants } => { + let FieldProjection { variant_id, field, field_id } = projection; + let client = client.crate_path(ctx); + parse_quote!( + #zerocopy_crate::ProjectField< + #client, + #field, + #invariants, + #variant_id, + #field_id + > + ) + } _ => parse_quote!(#zerocopy_crate::#self), } } @@ -704,7 +880,12 @@ fn only_derive_is_allowed_to_implement_this_trait() {} let outer_extras = self.outer_extras.filter(|e| !e.is_empty()); let cfg_compile_error = self.ctx.cfg_compile_error(); - const_block([Some(cfg_compile_error), Some(impl_tokens), outer_extras]) + let items = [Some(cfg_compile_error), Some(impl_tokens), outer_extras]; + if matches!(self.trt, Trait::TryFromBytes) { + self.ctx.const_block(items) + } else { + const_block(items) + } } } @@ -730,8 +911,7 @@ fn then_some(self, t: T) -> Option { } } -pub(crate) fn const_block(items: impl IntoIterator>) -> TokenStream { - let items = items.into_iter().flatten(); +pub(crate) fn allow_generated_code() -> TokenStream { quote! { #[allow( // FIXME(#553): Add a test that generates a warning when @@ -748,6 +928,14 @@ pub(crate) fn const_block(items: impl IntoIterator>) non_ascii_idents, clippy::missing_inline_in_public_items, )] + } +} + +pub(crate) fn const_block(items: impl IntoIterator>) -> TokenStream { + let items = items.into_iter().flatten(); + let allow = allow_generated_code(); + quote! { + #allow #[deny(ambiguous_associated_items)] // While there are not currently any warnings that this suppresses // (that we're aware of), it's good future-proofing hygiene. @@ -778,7 +966,8 @@ pub(crate) fn generate_tag_enum(ctx: &Ctx, repr: &EnumRepr, data: &DataEnum) -> quote! { #repr - #[allow(dead_code)] + #[allow(dead_code, clippy::derive_partial_eq_without_eq)] + #[derive(Copy, Clone, PartialEq)] pub enum ___ZerocopyTag { #(#variants,)* } @@ -824,18 +1013,40 @@ pub(crate) mod testutil { pub(crate) fn check_hygiene(ts: TokenStream) { struct AmbiguousItemVisitor; + fn panic_ambiguous_path(path: &impl quote::ToTokens) -> ! { + panic!( + "Found ambiguous path `{}` in generated output. \ + All associated item access must be fully qualified (e.g., `::Item`) \ + to prevent hygiene issues.", + quote::quote!(#path) + ); + } + impl<'ast> Visit<'ast> for AmbiguousItemVisitor { + fn visit_expr_path(&mut self, i: &'ast syn::ExprPath) { + if let Some(qself) = &i.qself { + if qself.position == 0 { + panic_ambiguous_path(i); + } + } + visit::visit_expr_path(self, i); + } + fn visit_path(&mut self, i: &'ast syn::Path) { if i.segments.len() > 1 && i.segments.first().unwrap().ident == "Self" { - panic!( - "Found ambiguous path `{}` in generated output. \ - All associated item access must be fully qualified (e.g., `::Item`) \ - to prevent hygiene issues.", - quote::quote!(#i) - ); + panic_ambiguous_path(i); } visit::visit_path(self, i); } + + fn visit_type_path(&mut self, i: &'ast syn::TypePath) { + if let Some(qself) = &i.qself { + if qself.position == 0 { + panic_ambiguous_path(i); + } + } + visit::visit_type_path(self, i); + } } let file = syn::parse2::(ts).expect("failed to parse generated output as File"); @@ -860,4 +1071,117 @@ fn foo() { } }); } + + #[test] + #[should_panic(expected = "Found ambiguous path `< T > :: Ambiguous`")] + fn test_check_hygiene_type_relative_expr_failure() { + check_hygiene(quote::quote! { + fn foo() { + let _ = ::Ambiguous; + } + }); + } + + #[test] + #[should_panic(expected = "Found ambiguous path `< T > :: Ambiguous`")] + fn test_check_hygiene_type_relative_type_failure() { + check_hygiene(quote::quote! { + fn foo() { + let _: ::Ambiguous; + } + }); + } + + #[test] + fn test_validate_crate_path() { + use syn::parse_str; + + let valid = [ + "zerocopy", + "crate", + "crate::foo::bar", + "self", + "self::foo", + "self::super::foo", + "super", + "super::foo", + "super::super::foo", + "super::super::super", + "foo::bar::baz", + "::foo::bar", + ]; + + for path_str in valid { + let path = parse_str::(path_str).unwrap(); + assert!( + super::validate_crate_path(&path).is_ok(), + "expected valid path for `{}`", + path_str + ); + } + + let invalid = [ + "::crate::foo", + "::self::foo", + "::super::foo", + "Self", + "Self::foo", + "foo::Self::bar", + "foo::crate::bar", + "foo::super::bar", + "foo::self", + "super::foo::super", + "super::crate::foo", + "crate::super::foo", + "self::self::foo", + ]; + + for path_str in invalid { + let path = parse_str::(path_str).unwrap(); + assert!( + super::validate_crate_path(&path).is_err(), + "expected invalid path for `{}`", + path_str + ); + } + } + + #[test] + fn test_path_is_ident() { + use syn::parse_str; + + for (expected, ordinary, raw) in [ + ("doc", "doc", "r#doc"), + ("repr", "repr", "r#repr"), + ("zerocopy", "zerocopy", "r#zerocopy"), + ("on_error", "on_error", "r#on_error"), + ] { + let ordinary = parse_str::(ordinary).unwrap(); + let raw = parse_str::(raw).unwrap(); + assert!(super::path_is_ident(&ordinary, expected)); + assert!(super::path_is_ident(&raw, expected)); + } + + // `r#crate` is forbidden by Rust's raw identifier grammar. + let crate_path = parse_str::("crate").unwrap(); + assert!(super::path_is_ident(&crate_path, "crate")); + + for path in ["::zerocopy", "module::zerocopy"] { + let path = parse_str::(path).unwrap(); + assert!(!super::path_is_ident(&path, "zerocopy")); + } + } + + #[test] + fn test_raw_zerocopy_attributes() { + let ast: syn::DeriveInput = syn::parse_quote! { + #[r#zerocopy(crate = "renamed", r#on_error = "skip")] + struct Foo; + }; + + let ctx = super::Ctx::try_from_derive_input(ast).unwrap(); + let path = &ctx.zerocopy_crate; + assert_eq!(quote::quote!(#path).to_string(), ":: renamed"); + assert!(ctx.skip_on_error); + } } diff --git a/rust/zerocopy/README.md b/rust/zerocopy/README.md index 8443f6166008..5cdea261e45f 100644 --- a/rust/zerocopy/README.md +++ b/rust/zerocopy/README.md @@ -1,13 +1,13 @@ # `zerocopy` -These source files come from the Rust `zerocopy` crate, version v0.8.54 -(released 2026-07-08), hosted in the +These source files come from the Rust `zerocopy` crate, version v0.8.62 +(released 2026-10-08), hosted in the repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only modified to tweak the SPDX license identifiers. For copyright details, please see: - https://github.com/google/zerocopy/blob/v0.8.54/zerocopy/README.md?plain=1 - https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD - https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE - https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT + https://github.com/google/zerocopy/blob/v0.8.62/zerocopy/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.62/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.62/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.62/LICENSE-MIT diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 index f68bad612695..12ab2e3c4b37 100644 --- a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 @@ -1,5 +1,4 @@ bench_as_bytes_dynamic_size: mov rax, rdi - lea rdx, [2*rsi + 5] - and rdx, -2 + lea rdx, [2*rsi + 4] ret diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca index c3b92a9a95fe..d661f093fea6 100644 --- a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca @@ -1,11 +1,11 @@ Iterations: 100 -Instructions: 400 -Total Cycles: 137 -Total uOps: 400 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 Dispatch Width: 4 -uOps Per Cycle: 2.92 -IPC: 2.92 +uOps Per Cycle: 2.88 +IPC: 2.88 Block RThroughput: 1.0 @@ -19,8 +19,7 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 mov rax, rdi - 1 1 0.50 lea rdx, [2*rsi + 5] - 1 1 0.33 and rdx, -2 + 1 1 0.50 lea rdx, [2*rsi + 4] 1 1 1.00 U ret @@ -37,11 +36,10 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 1.33 1.33 - 1.34 - - + - - 0.99 1.00 - 1.01 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 0.66 - 0.34 - - mov rax, rdi - - - 0.33 0.67 - - - - lea rdx, [2*rsi + 5] - - - 1.00 - - - - - and rdx, -2 + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - lea rdx, [2*rsi + 4] - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64 b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 index 831b2a075fec..0790ce20273d 100644 --- a/rust/zerocopy/benches/extend_vec_zeroed.x86-64 +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 @@ -41,8 +41,8 @@ bench_extend_vec_zeroed: .LBB6_6: mov r13, rsi lea rcx, [rax + rax] - cmp r15, rcx - cmova rcx, r15 + cmp rcx, r15 + cmovbe rcx, r15 cmp rcx, 5 mov r14d, 4 cmovae r14, rcx diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca index cfab1eea8f56..cf3c5a5ad5e3 100644 --- a/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca @@ -55,8 +55,8 @@ Instruction Info: 1 1 1.00 U ret 1 1 0.33 mov r13, rsi 1 1 0.50 lea rcx, [rax + rax] - 1 1 0.33 cmp r15, rcx - 3 3 1.00 cmova rcx, r15 + 1 1 0.33 cmp rcx, r15 + 3 3 1.00 cmovbe rcx, r15 1 1 0.33 cmp rcx, 5 1 1 0.33 mov r14d, 4 2 2 0.67 cmovae r14, rcx @@ -128,8 +128,8 @@ Resource pressure by instruction: - - - - - 1.00 - - ret - - 1.00 - - - - - mov r13, rsi - - 0.01 0.99 - - - - lea rcx, [rax + rax] - - - 0.99 0.01 - - - - cmp r15, rcx - - - 2.00 0.01 - 0.99 - - cmova rcx, r15 + - - 0.99 0.01 - - - - cmp rcx, r15 + - - 2.00 0.01 - 0.99 - - cmovbe rcx, r15 - - 0.01 0.99 - - - - cmp rcx, 5 - - 0.01 0.99 - - - - mov r14d, 4 - - 1.00 0.01 - 0.99 - - cmovae r14, rcx diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64 b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 index 9db87403cb55..b5426c3f4a3e 100644 --- a/rust/zerocopy/benches/insert_vec_zeroed.x86-64 +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 @@ -54,8 +54,8 @@ bench_insert_vec_zeroed: mov r15, rsi mov rbp, rdx lea rcx, [rax + rax] - cmp r12, rcx - cmova rcx, r12 + cmp rcx, r12 + cmovbe rcx, r12 cmp rcx, 5 mov r14d, 4 cmovae r14, rcx diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca index 665240667844..a6c17e236823 100644 --- a/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca @@ -68,8 +68,8 @@ Instruction Info: 1 1 0.33 mov r15, rsi 1 1 0.33 mov rbp, rdx 1 1 0.50 lea rcx, [rax + rax] - 1 1 0.33 cmp r12, rcx - 3 3 1.00 cmova rcx, r12 + 1 1 0.33 cmp rcx, r12 + 3 3 1.00 cmovbe rcx, r12 1 1 0.33 cmp rcx, 5 1 1 0.33 mov r14d, 4 2 2 0.67 cmovae r14, rcx @@ -159,8 +159,8 @@ Resource pressure by instruction: - - 0.49 0.51 - - - - mov r15, rsi - - 0.51 0.49 - - - - mov rbp, rdx - - 0.49 0.51 - - - - lea rcx, [rax + rax] - - - 0.49 0.50 - 0.01 - - cmp r12, rcx - - - 1.04 0.50 - 1.46 - - cmova rcx, r12 + - - 0.49 0.50 - 0.01 - - cmp rcx, r12 + - - 1.04 0.50 - 1.46 - - cmovbe rcx, r12 - - 0.49 0.49 - 0.02 - - cmp rcx, 5 - - 0.50 - - 0.50 - - mov r14d, 4 - - 0.50 0.51 - 0.99 - - cmovae r14, rcx diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64 b/rust/zerocopy/benches/new_vec_zeroed.x86-64 index b5c083aa0d36..edf421e3192c 100644 --- a/rust/zerocopy/benches/new_vec_zeroed.x86-64 +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64 @@ -29,8 +29,7 @@ bench_new_vec_zeroed: mov qword ptr [rax + 16], rsi ret .LBB5_5: - movabs rcx, -9223372036854775808 - mov qword ptr [rax], rcx + mov qword ptr [rax], -1 ret .LBB5_2: mov ecx, 2 diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca index b4fb4544ec39..f0e47d004ccc 100644 --- a/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca @@ -1,11 +1,11 @@ Iterations: 100 -Instructions: 3700 +Instructions: 3600 Total Cycles: 3486 -Total uOps: 4600 +Total uOps: 4500 Dispatch Width: 4 -uOps Per Cycle: 1.32 -IPC: 1.06 +uOps Per Cycle: 1.29 +IPC: 1.03 Block RThroughput: 12.0 @@ -47,8 +47,7 @@ Instruction Info: 1 1 1.00 * mov qword ptr [rax + 8], rcx 1 1 1.00 * mov qword ptr [rax + 16], rsi 1 1 1.00 U ret - 1 1 0.33 movabs rcx, -9223372036854775808 - 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax], -1 1 1 1.00 U ret 1 1 0.33 mov ecx, 2 1 1 1.00 * mov qword ptr [rax], rsi @@ -70,31 +69,31 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.99 6.99 12.00 10.02 8.00 9.00 + - - 5.99 6.01 12.00 11.00 8.00 9.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.01 0.98 - 0.01 - - mov rax, rdi - - - 0.98 0.02 - - - - movabs rcx, 1537228672809129301 - - - 0.02 0.98 - - - - cmp rsi, rcx + - - 0.97 0.02 - 0.01 - - mov rax, rdi + - - 0.02 0.98 - - - - movabs rcx, 1537228672809129301 + - - 0.97 0.03 - - - - cmp rsi, rcx - - - - - 1.00 - - ja .LBB5_5 - - - 0.98 - - 0.02 - - test rsi, rsi + - - 0.03 0.96 - 0.01 - - test rsi, rsi - - - - - 1.00 - - je .LBB5_2 - - - - 1.00 - - 1.00 push r15 - - - - 1.00 - 1.00 - push r14 - - - - 1.00 - - 1.00 push rbx - - - - 1.00 - - - - lea rcx, [rsi + rsi] - - - - 1.00 - - - - lea rbx, [rcx + 2*rcx] - - - 1.00 - - - - - mov r14, rax - - - 1.00 - - - - - mov r15, rsi + - - 0.99 0.01 - - - - lea rcx, [rsi + rsi] + - - 0.99 0.01 - - - - lea rbx, [rcx + 2*rcx] + - - 0.01 0.99 - - - - mov r14, rax + - - 0.01 0.01 - 0.98 - - mov r15, rsi - - - - 1.00 1.00 2.00 - call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] - - - - 0.01 - 0.99 - - mov esi, 2 - - - 0.01 0.99 - - - - mov rdi, rbx + - - - 0.98 - 0.02 - - mov esi, 2 + - - 0.96 0.02 - 0.02 - - mov rdi, rbx - - - - 1.00 1.00 - 2.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] - - - 0.01 - - 0.99 - - mov rsi, r15 - - - 0.99 0.01 - - - - mov rcx, rax - - - - 0.99 - 0.01 - - mov rax, r14 - - - 0.99 0.01 - - - - test rcx, rcx + - - 0.02 0.96 - 0.02 - - mov rsi, r15 + - - 0.02 0.02 - 0.96 - - mov rcx, rax + - - 0.96 0.02 - 0.02 - - mov rax, r14 + - - 0.02 0.02 - 0.96 - - test rcx, rcx - - - - - - - 1.00 pop rbx - - - - - - 1.00 - pop r14 - - - - - - - 1.00 pop r15 @@ -103,10 +102,9 @@ Resource pressure by instruction: - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi - - - - - 1.00 - - ret - - - 0.01 0.99 - - - - movabs rcx, -9223372036854775808 - - - - - 1.00 - - 1.00 mov qword ptr [rax], rcx + - - - - 1.00 - - 1.00 mov qword ptr [rax], -1 - - - - - 1.00 - - ret - - - 0.99 0.01 - - - - mov ecx, 2 + - - 0.02 0.98 - - - - mov ecx, 2 - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 index e844a4608fac..7f695f942e67 100644 --- a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 @@ -1,22 +1,21 @@ bench_ref_from_bytes_dynamic_padding: - test dil, 3 - jne .LBB5_3 - movabs rax, 9223372036854775804 - and rax, rsi - cmp rax, 9 - jb .LBB5_3 - add rax, -9 - movabs rcx, -6148914691236517205 - mul rcx - shr rdx - lea rax, [rdx + 2*rdx] - or rax, 3 - add rax, 9 - cmp rsi, rax - je .LBB5_4 -.LBB5_3: + cmp rsi, 12 + setae al + mov ecx, esi + or ecx, edi + test cl, 3 + sete cl + and cl, al + cmp cl, 1 + je .LBB5_2 xor edi, edi - mov rdx, rsi -.LBB5_4: + mov rax, rdi + ret +.LBB5_2: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + shr rdx mov rax, rdi ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca index 423ed38ba28d..5eb4fd1a113d 100644 --- a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca @@ -1,11 +1,11 @@ Iterations: 100 Instructions: 1900 -Total Cycles: 645 +Total Cycles: 641 Total uOps: 2000 Dispatch Width: 4 -uOps Per Cycle: 3.10 -IPC: 2.95 +uOps Per Cycle: 3.12 +IPC: 2.96 Block RThroughput: 5.0 @@ -18,23 +18,23 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 test dil, 3 - 1 1 1.00 jne .LBB5_3 - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rax, rsi - 1 1 0.33 cmp rax, 9 - 1 1 1.00 jb .LBB5_3 - 1 1 0.33 add rax, -9 + 1 1 0.33 cmp rsi, 12 + 1 1 0.50 setae al + 1 1 0.33 mov ecx, esi + 1 1 0.33 or ecx, edi + 1 1 0.33 test cl, 3 + 1 1 0.50 sete cl + 1 1 0.33 and cl, al + 1 1 0.33 cmp cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi 2 4 1.00 mul rcx 1 1 0.50 shr rdx - 1 1 0.50 lea rax, [rdx + 2*rdx] - 1 1 0.33 or rax, 3 - 1 1 0.33 add rax, 9 - 1 1 0.33 cmp rsi, rax - 1 1 1.00 je .LBB5_4 - 1 0 0.25 xor edi, edi - 1 1 0.33 mov rdx, rsi 1 1 0.33 mov rax, rdi 1 1 1.00 U ret @@ -52,26 +52,26 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.32 6.33 - 6.35 - - + - - 6.33 6.33 - 6.34 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.64 0.35 - 0.01 - - test dil, 3 - - - - - - 1.00 - - jne .LBB5_3 - - - 0.34 0.65 - 0.01 - - movabs rax, 9223372036854775804 - - - 0.35 0.65 - - - - and rax, rsi - - - 0.33 0.34 - 0.33 - - cmp rax, 9 - - - - - - 1.00 - - jb .LBB5_3 - - - 0.35 - - 0.65 - - add rax, -9 - - - 0.97 0.01 - 0.02 - - movabs rcx, -6148914691236517205 - - - 1.00 1.00 - - - - mul rcx - - - 0.99 - - 0.01 - - shr rdx - - - 0.33 0.67 - - - - lea rax, [rdx + 2*rdx] - - - 0.34 0.66 - - - - or rax, 3 - - - 0.33 0.66 - 0.01 - - add rax, 9 - - - 0.01 0.99 - - - - cmp rsi, rax - - - - - - 1.00 - - je .LBB5_4 + - - 0.99 - - 0.01 - - cmp rsi, 12 + - - 1.00 - - - - - setae al + - - - 0.34 - 0.66 - - mov ecx, esi + - - - 0.99 - 0.01 - - or ecx, edi + - - - 1.00 - - - - test cl, 3 + - - 0.67 - - 0.33 - - sete cl + - - - 1.00 - - - - and cl, al + - - - 1.00 - - - - cmp cl, 1 + - - - - - 1.00 - - je .LBB5_2 - - - - - - - - xor edi, edi - - - 0.32 0.01 - 0.67 - - mov rdx, rsi - - - 0.02 0.34 - 0.64 - - mov rax, rdi + - - 0.66 - - 0.34 - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.35 0.01 - 0.64 - - add rsi, -9 + - - 0.65 0.33 - 0.02 - - movabs rcx, -6148914691236517205 + - - 0.01 0.33 - 0.66 - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 1.00 - - - - - shr rdx + - - - 0.33 - 0.67 - - mov rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 index cc905b76c06f..43cadf352783 100644 --- a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 @@ -1,20 +1,11 @@ bench_ref_from_bytes_dynamic_size: mov rdx, rsi - cmp rsi, 4 - setb al - or al, dil - test al, 1 - je .LBB5_2 + mov ecx, edx + or ecx, edi xor eax, eax - ret -.LBB5_2: - lea rcx, [rdx - 4] - mov rsi, rcx - and rsi, -2 - add rsi, 4 - shr rcx - xor eax, eax - cmp rdx, rsi - cmove rdx, rcx + sub rdx, 4 + cmovb rdi, rax + shr rdx + test cl, 1 cmove rax, rdi ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca index 68aea583e401..444f4fad4ff3 100644 --- a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1800 -Total Cycles: 704 -Total uOps: 2000 +Instructions: 1000 +Total Cycles: 373 +Total uOps: 1200 Dispatch Width: 4 -uOps Per Cycle: 2.84 -IPC: 2.56 -Block RThroughput: 5.0 +uOps Per Cycle: 3.22 +IPC: 2.68 +Block RThroughput: 3.0 Instruction Info: @@ -19,21 +19,13 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 mov rdx, rsi - 1 1 0.33 cmp rsi, 4 - 1 1 0.50 setb al - 1 1 0.33 or al, dil - 1 1 0.33 test al, 1 - 1 1 1.00 je .LBB5_2 + 1 1 0.33 mov ecx, edx + 1 1 0.33 or ecx, edi 1 0 0.25 xor eax, eax - 1 1 1.00 U ret - 1 1 0.50 lea rcx, [rdx - 4] - 1 1 0.33 mov rsi, rcx - 1 1 0.33 and rsi, -2 - 1 1 0.33 add rsi, 4 - 1 1 0.50 shr rcx - 1 0 0.25 xor eax, eax - 1 1 0.33 cmp rdx, rsi - 2 2 0.67 cmove rdx, rcx + 1 1 0.33 sub rdx, 4 + 2 2 0.67 cmovb rdi, rax + 1 1 0.50 shr rdx + 1 1 0.33 test cl, 1 2 2 0.67 cmove rax, rdi 1 1 1.00 U ret @@ -51,25 +43,17 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 5.97 5.98 - 6.05 - - + - - 3.66 3.67 - 3.67 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.97 0.01 - 0.02 - - mov rdx, rsi - - - 0.01 0.02 - 0.97 - - cmp rsi, 4 - - - 0.03 - - 0.97 - - setb al - - - 0.01 0.02 - 0.97 - - or al, dil - - - - 0.98 - 0.02 - - test al, 1 - - - - - - 1.00 - - je .LBB5_2 + - - 0.32 0.01 - 0.67 - - mov rdx, rsi + - - 0.33 0.35 - 0.32 - - mov ecx, edx + - - 0.01 0.34 - 0.65 - - or ecx, edi - - - - - - - - xor eax, eax - - - - - - 1.00 - - ret - - - 0.98 0.02 - - - - lea rcx, [rdx - 4] - - - 0.01 0.99 - - - - mov rsi, rcx - - - - 0.98 - 0.02 - - and rsi, -2 - - - 0.98 0.01 - 0.01 - - add rsi, 4 - - - 0.99 - - 0.01 - - shr rcx - - - - - - - - - xor eax, eax - - - 0.02 0.97 - 0.01 - - cmp rdx, rsi - - - 0.99 0.99 - 0.02 - - cmove rdx, rcx - - - 0.98 0.99 - 0.03 - - cmove rax, rdi + - - 0.67 0.33 - - - - sub rdx, 4 + - - 0.99 0.98 - 0.03 - - cmovb rdi, rax + - - 0.03 - - 0.97 - - shr rdx + - - 0.32 0.67 - 0.01 - - test cl, 1 + - - 0.99 0.99 - 0.02 - - cmove rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 index 2c8da68c8b5b..0e16b103344f 100644 --- a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 @@ -1,8 +1,7 @@ bench_ref_from_bytes_static_size: - mov ecx, edi - and ecx, 1 - xor rsi, 6 xor eax, eax - or rsi, rcx + test dil, 1 + cmovne rdi, rax + cmp rsi, 6 cmove rax, rdi ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca index 832697801ee2..f1e118abf09a 100644 --- a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca @@ -1,11 +1,11 @@ Iterations: 100 -Instructions: 700 -Total Cycles: 240 +Instructions: 600 +Total Cycles: 305 Total uOps: 800 Dispatch Width: 4 -uOps Per Cycle: 3.33 -IPC: 2.92 +uOps Per Cycle: 2.62 +IPC: 1.97 Block RThroughput: 2.0 @@ -18,11 +18,10 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 mov ecx, edi - 1 1 0.33 and ecx, 1 - 1 1 0.33 xor rsi, 6 1 0 0.25 xor eax, eax - 1 1 0.33 or rsi, rcx + 1 1 0.33 test dil, 1 + 2 2 0.67 cmovne rdi, rax + 1 1 0.33 cmp rsi, 6 2 2 0.67 cmove rax, rdi 1 1 1.00 U ret @@ -40,14 +39,13 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 2.33 2.33 - 2.34 - - + - - 1.95 2.28 - 2.77 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.01 0.98 - 0.01 - - mov ecx, edi - - - 0.02 0.66 - 0.32 - - and ecx, 1 - - - 0.33 0.66 - 0.01 - - xor rsi, 6 - - - - - - - - xor eax, eax - - - 0.98 0.02 - - - - or rsi, rcx - - - 0.99 0.01 - 1.00 - - cmove rax, rdi + - - 0.05 0.06 - 0.89 - - test dil, 1 + - - 0.95 0.94 - 0.11 - - cmovne rdi, rax + - - - 0.34 - 0.66 - - cmp rsi, 6 + - - 0.95 0.94 - 0.11 - - cmove rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 index a58592a24503..e7f7820a69d1 100644 --- a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 @@ -1,19 +1,19 @@ bench_ref_from_prefix_dynamic_padding: - xor edx, edx - mov eax, 0 test dil, 3 - je .LBB5_1 + sete al + cmp rsi, 12 + setae cl + test al, cl + jne .LBB5_2 + xor edx, edx + test dil, 3 + sete dl + xor edi, edi + mov rax, rdi ret -.LBB5_1: - movabs rax, 9223372036854775804 - and rsi, rax - cmp rsi, 9 - jae .LBB5_3 - mov edx, 1 - xor eax, eax - ret -.LBB5_3: - add rsi, -9 +.LBB5_2: + or rsi, 3 + add rsi, -12 movabs rcx, -6148914691236517205 mov rax, rsi mul rcx diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca index 62ea4babaf28..a3d68f37a8be 100644 --- a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1900 -Total Cycles: 608 -Total uOps: 2000 +Instructions: 2000 +Total Cycles: 640 +Total uOps: 2100 Dispatch Width: 4 -uOps Per Cycle: 3.29 +uOps Per Cycle: 3.28 IPC: 3.13 -Block RThroughput: 5.0 +Block RThroughput: 5.3 Instruction Info: @@ -18,19 +18,20 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 0 0.25 xor edx, edx - 1 1 0.33 mov eax, 0 1 1 0.33 test dil, 3 - 1 1 1.00 je .LBB5_1 + 1 1 0.50 sete al + 1 1 0.33 cmp rsi, 12 + 1 1 0.50 setae cl + 1 1 0.33 test al, cl + 1 1 1.00 jne .LBB5_2 + 1 0 0.25 xor edx, edx + 1 1 0.33 test dil, 3 + 1 1 0.50 sete dl + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rax, rdi 1 1 1.00 U ret - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rsi, rax - 1 1 0.33 cmp rsi, 9 - 1 1 1.00 jae .LBB5_3 - 1 1 0.33 mov edx, 1 - 1 0 0.25 xor eax, eax - 1 1 1.00 U ret - 1 1 0.33 add rsi, -9 + 1 1 0.33 or rsi, 3 + 1 1 0.33 add rsi, -12 1 1 0.33 movabs rcx, -6148914691236517205 1 1 0.33 mov rax, rsi 2 4 1.00 mul rcx @@ -52,26 +53,27 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.00 6.00 - 6.00 - - + - - 6.33 6.33 - 6.34 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.65 0.34 - 0.01 - - test dil, 3 + - - 0.99 - - 0.01 - - sete al + - - 0.01 0.33 - 0.66 - - cmp rsi, 12 + - - 0.34 - - 0.66 - - setae cl + - - 1.00 - - - - - test al, cl + - - - - - 1.00 - - jne .LBB5_2 - - - - - - - - xor edx, edx - - - 0.01 0.98 - 0.01 - - mov eax, 0 - - - 0.98 0.01 - 0.01 - - test dil, 3 - - - - - - 1.00 - - je .LBB5_1 + - - - 0.99 - 0.01 - - test dil, 3 + - - - - - 1.00 - - sete dl + - - - - - - - - xor edi, edi + - - - 1.00 - - - - mov rax, rdi - - - - - 1.00 - - ret - - - 0.01 0.99 - - - - movabs rax, 9223372036854775804 - - - - 1.00 - - - - and rsi, rax - - - - 1.00 - - - - cmp rsi, 9 - - - - - - 1.00 - - jae .LBB5_3 - - - 1.00 - - - - - mov edx, 1 - - - - - - - - - xor eax, eax - - - - - - 1.00 - - ret - - - 0.02 0.02 - 0.96 - - add rsi, -9 - - - 0.99 0.01 - - - - movabs rcx, -6148914691236517205 - - - 0.01 0.99 - - - - mov rax, rsi + - - 0.01 0.99 - - - - or rsi, 3 + - - 0.33 0.67 - - - - add rsi, -12 + - - 0.66 0.34 - - - - movabs rcx, -6148914691236517205 + - - 0.34 0.33 - 0.33 - - mov rax, rsi - - 1.00 1.00 - - - - mul rcx - - 1.00 - - - - - shr rdx - - - 0.98 - - 0.02 - - mov rax, rdi + - - - 0.34 - 0.66 - - mov rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 index fe6332c9100c..3f2e87467897 100644 --- a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 @@ -1,17 +1,10 @@ bench_ref_from_prefix_dynamic_size: - xor edx, edx - mov eax, 0 - test dil, 1 - jne .LBB5_4 - cmp rsi, 4 - jae .LBB5_3 - mov edx, 1 - xor eax, eax - ret -.LBB5_3: - add rsi, -4 - shr rsi mov rdx, rsi - mov rax, rdi -.LBB5_4: + xor eax, eax + sub rdx, 4 + mov rcx, rdi + cmovb rcx, rax + shr rdx + test dil, 1 + cmove rax, rcx ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca index 3900a5946138..7ee51e0b8a40 100644 --- a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1400 -Total Cycles: 405 -Total uOps: 1400 +Instructions: 900 +Total Cycles: 339 +Total uOps: 1100 Dispatch Width: 4 -uOps Per Cycle: 3.46 -IPC: 3.46 -Block RThroughput: 4.0 +uOps Per Cycle: 3.24 +IPC: 2.65 +Block RThroughput: 2.8 Instruction Info: @@ -18,19 +18,14 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 0 0.25 xor edx, edx - 1 1 0.33 mov eax, 0 - 1 1 0.33 test dil, 1 - 1 1 1.00 jne .LBB5_4 - 1 1 0.33 cmp rsi, 4 - 1 1 1.00 jae .LBB5_3 - 1 1 0.33 mov edx, 1 - 1 0 0.25 xor eax, eax - 1 1 1.00 U ret - 1 1 0.33 add rsi, -4 - 1 1 0.50 shr rsi 1 1 0.33 mov rdx, rsi - 1 1 0.33 mov rax, rdi + 1 0 0.25 xor eax, eax + 1 1 0.33 sub rdx, 4 + 1 1 0.33 mov rcx, rdi + 2 2 0.67 cmovb rcx, rax + 1 1 0.50 shr rdx + 1 1 0.33 test dil, 1 + 2 2 0.67 cmove rax, rcx 1 1 1.00 U ret @@ -47,21 +42,16 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 3.99 3.99 - 4.02 - - + - - 3.33 3.33 - 3.34 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - - - - - - xor edx, edx - - - 0.01 0.98 - 0.01 - - mov eax, 0 - - - 0.98 0.02 - - - - test dil, 1 - - - - - - 1.00 - - jne .LBB5_4 - - - 0.02 0.98 - - - - cmp rsi, 4 - - - - - - 1.00 - - jae .LBB5_3 - - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - 0.33 0.66 - 0.01 - - mov rdx, rsi - - - - - - - - xor eax, eax - - - - - - 1.00 - - ret - - - 0.01 0.99 - - - - add rsi, -4 - - - 1.00 - - - - - shr rsi - - - - 1.00 - - - - mov rdx, rsi - - - 0.99 0.01 - - - - mov rax, rdi + - - 0.01 - - 0.99 - - sub rdx, 4 + - - 0.66 0.34 - - - - mov rcx, rdi + - - 1.00 1.00 - - - - cmovb rcx, rax + - - 0.33 - - 0.67 - - shr rdx + - - - 0.33 - 0.67 - - test dil, 1 + - - 1.00 1.00 - - - - cmove rax, rcx - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 index 3e05f6023f38..68f21fe8018a 100644 --- a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 @@ -1,23 +1,22 @@ bench_ref_from_suffix_dynamic_padding: lea eax, [rsi + rdi] test al, 3 - jne .LBB5_1 - movabs rax, 9223372036854775804 - and rax, rsi - cmp rax, 9 - jae .LBB5_3 -.LBB5_1: - xor eax, eax + setne al + cmp rsi, 12 + setb cl + or cl, al + je .LBB5_2 + xor edi, edi + mov rax, rdi ret -.LBB5_3: - add rax, -9 +.LBB5_2: + mov rax, rsi + or rax, 3 + add rax, -12 movabs rcx, -6148914691236517205 mul rcx shr rdx - lea rax, [rdx + 2*rdx] - sub rsi, rax - or rax, -4 - add rsi, rdi - add rax, rsi - add rax, -8 + and esi, 3 + add rdi, rsi + mov rax, rdi ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca index 73599d5b6aab..c31e098aed4f 100644 --- a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca @@ -1,11 +1,11 @@ Iterations: 100 Instructions: 2000 -Total Cycles: 682 +Total Cycles: 673 Total uOps: 2100 Dispatch Width: 4 -uOps Per Cycle: 3.08 -IPC: 2.93 +uOps Per Cycle: 3.12 +IPC: 2.97 Block RThroughput: 5.3 @@ -20,23 +20,23 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.50 lea eax, [rsi + rdi] 1 1 0.33 test al, 3 - 1 1 1.00 jne .LBB5_1 - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rax, rsi - 1 1 0.33 cmp rax, 9 - 1 1 1.00 jae .LBB5_3 - 1 0 0.25 xor eax, eax + 1 1 0.50 setne al + 1 1 0.33 cmp rsi, 12 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rax, rdi 1 1 1.00 U ret - 1 1 0.33 add rax, -9 + 1 1 0.33 mov rax, rsi + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, -12 1 1 0.33 movabs rcx, -6148914691236517205 2 4 1.00 mul rcx 1 1 0.50 shr rdx - 1 1 0.50 lea rax, [rdx + 2*rdx] - 1 1 0.33 sub rsi, rax - 1 1 0.33 or rax, -4 - 1 1 0.33 add rsi, rdi - 1 1 0.33 add rax, rsi - 1 1 0.33 add rax, -8 + 1 1 0.33 and esi, 3 + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rax, rdi 1 1 1.00 U ret @@ -53,27 +53,27 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.65 6.67 - 6.68 - - + - - 6.67 6.66 - 6.67 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.90 0.10 - - - - lea eax, [rsi + rdi] - - - 0.93 - - 0.07 - - test al, 3 - - - - - - 1.00 - - jne .LBB5_1 - - - 0.51 0.47 - 0.02 - - movabs rax, 9223372036854775804 - - - - - - 1.00 - - and rax, rsi - - - - 0.09 - 0.91 - - cmp rax, 9 - - - - - - 1.00 - - jae .LBB5_3 - - - - - - - - - xor eax, eax + - - 0.35 0.65 - - - - lea eax, [rsi + rdi] + - - 0.01 0.99 - - - - test al, 3 + - - 1.00 - - - - - setne al + - - 0.97 - - 0.03 - - cmp rsi, 12 + - - 0.66 - - 0.34 - - setb cl + - - - 1.00 - - - - or cl, al + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor edi, edi + - - 0.01 0.01 - 0.98 - - mov rax, rdi - - - - - 1.00 - - ret - - - 0.43 0.47 - 0.10 - - add rax, -9 - - - 0.42 0.39 - 0.19 - - movabs rcx, -6148914691236517205 + - - 0.01 0.99 - - - - mov rax, rsi + - - - 0.34 - 0.66 - - or rax, 3 + - - - 1.00 - - - - add rax, -12 + - - 0.66 0.31 - 0.03 - - movabs rcx, -6148914691236517205 - - 1.00 1.00 - - - - mul rcx - - - 0.69 - - 0.31 - - shr rdx - - - 0.54 0.46 - - - - lea rax, [rdx + 2*rdx] - - - 0.07 0.91 - 0.02 - - sub rsi, rax - - - 0.91 0.05 - 0.04 - - or rax, -4 - - - 0.08 0.90 - 0.02 - - add rsi, rdi - - - 0.09 0.91 - - - - add rax, rsi - - - 0.08 0.92 - - - - add rax, -8 + - - 1.00 - - - - - shr rdx + - - 0.67 0.01 - 0.32 - - and esi, 3 + - - 0.32 0.01 - 0.67 - - add rdi, rsi + - - 0.01 0.35 - 0.64 - - mov rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 index c3d10b5fc685..7d11849dd5c5 100644 --- a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -1,27 +1,21 @@ bench_ref_from_suffix_with_elems_dynamic_padding: - movabs rax, 3074457345618258598 - cmp rdx, rax - ja .LBB5_1 - lea r8d, [rsi + rdi] - xor ecx, ecx - mov eax, 0 - test r8b, 3 - je .LBB5_3 - mov rdx, rcx - ret + movabs rcx, 3074457345618258598 + cmp rdx, rcx + ja .LBB5_3 + mov rax, rdi + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + add edi, esi + test dil, 3 + setne dil + sub rsi, rcx + setb cl + or cl, dil + je .LBB5_4 .LBB5_3: - lea rax, [rdx + 2*rdx] - or rax, 3 - add rax, 9 - sub rsi, rax - jae .LBB5_4 -.LBB5_1: xor eax, eax - mov edx, 1 ret .LBB5_4: - add rdi, rsi - mov rcx, rdx - mov rax, rdi - mov rdx, rcx + add rax, rsi ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca index 92e6280bb4cc..d7850d93bc10 100644 --- a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2300 -Total Cycles: 706 -Total uOps: 2300 +Instructions: 1800 +Total Cycles: 572 +Total uOps: 1800 Dispatch Width: 4 -uOps Per Cycle: 3.26 -IPC: 3.26 -Block RThroughput: 6.0 +uOps Per Cycle: 3.15 +IPC: 3.15 +Block RThroughput: 4.5 Instruction Info: @@ -18,28 +18,23 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 movabs rax, 3074457345618258598 - 1 1 0.33 cmp rdx, rax - 1 1 1.00 ja .LBB5_1 - 1 1 0.50 lea r8d, [rsi + rdi] - 1 0 0.25 xor ecx, ecx - 1 1 0.33 mov eax, 0 - 1 1 0.33 test r8b, 3 - 1 1 1.00 je .LBB5_3 - 1 1 0.33 mov rdx, rcx - 1 1 1.00 U ret - 1 1 0.50 lea rax, [rdx + 2*rdx] - 1 1 0.33 or rax, 3 - 1 1 0.33 add rax, 9 - 1 1 0.33 sub rsi, rax - 1 1 1.00 jae .LBB5_4 - 1 0 0.25 xor eax, eax - 1 1 0.33 mov edx, 1 - 1 1 1.00 U ret - 1 1 0.33 add rdi, rsi - 1 1 0.33 mov rcx, rdx + 1 1 0.33 movabs rcx, 3074457345618258598 + 1 1 0.33 cmp rdx, rcx + 1 1 1.00 ja .LBB5_3 1 1 0.33 mov rax, rdi - 1 1 0.33 mov rdx, rcx + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 add edi, esi + 1 1 0.33 test dil, 3 + 1 1 0.50 setne dil + 1 1 0.33 sub rsi, rcx + 1 1 0.50 setb cl + 1 1 0.33 or cl, dil + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, rsi 1 1 1.00 U ret @@ -56,30 +51,25 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.99 7.00 - 7.01 - - + - - 5.66 5.66 - 5.68 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 - - - 0.01 0.50 - 0.49 - - cmp rdx, rax - - - - - - 1.00 - - ja .LBB5_1 - - - - 1.00 - - - - lea r8d, [rsi + rdi] - - - - - - - - - xor ecx, ecx - - - 0.50 0.49 - 0.01 - - mov eax, 0 - - - 0.49 0.51 - - - - test r8b, 3 - - - - - - 1.00 - - je .LBB5_3 - - - 0.51 0.49 - - - - mov rdx, rcx - - - - - - 1.00 - - ret - - - 0.50 0.50 - - - - lea rax, [rdx + 2*rdx] - - - 1.00 - - - - - or rax, 3 - - - 1.00 - - - - - add rax, 9 - - - 0.99 0.01 - - - - sub rsi, rax - - - - - - 1.00 - - jae .LBB5_4 + - - 0.67 - - 0.33 - - movabs rcx, 3074457345618258598 + - - 0.34 - - 0.66 - - cmp rdx, rcx + - - - - - 1.00 - - ja .LBB5_3 + - - 0.33 0.01 - 0.66 - - mov rax, rdi + - - 0.33 0.67 - - - - lea rcx, [rdx + 2*rdx] + - - 0.01 0.99 - - - - or rcx, 3 + - - 0.01 0.99 - - - - add rcx, 9 + - - 0.99 - - 0.01 - - add edi, esi + - - 0.99 0.01 - - - - test dil, 3 + - - 0.99 - - 0.01 - - setne dil + - - - 1.00 - - - - sub rsi, rcx + - - 1.00 - - - - - setb cl + - - - 0.99 - 0.01 - - or cl, dil + - - - - - 1.00 - - je .LBB5_4 - - - - - - - - xor eax, eax - - - - 1.00 - - - - mov edx, 1 - - - - - 1.00 - - ret - - - 1.00 - - - - - add rdi, rsi - - - - 1.00 - - - - mov rcx, rdx - - - 0.99 0.01 - - - - mov rax, rdi - - - - 0.50 - 0.50 - - mov rdx, rcx + - - - 1.00 - - - - add rax, rsi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 index bdca57192455..7ee0278dbf68 100644 --- a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -1,23 +1,18 @@ bench_ref_from_suffix_with_elems_dynamic_size: - movabs rax, 4611686018427387901 - cmp rdx, rax - ja .LBB5_1 - lea r8d, [rsi + rdi] - xor ecx, ecx - mov eax, 0 - test r8b, 1 - jne .LBB5_5 - lea rax, [2*rdx + 4] - sub rsi, rax - jae .LBB5_4 -.LBB5_1: + movabs rcx, 4611686018427387901 + cmp rdx, rcx + ja .LBB5_3 + mov rax, rdi + lea rcx, [2*rdx + 4] + add edi, esi + sub rsi, rcx + setb cl + or cl, dil + test cl, 1 + je .LBB5_4 +.LBB5_3: xor eax, eax - mov edx, 1 ret .LBB5_4: - add rdi, rsi - mov rcx, rdx - mov rax, rdi -.LBB5_5: - mov rdx, rcx + add rax, rsi ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca index 6d9de0b3eb5c..882316de3d27 100644 --- a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1900 -Total Cycles: 571 -Total uOps: 1900 +Instructions: 1500 +Total Cycles: 472 +Total uOps: 1500 Dispatch Width: 4 -uOps Per Cycle: 3.33 -IPC: 3.33 -Block RThroughput: 5.0 +uOps Per Cycle: 3.18 +IPC: 3.18 +Block RThroughput: 4.0 Instruction Info: @@ -18,24 +18,20 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 movabs rax, 4611686018427387901 - 1 1 0.33 cmp rdx, rax - 1 1 1.00 ja .LBB5_1 - 1 1 0.50 lea r8d, [rsi + rdi] - 1 0 0.25 xor ecx, ecx - 1 1 0.33 mov eax, 0 - 1 1 0.33 test r8b, 1 - 1 1 1.00 jne .LBB5_5 - 1 1 0.50 lea rax, [2*rdx + 4] - 1 1 0.33 sub rsi, rax - 1 1 1.00 jae .LBB5_4 - 1 0 0.25 xor eax, eax - 1 1 0.33 mov edx, 1 - 1 1 1.00 U ret - 1 1 0.33 add rdi, rsi - 1 1 0.33 mov rcx, rdx + 1 1 0.33 movabs rcx, 4611686018427387901 + 1 1 0.33 cmp rdx, rcx + 1 1 1.00 ja .LBB5_3 1 1 0.33 mov rax, rdi - 1 1 0.33 mov rdx, rcx + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 add edi, esi + 1 1 0.33 sub rsi, rcx + 1 1 0.50 setb cl + 1 1 0.33 or cl, dil + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, rsi 1 1 1.00 U ret @@ -52,26 +48,22 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 5.66 5.66 - 5.68 - - + - - 4.66 4.66 - 4.68 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 - - - 0.01 0.99 - - - - cmp rdx, rax - - - - - - 1.00 - - ja .LBB5_1 - - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] - - - - - - - - - xor ecx, ecx - - - 0.33 0.33 - 0.34 - - mov eax, 0 - - - 0.33 0.34 - 0.33 - - test r8b, 1 - - - - - - 1.00 - - jne .LBB5_5 - - - 0.34 0.66 - - - - lea rax, [2*rdx + 4] - - - - 1.00 - - - - sub rsi, rax - - - - - - 1.00 - - jae .LBB5_4 + - - 0.99 - - 0.01 - - movabs rcx, 4611686018427387901 + - - 1.00 - - - - - cmp rdx, rcx + - - - - - 1.00 - - ja .LBB5_3 + - - 0.63 0.02 - 0.35 - - mov rax, rdi + - - 0.35 0.65 - - - - lea rcx, [2*rdx + 4] + - - 0.65 0.03 - 0.32 - - add edi, esi + - - 0.03 0.97 - - - - sub rsi, rcx + - - 1.00 - - - - - setb cl + - - - 1.00 - - - - or cl, dil + - - 0.01 0.99 - - - - test cl, 1 + - - - - - 1.00 - - je .LBB5_4 - - - - - - - - xor eax, eax - - - 1.00 - - - - - mov edx, 1 - - - - - 1.00 - - ret - - - - 1.00 - - - - add rdi, rsi - - - 1.00 - - - - - mov rcx, rdx - - - 0.32 0.68 - - - - mov rax, rdi - - - 0.68 0.32 - - - - mov rdx, rcx + - - - 1.00 - - - - add rax, rsi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..8d4e26394596 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_bytes_dynamic_padding<'a>( + source: &'a Ref<&[u8], format::LocoPacket>, +) -> &'a [u8] { + Ref::bytes(source) +} diff --git a/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..a44c1510fe02 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64 @@ -0,0 +1,4 @@ +bench_ref_type_bytes_dynamic_padding: + mov rax, qword ptr [rdi] + mov rdx, qword ptr [rdi + 8] + ret diff --git a/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..a7f8947c0413 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 107 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.80 +IPC: 2.80 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rdx, qword ptr [rdi + 8] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - 1.00 mov rax, qword ptr [rdi] + - - - - - - 1.00 - mov rdx, qword ptr [rdi + 8] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_bytes_dynamic_size.rs b/rust/zerocopy/benches/ref_type_bytes_dynamic_size.rs new file mode 100644 index 000000000000..ef7dcf83989d --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_dynamic_size.rs @@ -0,0 +1,17 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_bytes_dynamic_size<'a>(source: &'a Ref<&[u8], format::LocoPacket>) -> &'a [u8] { + Ref::bytes(source) +} diff --git a/rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..6ac3ad2f1fef --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64 @@ -0,0 +1,4 @@ +bench_ref_type_bytes_dynamic_size: + mov rax, qword ptr [rdi] + mov rdx, qword ptr [rdi + 8] + ret diff --git a/rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..a7f8947c0413 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 107 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.80 +IPC: 2.80 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rdx, qword ptr [rdi + 8] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - 1.00 mov rax, qword ptr [rdi] + - - - - - - 1.00 - mov rdx, qword ptr [rdi + 8] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.rs new file mode 100644 index 000000000000..e99b6704bd9b --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_bytes_mut_dynamic_padding<'a>( + source: &'a mut Ref<&mut [u8], format::LocoPacket>, +) -> &'a mut [u8] { + Ref::bytes_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64 new file mode 100644 index 000000000000..ebc6ee04941e --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64 @@ -0,0 +1,4 @@ +bench_ref_type_bytes_mut_dynamic_padding: + mov rax, qword ptr [rdi] + mov rdx, qword ptr [rdi + 8] + ret diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..a7f8947c0413 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_padding.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 107 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.80 +IPC: 2.80 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rdx, qword ptr [rdi + 8] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - 1.00 mov rax, qword ptr [rdi] + - - - - - - 1.00 - mov rdx, qword ptr [rdi + 8] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.rs b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.rs new file mode 100644 index 000000000000..badf9de19f06 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_bytes_mut_dynamic_size<'a>( + source: &'a mut Ref<&mut [u8], format::LocoPacket>, +) -> &'a mut [u8] { + Ref::bytes_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64 new file mode 100644 index 000000000000..5d97d01e51e2 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64 @@ -0,0 +1,4 @@ +bench_ref_type_bytes_mut_dynamic_size: + mov rax, qword ptr [rdi] + mov rdx, qword ptr [rdi + 8] + ret diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..a7f8947c0413 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_dynamic_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 107 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.80 +IPC: 2.80 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rdx, qword ptr [rdi + 8] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - 1.00 mov rax, qword ptr [rdi] + - - - - - - 1.00 - mov rdx, qword ptr [rdi + 8] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_static_size.rs b/rust/zerocopy/benches/ref_type_bytes_mut_static_size.rs new file mode 100644 index 000000000000..1a49c2492a41 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_static_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_bytes_mut_static_size<'a>( + source: &'a mut Ref<&mut [u8], format::LocoPacket>, +) -> &'a mut [u8] { + Ref::bytes_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64 b/rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64 new file mode 100644 index 000000000000..3411bd45e96a --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_ref_type_bytes_mut_static_size: + mov rax, qword ptr [rdi] + mov rdx, qword ptr [rdi + 8] + ret diff --git a/rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64.mca new file mode 100644 index 000000000000..a7f8947c0413 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_mut_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 107 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.80 +IPC: 2.80 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rdx, qword ptr [rdi + 8] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - 1.00 mov rax, qword ptr [rdi] + - - - - - - 1.00 - mov rdx, qword ptr [rdi + 8] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_bytes_static_size.rs b/rust/zerocopy/benches/ref_type_bytes_static_size.rs new file mode 100644 index 000000000000..b9c3fe5b7bb5 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_static_size.rs @@ -0,0 +1,17 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_bytes_static_size<'a>(source: &'a Ref<&[u8], format::LocoPacket>) -> &'a [u8] { + Ref::bytes(source) +} diff --git a/rust/zerocopy/benches/ref_type_bytes_static_size.x86-64 b/rust/zerocopy/benches/ref_type_bytes_static_size.x86-64 new file mode 100644 index 000000000000..10a024acaafd --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_ref_type_bytes_static_size: + mov rax, qword ptr [rdi] + mov rdx, qword ptr [rdi + 8] + ret diff --git a/rust/zerocopy/benches/ref_type_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..a7f8947c0413 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_bytes_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 107 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.80 +IPC: 2.80 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rdx, qword ptr [rdi + 8] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - 1.00 mov rax, qword ptr [rdi] + - - - - - - 1.00 - mov rdx, qword ptr [rdi + 8] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_deref_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_deref_dynamic_padding.rs new file mode 100644 index 000000000000..c0ebe56ccc75 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_dynamic_padding.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_deref_dynamic_padding<'a>( + source: &'a Ref<&[u8], format::LocoPacket>, +) -> &'a format::LocoPacket { + core::ops::Deref::deref(source) +} diff --git a/rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64 new file mode 100644 index 000000000000..765c5305d86d --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64 @@ -0,0 +1,29 @@ +bench_ref_type_deref_dynamic_padding: + mov rdx, qword ptr [rdi + 16] + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 8] + mov esi, 1 + movabs rdi, 3074457345618258599 + cmp rdx, rdi + jae .LBB8_3 + test al, 3 + jne .LBB8_2 + lea rdi, [rdx + 2*rdx] + or rdi, 3 + add rdi, 9 + cmp rcx, rdi + jne .LBB8_3 + ret +.LBB8_2: + xor esi, esi +.LBB8_3: + sub rsp, 24 + mov qword ptr [rsp], rsi + mov qword ptr [rsp + 8], rax + mov qword ptr [rsp + 16], rcx + lea rdi, [rip + .Lanon.HASH.7] + lea rcx, [rip + .Lanon.HASH.1] + lea r8, [rip + .Lanon.HASH.9] + mov rdx, rsp + mov esi, 58 + call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..bea2c8a20ea7 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 1812 +Total uOps: 2900 + +Dispatch Width: 4 +uOps Per Cycle: 1.60 +IPC: 1.43 +Block RThroughput: 7.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rdx, qword ptr [rdi + 16] + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rcx, qword ptr [rdi + 8] + 1 1 0.33 mov esi, 1 + 1 1 0.33 movabs rdi, 3074457345618258599 + 1 1 0.33 cmp rdx, rdi + 1 1 1.00 jae .LBB8_3 + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB8_2 + 1 1 0.50 lea rdi, [rdx + 2*rdx] + 1 1 0.33 or rdi, 3 + 1 1 0.33 add rdi, 9 + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 jne .LBB8_3 + 1 1 1.00 U ret + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rsp, 24 + 1 1 1.00 * mov qword ptr [rsp], rsi + 1 1 1.00 * mov qword ptr [rsp + 8], rax + 1 1 1.00 * mov qword ptr [rsp + 16], rcx + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.7] + 1 1 0.50 lea rcx, [rip + .Lanon.HASH.1] + 1 1 0.50 lea r8, [rip + .Lanon.HASH.9] + 1 1 0.33 mov rdx, rsp + 1 1 0.33 mov esi, 58 + 4 7 1.00 * call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.41 6.42 4.00 6.17 3.99 4.01 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.54 0.46 mov rdx, qword ptr [rdi + 16] + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - - 0.49 0.51 mov rcx, qword ptr [rdi + 8] + - - 0.52 0.47 - 0.01 - - mov esi, 1 + - - 0.47 0.53 - - - - movabs rdi, 3074457345618258599 + - - 0.47 0.53 - - - - cmp rdx, rdi + - - - - - 1.00 - - jae .LBB8_3 + - - 0.54 0.44 - 0.02 - - test al, 3 + - - - - - 1.00 - - jne .LBB8_2 + - - 0.44 0.56 - - - - lea rdi, [rdx + 2*rdx] + - - 0.48 0.52 - - - - or rdi, 3 + - - 0.48 0.48 - 0.04 - - add rdi, 9 + - - 0.51 0.39 - 0.10 - - cmp rcx, rdi + - - - - - 1.00 - - jne .LBB8_3 + - - - - - 1.00 - - ret + - - - - - - - - xor esi, esi + - - 0.53 0.47 - - - - sub rsp, 24 + - - - - 1.00 - 0.52 0.48 mov qword ptr [rsp], rsi + - - - - 1.00 - 0.48 0.52 mov qword ptr [rsp + 8], rax + - - - - 1.00 - 0.52 0.48 mov qword ptr [rsp + 16], rcx + - - 0.49 0.51 - - - - lea rdi, [rip + .Lanon.HASH.7] + - - 0.46 0.54 - - - - lea rcx, [rip + .Lanon.HASH.1] + - - 0.54 0.46 - - - - lea r8, [rip + .Lanon.HASH.9] + - - 0.04 0.06 - 0.90 - - mov rdx, rsp + - - 0.44 0.46 - 0.10 - - mov esi, 58 + - - - - 1.00 1.00 0.94 1.06 call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_deref_dynamic_size.rs b/rust/zerocopy/benches/ref_type_deref_dynamic_size.rs new file mode 100644 index 000000000000..adcf89c22508 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_dynamic_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_deref_dynamic_size<'a>( + source: &'a Ref<&[u8], format::LocoPacket>, +) -> &'a format::LocoPacket { + core::ops::Deref::deref(source) +} diff --git a/rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64 new file mode 100644 index 000000000000..60ca3b508a7e --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_deref_dynamic_size: + mov rdx, qword ptr [rdi + 16] + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 8] + mov esi, 1 + movabs rdi, 4611686018427387902 + cmp rdx, rdi + jae .LBB8_3 + test al, 1 + jne .LBB8_2 + lea rdi, [2*rdx + 4] + cmp rcx, rdi + jne .LBB8_3 + ret +.LBB8_2: + xor esi, esi +.LBB8_3: + sub rsp, 24 + mov qword ptr [rsp], rsi + mov qword ptr [rsp + 8], rax + mov qword ptr [rsp + 16], rcx + lea rdi, [rip + .Lanon.HASH.7] + lea rcx, [rip + .Lanon.HASH.1] + lea r8, [rip + .Lanon.HASH.9] + mov rdx, rsp + mov esi, 58 + call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..66a8f8575acd --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1647 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 1.64 +IPC: 1.46 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rdx, qword ptr [rdi + 16] + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rcx, qword ptr [rdi + 8] + 1 1 0.33 mov esi, 1 + 1 1 0.33 movabs rdi, 4611686018427387902 + 1 1 0.33 cmp rdx, rdi + 1 1 1.00 jae .LBB8_3 + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB8_2 + 1 1 0.50 lea rdi, [2*rdx + 4] + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 jne .LBB8_3 + 1 1 1.00 U ret + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rsp, 24 + 1 1 1.00 * mov qword ptr [rsp], rsi + 1 1 1.00 * mov qword ptr [rsp + 8], rax + 1 1 1.00 * mov qword ptr [rsp + 16], rcx + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.7] + 1 1 0.50 lea rcx, [rip + .Lanon.HASH.1] + 1 1 0.50 lea r8, [rip + .Lanon.HASH.9] + 1 1 0.33 mov rdx, rsp + 1 1 0.33 mov esi, 58 + 4 7 1.00 * call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.15 5.74 4.00 6.11 3.98 4.02 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.71 0.29 mov rdx, qword ptr [rdi + 16] + - - - - - - 0.72 0.28 mov rax, qword ptr [rdi] + - - - - - - 0.29 0.71 mov rcx, qword ptr [rdi + 8] + - - 0.83 0.16 - 0.01 - - mov esi, 1 + - - 0.16 0.84 - - - - movabs rdi, 4611686018427387902 + - - 0.80 0.04 - 0.16 - - cmp rdx, rdi + - - - - - 1.00 - - jae .LBB8_3 + - - 0.13 0.74 - 0.13 - - test al, 1 + - - - - - 1.00 - - jne .LBB8_2 + - - 0.10 0.90 - - - - lea rdi, [2*rdx + 4] + - - 0.09 0.91 - - - - cmp rcx, rdi + - - - - - 1.00 - - jne .LBB8_3 + - - - - - 1.00 - - ret + - - - - - - - - xor esi, esi + - - 0.29 0.13 - 0.58 - - sub rsp, 24 + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp], rsi + - - - - 1.00 - 0.28 0.72 mov qword ptr [rsp + 8], rax + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp + 16], rcx + - - 0.76 0.24 - - - - lea rdi, [rip + .Lanon.HASH.7] + - - 0.82 0.18 - - - - lea rcx, [rip + .Lanon.HASH.1] + - - 0.92 0.08 - - - - lea r8, [rip + .Lanon.HASH.9] + - - 0.09 0.68 - 0.23 - - mov rdx, rsp + - - 0.16 0.84 - - - - mov esi, 58 + - - - - 1.00 1.00 0.54 1.46 call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.rs b/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.rs new file mode 100644 index 000000000000..263f2c575379 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_deref_mut_dynamic_size<'a>( + source: &'a mut Ref<&mut [u8], format::LocoPacket>, +) -> &'a mut format::LocoPacket { + core::ops::DerefMut::deref_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64 new file mode 100644 index 000000000000..5ae0ae69efad --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_deref_mut_dynamic_size: + mov rdx, qword ptr [rdi + 16] + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 8] + mov esi, 1 + movabs rdi, 4611686018427387902 + cmp rdx, rdi + jae .LBB8_3 + test al, 1 + jne .LBB8_2 + lea rdi, [2*rdx + 4] + cmp rcx, rdi + jne .LBB8_3 + ret +.LBB8_2: + xor esi, esi +.LBB8_3: + sub rsp, 24 + mov qword ptr [rsp], rsi + mov qword ptr [rsp + 8], rax + mov qword ptr [rsp + 16], rcx + lea rdi, [rip + .Lanon.HASH.7] + lea rcx, [rip + .Lanon.HASH.1] + lea r8, [rip + .Lanon.HASH.9] + mov rdx, rsp + mov esi, 65 + call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ed3a2bc24701 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_mut_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1647 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 1.64 +IPC: 1.46 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rdx, qword ptr [rdi + 16] + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rcx, qword ptr [rdi + 8] + 1 1 0.33 mov esi, 1 + 1 1 0.33 movabs rdi, 4611686018427387902 + 1 1 0.33 cmp rdx, rdi + 1 1 1.00 jae .LBB8_3 + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB8_2 + 1 1 0.50 lea rdi, [2*rdx + 4] + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 jne .LBB8_3 + 1 1 1.00 U ret + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rsp, 24 + 1 1 1.00 * mov qword ptr [rsp], rsi + 1 1 1.00 * mov qword ptr [rsp + 8], rax + 1 1 1.00 * mov qword ptr [rsp + 16], rcx + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.7] + 1 1 0.50 lea rcx, [rip + .Lanon.HASH.1] + 1 1 0.50 lea r8, [rip + .Lanon.HASH.9] + 1 1 0.33 mov rdx, rsp + 1 1 0.33 mov esi, 65 + 4 7 1.00 * call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.15 5.74 4.00 6.11 3.98 4.02 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.71 0.29 mov rdx, qword ptr [rdi + 16] + - - - - - - 0.72 0.28 mov rax, qword ptr [rdi] + - - - - - - 0.29 0.71 mov rcx, qword ptr [rdi + 8] + - - 0.83 0.16 - 0.01 - - mov esi, 1 + - - 0.16 0.84 - - - - movabs rdi, 4611686018427387902 + - - 0.80 0.04 - 0.16 - - cmp rdx, rdi + - - - - - 1.00 - - jae .LBB8_3 + - - 0.13 0.74 - 0.13 - - test al, 1 + - - - - - 1.00 - - jne .LBB8_2 + - - 0.10 0.90 - - - - lea rdi, [2*rdx + 4] + - - 0.09 0.91 - - - - cmp rcx, rdi + - - - - - 1.00 - - jne .LBB8_3 + - - - - - 1.00 - - ret + - - - - - - - - xor esi, esi + - - 0.29 0.13 - 0.58 - - sub rsp, 24 + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp], rsi + - - - - 1.00 - 0.28 0.72 mov qword ptr [rsp + 8], rax + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp + 16], rcx + - - 0.76 0.24 - - - - lea rdi, [rip + .Lanon.HASH.7] + - - 0.82 0.18 - - - - lea rcx, [rip + .Lanon.HASH.1] + - - 0.92 0.08 - - - - lea r8, [rip + .Lanon.HASH.9] + - - 0.09 0.68 - 0.23 - - mov rdx, rsp + - - 0.16 0.84 - - - - mov esi, 65 + - - - - 1.00 1.00 0.54 1.46 call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_deref_mut_static_size.rs b/rust/zerocopy/benches/ref_type_deref_mut_static_size.rs new file mode 100644 index 000000000000..4f85a031c845 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_mut_static_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_deref_mut_static_size<'a>( + source: &'a mut Ref<&mut [u8], format::LocoPacket>, +) -> &'a mut format::LocoPacket { + core::ops::DerefMut::deref_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64 b/rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64 new file mode 100644 index 000000000000..259c39646bb3 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_ref_type_deref_mut_static_size: + mov rax, qword ptr [rdi] + ret diff --git a/rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64.mca new file mode 100644 index 000000000000..2de0e6f9791c --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_mut_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_deref_static_size.rs b/rust/zerocopy/benches/ref_type_deref_static_size.rs new file mode 100644 index 000000000000..60397dc28580 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_static_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_deref_static_size<'a>( + source: &'a Ref<&[u8], format::LocoPacket>, +) -> &'a format::LocoPacket { + core::ops::Deref::deref(source) +} diff --git a/rust/zerocopy/benches/ref_type_deref_static_size.x86-64 b/rust/zerocopy/benches/ref_type_deref_static_size.x86-64 new file mode 100644 index 000000000000..2d4368dd61df --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_ref_type_deref_static_size: + mov rax, qword ptr [rdi] + ret diff --git a/rust/zerocopy/benches/ref_type_deref_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_deref_static_size.x86-64.mca new file mode 100644 index 000000000000..2de0e6f9791c --- /dev/null +++ b/rust/zerocopy/benches/ref_type_deref_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..d11cae87a1df --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_bytes_dynamic_padding( + source: &[u8], +) -> Option> { + Ref::from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..9f2beef51011 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,25 @@ +bench_ref_type_from_bytes_dynamic_padding: + cmp rdx, 12 + setae al + mov ecx, edx + or ecx, esi + test cl, 3 + sete cl + and cl, al + cmp cl, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rdi], rsi + mov rax, rdi + ret +.LBB5_2: + lea rax, [rdx - 9] + movabs rcx, -6148914691236517205 + mov r8, rdx + mul rcx + shr rdx + mov qword ptr [rdi + 8], r8 + mov qword ptr [rdi + 16], rdx + mov qword ptr [rdi], rsi + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e239f2fbe9e7 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 909 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 2.64 +IPC: 2.53 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 12 + 1 1 0.50 setae al + 1 1 0.33 mov ecx, edx + 1 1 0.33 or ecx, esi + 1 1 0.33 test cl, 3 + 1 1 0.50 sete cl + 1 1 0.33 and cl, al + 1 1 0.33 cmp cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx - 9] + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov r8, rdx + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 1.00 * mov qword ptr [rdi + 8], r8 + 1 1 1.00 * mov qword ptr [rdi + 16], rdx + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.49 5.99 4.00 6.52 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - cmp rdx, 12 + - - 0.53 - - 0.47 - - setae al + - - 0.52 0.01 - 0.47 - - mov ecx, edx + - - - 0.99 - 0.01 - - or ecx, esi + - - - 0.51 - 0.49 - - test cl, 3 + - - 0.52 - - 0.48 - - sete cl + - - 0.47 0.48 - 0.05 - - and cl, al + - - 0.48 0.48 - 0.04 - - cmp cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rdi], rsi + - - 0.03 0.48 - 0.49 - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.97 0.03 - - - - lea rax, [rdx - 9] + - - 0.49 0.51 - - - - movabs rcx, -6148914691236517205 + - - 0.47 0.01 - 0.52 - - mov r8, rdx + - - 1.00 1.00 - - - - mul rcx + - - 1.00 - - - - - shr rdx + - - - - 1.00 - 1.00 - mov qword ptr [rdi + 8], r8 + - - - - 1.00 - - 1.00 mov qword ptr [rdi + 16], rdx + - - - - 1.00 - 1.00 - mov qword ptr [rdi], rsi + - - 0.01 0.50 - 0.49 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..cf0b278eaf09 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.rs @@ -0,0 +1,17 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_bytes_dynamic_size(source: &[u8]) -> Option> { + Ref::from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..c50dfc333bd6 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_ref_type_from_bytes_dynamic_size: + mov rax, rdi + cmp rdx, 4 + setae cl + mov edi, edx + or edi, esi + test dil, 1 + sete dil + and dil, cl + cmp dil, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + lea rcx, [rdx - 4] + shr rcx + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..92d98788cc09 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 482 +Total uOps: 1900 + +Dispatch Width: 4 +uOps Per Cycle: 3.94 +IPC: 3.94 +Block RThroughput: 4.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rdx, 4 + 1 1 0.50 setae cl + 1 1 0.33 mov edi, edx + 1 1 0.33 or edi, esi + 1 1 0.33 test dil, 1 + 1 1 0.50 sete dil + 1 1 0.33 and dil, cl + 1 1 0.33 cmp dil, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.50 shr rcx + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.66 4.00 4.68 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.40 0.43 - 0.17 - - mov rax, rdi + - - 0.13 0.83 - 0.04 - - cmp rdx, 4 + - - 0.67 - - 0.33 - - setae cl + - - 0.05 0.79 - 0.16 - - mov edi, edx + - - 0.36 0.52 - 0.12 - - or edi, esi + - - 0.39 0.52 - 0.09 - - test dil, 1 + - - 0.98 - - 0.02 - - sete dil + - - 0.42 0.49 - 0.09 - - and dil, cl + - - 0.43 0.41 - 0.16 - - cmp dil, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - 0.33 0.67 - - - - lea rcx, [rdx - 4] + - - 0.50 - - 0.50 - - shr rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_static_size.rs b/rust/zerocopy/benches/ref_type_from_bytes_static_size.rs new file mode 100644 index 000000000000..6a2ced8aad89 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_static_size.rs @@ -0,0 +1,17 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_bytes_static_size(source: &[u8]) -> Option> { + Ref::from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..59c3862eb569 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64 @@ -0,0 +1,15 @@ +bench_ref_type_from_bytes_static_size: + mov rax, rdi + cmp rdx, 6 + setne cl + or cl, sil + test cl, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], 6 + mov qword ptr [rax + 16], 0 + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..286354a1a014 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_static_size.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 404 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 3.22 +IPC: 3.22 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setne cl + 1 1 0.33 or cl, sil + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], 6 + 1 1 1.00 * mov qword ptr [rax + 16], 0 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.55 2.34 4.00 3.11 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.81 0.18 - 0.01 - - mov rax, rdi + - - 0.56 0.43 - 0.01 - - cmp rdx, 6 + - - 1.00 - - - - - setne cl + - - - 0.99 - 0.01 - - or cl, sil + - - 0.18 0.74 - 0.08 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], 6 + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], 0 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..5d3ee937c2b3 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,20 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option> { + Ref::from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..0eb8287a145e --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_ref_type_from_bytes_with_elems_dynamic_padding: + mov rax, rdi + movabs rdi, 6148914691236517201 + cmp rcx, rdi + ja .LBB5_3 + lea rdi, [rcx + 2*rcx] + or rdi, 3 + add rdi, 9 + cmp rdx, rdi + sete dil + movabs r8, 3074457345618258599 + cmp rcx, r8 + setb r8b + and r8b, dil + cmp r8b, 1 + jne .LBB5_3 + test sil, 3 + je .LBB5_4 +.LBB5_3: + mov qword ptr [rax], 0 + ret +.LBB5_4: + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..19928fdca6b8 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 640 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.59 +IPC: 3.59 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 movabs rdi, 6148914691236517201 + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 ja .LBB5_3 + 1 1 0.50 lea rdi, [rcx + 2*rcx] + 1 1 0.33 or rdi, 3 + 1 1 0.33 add rdi, 9 + 1 1 0.33 cmp rdx, rdi + 1 1 0.50 sete dil + 1 1 0.33 movabs r8, 3074457345618258599 + 1 1 0.33 cmp rcx, r8 + 1 1 0.50 setb r8b + 1 1 0.33 and r8b, dil + 1 1 0.33 cmp r8b, 1 + 1 1 1.00 jne .LBB5_3 + 1 1 0.33 test sil, 3 + 1 1 1.00 je .LBB5_4 + 1 1 1.00 * mov qword ptr [rax], 0 + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.33 4.00 6.35 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - 0.66 0.34 - - - - movabs rdi, 6148914691236517201 + - - 0.34 0.66 - - - - cmp rcx, rdi + - - - - - 1.00 - - ja .LBB5_3 + - - 0.66 0.34 - - - - lea rdi, [rcx + 2*rcx] + - - 0.01 0.99 - - - - or rdi, 3 + - - 0.01 0.99 - - - - add rdi, 9 + - - 0.01 0.99 - - - - cmp rdx, rdi + - - 0.99 - - 0.01 - - sete dil + - - 0.35 0.01 - 0.64 - - movabs r8, 3074457345618258599 + - - 0.99 - - 0.01 - - cmp rcx, r8 + - - 0.99 - - 0.01 - - setb r8b + - - - 1.00 - - - - and r8b, dil + - - - 1.00 - - - - cmp r8b, 1 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.32 0.01 - 0.67 - - test sil, 3 + - - - - - 1.00 - - je .LBB5_4 + - - - - 1.00 - - 1.00 mov qword ptr [rax], 0 + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..0ea631ab0c48 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,20 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option> { + Ref::from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..67cc5b58dd14 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,28 @@ +bench_ref_type_from_bytes_with_elems_dynamic_size: + mov rax, rdi + lea rdi, [rcx + rcx] + test rcx, rcx + setns r8b + cmp rdi, -5 + setb dil + and dil, r8b + lea r8, [2*rcx + 4] + cmp rdx, r8 + sete r8b + movabs r9, 4611686018427387902 + cmp rcx, r9 + setb r9b + and r9b, r8b + and r9b, dil + cmp r9b, 1 + je .LBB5_1 +.LBB5_2: + mov qword ptr [rax], 0 + ret +.LBB5_1: + test sil, 1 + jne .LBB5_2 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..cc99f6a4c6f3 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,89 @@ +Iterations: 100 +Instructions: 2500 +Total Cycles: 706 +Total uOps: 2500 + +Dispatch Width: 4 +uOps Per Cycle: 3.54 +IPC: 3.54 +Block RThroughput: 6.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdi, [rcx + rcx] + 1 1 0.33 test rcx, rcx + 1 1 0.50 setns r8b + 1 1 0.33 cmp rdi, -5 + 1 1 0.50 setb dil + 1 1 0.33 and dil, r8b + 1 1 0.50 lea r8, [2*rcx + 4] + 1 1 0.33 cmp rdx, r8 + 1 1 0.50 sete r8b + 1 1 0.33 movabs r9, 4611686018427387902 + 1 1 0.33 cmp rcx, r9 + 1 1 0.50 setb r9b + 1 1 0.33 and r9b, r8b + 1 1 0.33 and r9b, dil + 1 1 0.33 cmp r9b, 1 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 * mov qword ptr [rax], 0 + 1 1 1.00 U ret + 1 1 0.33 test sil, 1 + 1 1 1.00 jne .LBB5_2 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 4.00 7.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.08 - - 0.92 - - mov rax, rdi + - - 0.99 0.01 - - - - lea rdi, [rcx + rcx] + - - 0.92 0.01 - 0.07 - - test rcx, rcx + - - 0.99 - - 0.01 - - setns r8b + - - 0.01 0.01 - 0.98 - - cmp rdi, -5 + - - 0.98 - - 0.02 - - setb dil + - - - 0.99 - 0.01 - - and dil, r8b + - - 0.01 0.99 - - - - lea r8, [2*rcx + 4] + - - 0.99 0.01 - - - - cmp rdx, r8 + - - 1.00 - - - - - sete r8b + - - 0.01 - - 0.99 - - movabs r9, 4611686018427387902 + - - - 1.00 - - - - cmp rcx, r9 + - - 0.99 - - 0.01 - - setb r9b + - - 0.01 0.99 - - - - and r9b, r8b + - - 0.01 0.99 - - - - and r9b, dil + - - - 1.00 - - - - cmp r9b, 1 + - - - - - 1.00 - - je .LBB5_1 + - - - - 1.00 - - 1.00 mov qword ptr [rax], 0 + - - - - - 1.00 - - ret + - - - 1.00 - - - - test sil, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..7826991fc546 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.rs @@ -0,0 +1,22 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_prefix_dynamic_padding( + source: &[u8], +) -> Option> { + match Ref::from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..a549e3337985 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_ref_type_from_prefix_dynamic_padding: + test sil, 3 + sete al + cmp rdx, 12 + setae cl + and cl, al + cmp cl, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rdi], rsi + mov rax, rdi + ret +.LBB5_2: + movabs rcx, 9223372036854775804 + and rcx, rdx + or rdx, 3 + add rdx, -12 + movabs r8, -6148914691236517205 + mov rax, rdx + mul r8 + shr rdx + mov qword ptr [rdi + 8], rcx + mov qword ptr [rdi + 16], rdx + mov qword ptr [rdi], rsi + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..d85fe71ca66f --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 907 +Total uOps: 2500 + +Dispatch Width: 4 +uOps Per Cycle: 2.76 +IPC: 2.65 +Block RThroughput: 6.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test sil, 3 + 1 1 0.50 sete al + 1 1 0.33 cmp rdx, 12 + 1 1 0.50 setae cl + 1 1 0.33 and cl, al + 1 1 0.33 cmp cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.33 movabs rcx, 9223372036854775804 + 1 1 0.33 and rcx, rdx + 1 1 0.33 or rdx, 3 + 1 1 0.33 add rdx, -12 + 1 1 0.33 movabs r8, -6148914691236517205 + 1 1 0.33 mov rax, rdx + 2 4 1.00 mul r8 + 1 1 0.50 shr rdx + 1 1 1.00 * mov qword ptr [rdi + 8], rcx + 1 1 1.00 * mov qword ptr [rdi + 16], rdx + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.96 6.07 4.00 6.97 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.92 0.07 - 0.01 - - test sil, 3 + - - 0.98 - - 0.02 - - sete al + - - 0.03 0.97 - - - - cmp rdx, 12 + - - 0.99 - - 0.01 - - setae cl + - - 0.03 0.94 - 0.03 - - and cl, al + - - 0.06 0.94 - - - - cmp cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rdi], rsi + - - 0.01 0.05 - 0.94 - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.92 0.08 - - - - movabs rcx, 9223372036854775804 + - - 0.96 0.01 - 0.03 - - and rcx, rdx + - - - 0.03 - 0.97 - - or rdx, 3 + - - 0.01 0.96 - 0.03 - - add rdx, -12 + - - 0.03 0.06 - 0.91 - - movabs r8, -6148914691236517205 + - - 0.94 0.03 - 0.03 - - mov rax, rdx + - - 1.00 1.00 - - - - mul r8 + - - 0.05 - - 0.95 - - shr rdx + - - - - 1.00 - 1.00 - mov qword ptr [rdi + 8], rcx + - - - - 1.00 - - 1.00 mov qword ptr [rdi + 16], rdx + - - - - 1.00 - 1.00 - mov qword ptr [rdi], rsi + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..131a12103656 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.rs @@ -0,0 +1,22 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_prefix_dynamic_size( + source: &[u8], +) -> Option> { + match Ref::from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..fc050aac2637 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,19 @@ +bench_ref_type_from_prefix_dynamic_size: + mov rax, rdi + cmp rdx, 4 + setb cl + or cl, sil + test cl, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + movabs rcx, 9223372036854775806 + and rcx, rdx + add rdx, -4 + shr rdx + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rdx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..a249f0305291 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,73 @@ +Iterations: 100 +Instructions: 1700 +Total Cycles: 431 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.94 +IPC: 3.94 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rdx, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, sil + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 0.33 movabs rcx, 9223372036854775806 + 1 1 0.33 and rcx, rdx + 1 1 0.33 add rdx, -4 + 1 1 0.50 shr rdx + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rdx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.99 3.75 4.00 4.26 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov rax, rdi + - - 0.74 0.26 - - - - cmp rdx, 4 + - - 1.00 - - - - - setb cl + - - 0.25 0.74 - 0.01 - - or cl, sil + - - 0.01 0.97 - 0.02 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - 0.48 0.52 - - - - movabs rcx, 9223372036854775806 + - - 0.73 0.27 - - - - and rcx, rdx + - - 0.02 - - 0.98 - - add rdx, -4 + - - 0.76 - - 0.24 - - shr rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rcx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_static_size.rs b/rust/zerocopy/benches/ref_type_from_prefix_static_size.rs new file mode 100644 index 000000000000..c149fd5088ee --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_static_size.rs @@ -0,0 +1,20 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_prefix_static_size(source: &[u8]) -> Option> { + match Ref::from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..420fd6f3b4bd --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64 @@ -0,0 +1,15 @@ +bench_ref_type_from_prefix_static_size: + mov rax, rdi + cmp rdx, 6 + setb cl + or cl, sil + test cl, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], 6 + mov qword ptr [rax + 16], 0 + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..6b65f303a786 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_static_size.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 404 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 3.22 +IPC: 3.22 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, sil + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], 6 + 1 1 1.00 * mov qword ptr [rax + 16], 0 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.55 2.34 4.00 3.11 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.81 0.18 - 0.01 - - mov rax, rdi + - - 0.56 0.43 - 0.01 - - cmp rdx, 6 + - - 1.00 - - - - - setb cl + - - - 0.99 - 0.01 - - or cl, sil + - - 0.18 0.74 - 0.08 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], 6 + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], 0 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..a4fe5a6c86ca --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,23 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option> { + match Ref::from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..fb9d6bfaac27 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_ref_type_from_prefix_with_elems_dynamic_padding: + mov rax, rdi + movabs rdi, 6148914691236517201 + cmp rcx, rdi + ja .LBB5_4 + lea rdi, [rcx + 2*rcx] + or rdi, 3 + add rdi, 9 + cmp rdi, rdx + setbe dl + movabs r8, 3074457345618258599 + cmp rcx, r8 + setb r8b + and r8b, dl + cmp r8b, 1 + jne .LBB5_4 + test sil, 3 + je .LBB5_3 +.LBB5_4: + mov qword ptr [rax], 0 + ret +.LBB5_3: + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rdi + mov qword ptr [rax + 16], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..0ef4c84142b0 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 674 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.41 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 movabs rdi, 6148914691236517201 + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 ja .LBB5_4 + 1 1 0.50 lea rdi, [rcx + 2*rcx] + 1 1 0.33 or rdi, 3 + 1 1 0.33 add rdi, 9 + 1 1 0.33 cmp rdi, rdx + 2 2 1.00 setbe dl + 1 1 0.33 movabs r8, 3074457345618258599 + 1 1 0.33 cmp rcx, r8 + 1 1 0.50 setb r8b + 1 1 0.33 and r8b, dl + 1 1 0.33 cmp r8b, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 test sil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 1.00 * mov qword ptr [rax], 0 + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rdi + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.66 6.65 4.00 6.69 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov rax, rdi + - - 0.01 0.99 - - - - movabs rdi, 6148914691236517201 + - - 0.02 0.98 - - - - cmp rcx, rdi + - - - - - 1.00 - - ja .LBB5_4 + - - 0.02 0.98 - - - - lea rdi, [rcx + 2*rcx] + - - 0.68 0.32 - - - - or rdi, 3 + - - 0.98 0.02 - - - - add rdi, 9 + - - 0.97 0.03 - - - - cmp rdi, rdx + - - 1.96 - - 0.04 - - setbe dl + - - 0.02 0.98 - - - - movabs r8, 3074457345618258599 + - - 0.02 0.34 - 0.64 - - cmp rcx, r8 + - - 0.02 - - 0.98 - - setb r8b + - - 0.97 0.02 - 0.01 - - and r8b, dl + - - 0.97 0.02 - 0.01 - - cmp r8b, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.01 0.99 - - - - test sil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - - - 1.00 - - 1.00 mov qword ptr [rax], 0 + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rdi + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..b0911e8fd8e7 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,23 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option> { + match Ref::from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..20ba73bd552e --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_from_prefix_with_elems_dynamic_size: + mov rax, rdi + lea rdi, [rcx + rcx] + test rcx, rcx + sets r8b + cmp rdi, -5 + setae r9b + or r9b, r8b + jne .LBB5_3 + add rdi, 4 + cmp rdi, rdx + ja .LBB5_3 + movabs rdx, 4611686018427387902 + cmp rcx, rdx + setb dl + test sil, 1 + sete r8b + test dl, r8b + jne .LBB5_4 +.LBB5_3: + mov qword ptr [rax], 0 + ret +.LBB5_4: + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rdi + mov qword ptr [rax + 16], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..5600d4ea6f69 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 672 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.57 +IPC: 3.57 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdi, [rcx + rcx] + 1 1 0.33 test rcx, rcx + 1 1 0.50 sets r8b + 1 1 0.33 cmp rdi, -5 + 1 1 0.50 setae r9b + 1 1 0.33 or r9b, r8b + 1 1 1.00 jne .LBB5_3 + 1 1 0.33 add rdi, 4 + 1 1 0.33 cmp rdi, rdx + 1 1 1.00 ja .LBB5_3 + 1 1 0.33 movabs rdx, 4611686018427387902 + 1 1 0.33 cmp rcx, rdx + 1 1 0.50 setb dl + 1 1 0.33 test sil, 1 + 1 1 0.50 sete r8b + 1 1 0.33 test dl, r8b + 1 1 1.00 jne .LBB5_4 + 1 1 1.00 * mov qword ptr [rax], 0 + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rdi + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.66 6.65 4.00 6.69 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.96 - 0.01 - - mov rax, rdi + - - 0.64 0.36 - - - - lea rdi, [rcx + rcx] + - - 0.34 0.66 - - - - test rcx, rcx + - - 0.99 - - 0.01 - - sets r8b + - - 0.66 0.34 - - - - cmp rdi, -5 + - - 0.67 - - 0.33 - - setae r9b + - - 0.01 0.99 - - - - or r9b, r8b + - - - - - 1.00 - - jne .LBB5_3 + - - - 0.99 - 0.01 - - add rdi, 4 + - - 0.66 0.33 - 0.01 - - cmp rdi, rdx + - - - - - 1.00 - - ja .LBB5_3 + - - - 0.01 - 0.99 - - movabs rdx, 4611686018427387902 + - - 0.33 0.34 - 0.33 - - cmp rcx, rdx + - - 1.00 - - - - - setb dl + - - 0.33 0.67 - - - - test sil, 1 + - - 1.00 - - - - - sete r8b + - - - 1.00 - - - - test dl, r8b + - - - - - 1.00 - - jne .LBB5_4 + - - - - 1.00 - - 1.00 mov qword ptr [rax], 0 + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rdi + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..894b7d875b42 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.rs @@ -0,0 +1,22 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_suffix_dynamic_padding( + source: &[u8], +) -> Option> { + match Ref::from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..4a0f5e88612e --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,31 @@ +bench_ref_type_from_suffix_dynamic_padding: + lea eax, [rdx + rsi] + test al, 3 + sete al + cmp rdx, 12 + setae cl + and cl, al + cmp cl, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rdi], rsi + mov rax, rdi + ret +.LBB5_2: + mov rax, rdx + or rax, 3 + add rax, -12 + movabs rcx, -6148914691236517205 + mov r8, rdx + mul rcx + shr rdx + mov eax, r8d + and eax, 3 + movabs rcx, 9223372036854775804 + and rcx, r8 + add rsi, rax + mov qword ptr [rdi + 8], rcx + mov qword ptr [rdi + 16], rdx + mov qword ptr [rdi], rsi + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..ad884432de74 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,97 @@ +Iterations: 100 +Instructions: 2900 +Total Cycles: 942 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 3.18 +IPC: 3.08 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rdx + rsi] + 1 1 0.33 test al, 3 + 1 1 0.50 sete al + 1 1 0.33 cmp rdx, 12 + 1 1 0.50 setae cl + 1 1 0.33 and cl, al + 1 1 0.33 cmp cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.33 mov rax, rdx + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, -12 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov r8, rdx + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.33 mov eax, r8d + 1 1 0.33 and eax, 3 + 1 1 0.33 movabs rcx, 9223372036854775804 + 1 1 0.33 and rcx, r8 + 1 1 0.33 add rsi, rax + 1 1 1.00 * mov qword ptr [rdi + 8], rcx + 1 1 1.00 * mov qword ptr [rdi + 16], rdx + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.34 8.01 4.00 8.65 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.36 0.64 - - - - lea eax, [rdx + rsi] + - - 0.32 0.36 - 0.32 - - test al, 3 + - - 0.69 - - 0.31 - - sete al + - - 0.32 0.04 - 0.64 - - cmp rdx, 12 + - - 0.68 - - 0.32 - - setae cl + - - 0.31 0.37 - 0.32 - - and cl, al + - - 0.31 0.37 - 0.32 - - cmp cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rdi], rsi + - - 0.33 0.36 - 0.31 - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.32 0.32 - 0.36 - - mov rax, rdx + - - 0.01 0.63 - 0.36 - - or rax, 3 + - - - 0.68 - 0.32 - - add rax, -12 + - - 0.34 0.63 - 0.03 - - movabs rcx, -6148914691236517205 + - - 0.32 0.32 - 0.36 - - mov r8, rdx + - - 1.00 1.00 - - - - mul rcx + - - 0.69 - - 0.31 - - shr rdx + - - 0.37 0.32 - 0.31 - - mov eax, r8d + - - 0.35 0.33 - 0.32 - - and eax, 3 + - - 0.63 0.35 - 0.02 - - movabs rcx, 9223372036854775804 + - - 0.32 0.32 - 0.36 - - and rcx, r8 + - - 0.33 0.35 - 0.32 - - add rsi, rax + - - - - 1.00 - 1.00 - mov qword ptr [rdi + 8], rcx + - - - - 1.00 - - 1.00 mov qword ptr [rdi + 16], rdx + - - - - 1.00 - 1.00 - mov qword ptr [rdi], rsi + - - 0.34 0.62 - 0.04 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..2cdadc3316d6 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.rs @@ -0,0 +1,22 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_suffix_dynamic_size( + source: &[u8], +) -> Option> { + match Ref::from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..9dc329867dde --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,23 @@ +bench_ref_type_from_suffix_dynamic_size: + mov rax, rdi + lea ecx, [rdx + rsi] + cmp rdx, 4 + setb dil + or dil, cl + test dil, 1 + je .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + lea rcx, [rdx - 4] + shr rcx + mov edi, edx + and edi, 1 + movabs r8, 9223372036854775806 + and r8, rdx + add rsi, rdi + mov qword ptr [rax + 8], r8 + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..7639f237aa87 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 540 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.89 +IPC: 3.89 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea ecx, [rdx + rsi] + 1 1 0.33 cmp rdx, 4 + 1 1 0.50 setb dil + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.50 shr rcx + 1 1 0.33 mov edi, edx + 1 1 0.33 and edi, 1 + 1 1 0.33 movabs r8, 9223372036854775806 + 1 1 0.33 and r8, rdx + 1 1 0.33 add rsi, rdi + 1 1 1.00 * mov qword ptr [rax + 8], r8 + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.32 5.33 4.00 5.35 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.66 - 0.01 - - mov rax, rdi + - - 0.99 0.01 - - - - lea ecx, [rdx + rsi] + - - 0.01 0.98 - 0.01 - - cmp rdx, 4 + - - 0.99 - - 0.01 - - setb dil + - - - 1.00 - - - - or dil, cl + - - - 1.00 - - - - test dil, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - 0.65 0.35 - - - - lea rcx, [rdx - 4] + - - 0.68 - - 0.32 - - shr rcx + - - - 0.64 - 0.36 - - mov edi, edx + - - 0.66 0.01 - 0.33 - - and edi, 1 + - - 0.34 - - 0.66 - - movabs r8, 9223372036854775806 + - - 0.32 0.68 - - - - and r8, rdx + - - 0.35 - - 0.65 - - add rsi, rdi + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], r8 + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_static_size.rs b/rust/zerocopy/benches/ref_type_from_suffix_static_size.rs new file mode 100644 index 000000000000..2ad2d2f5ee33 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_static_size.rs @@ -0,0 +1,20 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_suffix_static_size(source: &[u8]) -> Option> { + match Ref::from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..37a3d72e7ca1 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64 @@ -0,0 +1,18 @@ +bench_ref_type_from_suffix_static_size: + mov rax, rdi + lea ecx, [rdx + rsi] + cmp rdx, 6 + setb dil + or dil, cl + test dil, 1 + je .LBB5_2 + xor ecx, ecx + mov qword ptr [rax], rcx + ret +.LBB5_2: + lea rcx, [rsi + rdx] + add rcx, -6 + mov qword ptr [rax + 8], 6 + mov qword ptr [rax + 16], 0 + mov qword ptr [rax], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..4dfa9bda8ceb --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_static_size.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 405 +Total uOps: 1600 + +Dispatch Width: 4 +uOps Per Cycle: 3.95 +IPC: 3.95 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea ecx, [rdx + rsi] + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setb dil + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor ecx, ecx + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rsi + rdx] + 1 1 0.33 add rcx, -6 + 1 1 1.00 * mov qword ptr [rax + 8], 6 + 1 1 1.00 * mov qword ptr [rax + 16], 0 + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.66 4.00 3.68 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - mov rax, rdi + - - 0.33 0.67 - - - - lea ecx, [rdx + rsi] + - - 0.34 0.33 - 0.33 - - cmp rdx, 6 + - - 0.99 - - 0.01 - - setb dil + - - 0.33 0.67 - - - - or dil, cl + - - - 1.00 - - - - test dil, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor ecx, ecx + - - - - 1.00 - - 1.00 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - lea rcx, [rsi + rdx] + - - 0.67 - - 0.33 - - add rcx, -6 + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], 6 + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], 0 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..105086dd4dc9 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,23 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option> { + match Ref::from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..a52ec2c34dd9 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_from_suffix_with_elems_dynamic_padding: + mov rax, rdi + movabs rdi, 6148914691236517201 + cmp rcx, rdi + ja .LBB5_3 + lea rdi, [rcx + 2*rcx] + or rdi, 3 + add rdi, 9 + sub rdx, rdi + setae r8b + movabs r9, 3074457345618258599 + cmp rcx, r9 + setb r9b + and r9b, r8b + cmp r9b, 1 + jne .LBB5_3 + add rsi, rdx + test sil, 3 + je .LBB5_4 +.LBB5_3: + mov qword ptr [rax], 0 + ret +.LBB5_4: + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rdi + mov qword ptr [rax + 16], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..b8036059b98a --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 673 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.57 +IPC: 3.57 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 movabs rdi, 6148914691236517201 + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 ja .LBB5_3 + 1 1 0.50 lea rdi, [rcx + 2*rcx] + 1 1 0.33 or rdi, 3 + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rdi + 1 1 0.50 setae r8b + 1 1 0.33 movabs r9, 3074457345618258599 + 1 1 0.33 cmp rcx, r9 + 1 1 0.50 setb r9b + 1 1 0.33 and r9b, r8b + 1 1 0.33 cmp r9b, 1 + 1 1 1.00 jne .LBB5_3 + 1 1 0.33 add rsi, rdx + 1 1 0.33 test sil, 3 + 1 1 1.00 je .LBB5_4 + 1 1 1.00 * mov qword ptr [rax], 0 + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rdi + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.66 6.66 4.00 6.68 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.01 - 0.66 - - mov rax, rdi + - - 0.34 0.02 - 0.64 - - movabs rdi, 6148914691236517201 + - - 0.34 0.34 - 0.32 - - cmp rcx, rdi + - - - - - 1.00 - - ja .LBB5_3 + - - 0.66 0.34 - - - - lea rdi, [rcx + 2*rcx] + - - 0.02 0.97 - 0.01 - - or rdi, 3 + - - 0.02 0.98 - - - - add rdi, 9 + - - 0.01 0.99 - - - - sub rdx, rdi + - - 1.00 - - - - - setae r8b + - - 0.97 0.02 - 0.01 - - movabs r9, 3074457345618258599 + - - 0.98 0.01 - 0.01 - - cmp rcx, r9 + - - 0.98 - - 0.02 - - setb r9b + - - 0.01 0.98 - 0.01 - - and r9b, r8b + - - 0.02 0.98 - - - - cmp r9b, 1 + - - - - - 1.00 - - jne .LBB5_3 + - - - 1.00 - - - - add rsi, rdx + - - 0.98 0.02 - - - - test sil, 3 + - - - - - 1.00 - - je .LBB5_4 + - - - - 1.00 - - 1.00 mov qword ptr [rax], 0 + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rdi + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..926f3044b4bf --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,23 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option> { + match Ref::from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..a6c84ec32242 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_from_suffix_with_elems_dynamic_size: + mov rax, rdi + lea rdi, [rcx + rcx] + test rcx, rcx + sets r8b + cmp rdi, -5 + setae r9b + or r9b, r8b + jne .LBB5_4 + add rdi, 4 + sub rdx, rdi + jb .LBB5_4 + movabs r8, 4611686018427387901 + cmp rcx, r8 + ja .LBB5_4 + add rsi, rdx + mov edx, esi + and edx, 1 + je .LBB5_5 +.LBB5_4: + mov qword ptr [rax], 0 + ret +.LBB5_5: + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rdi + mov qword ptr [rax + 16], rcx + ret diff --git a/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..113b75b4172d --- /dev/null +++ b/rust/zerocopy/benches/ref_type_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 673 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.57 +IPC: 3.57 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdi, [rcx + rcx] + 1 1 0.33 test rcx, rcx + 1 1 0.50 sets r8b + 1 1 0.33 cmp rdi, -5 + 1 1 0.50 setae r9b + 1 1 0.33 or r9b, r8b + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rdi + 1 1 1.00 jb .LBB5_4 + 1 1 0.33 movabs r8, 4611686018427387901 + 1 1 0.33 cmp rcx, r8 + 1 1 1.00 ja .LBB5_4 + 1 1 0.33 add rsi, rdx + 1 1 0.33 mov edx, esi + 1 1 0.33 and edx, 1 + 1 1 1.00 je .LBB5_5 + 1 1 1.00 * mov qword ptr [rax], 0 + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rdi + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.65 4.00 6.70 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov rax, rdi + - - 0.01 0.99 - - - - lea rdi, [rcx + rcx] + - - 0.67 0.33 - - - - test rcx, rcx + - - 0.99 - - 0.01 - - sets r8b + - - - 1.00 - - - - cmp rdi, -5 + - - 1.00 - - - - - setae r9b + - - 1.00 - - - - - or r9b, r8b + - - - - - 1.00 - - jne .LBB5_4 + - - - 0.99 - 0.01 - - add rdi, 4 + - - - 0.33 - 0.67 - - sub rdx, rdi + - - - - - 1.00 - - jb .LBB5_4 + - - - 1.00 - - - - movabs r8, 4611686018427387901 + - - 0.66 0.34 - - - - cmp rcx, r8 + - - - - - 1.00 - - ja .LBB5_4 + - - 0.34 0.66 - - - - add rsi, rdx + - - 0.98 0.02 - - - - mov edx, esi + - - 1.00 - - - - - and edx, 1 + - - - - - 1.00 - - je .LBB5_5 + - - - - 1.00 - - 1.00 mov qword ptr [rax], 0 + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rdi + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.rs b/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.rs new file mode 100644 index 000000000000..04ac859a98a8 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_into_mut_dynamic_size( + source: Ref<&mut [u8], format::LocoPacket>, +) -> &mut format::LocoPacket { + Ref::into_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64 new file mode 100644 index 000000000000..b7568f6631ce --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_into_mut_dynamic_size: + mov rdx, qword ptr [rdi + 16] + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 8] + mov esi, 1 + movabs rdi, 4611686018427387902 + cmp rdx, rdi + jae .LBB8_3 + test al, 1 + jne .LBB8_2 + lea rdi, [2*rdx + 4] + cmp rcx, rdi + jne .LBB8_3 + ret +.LBB8_2: + xor esi, esi +.LBB8_3: + sub rsp, 24 + mov qword ptr [rsp], rsi + mov qword ptr [rsp + 8], rax + mov qword ptr [rsp + 16], rcx + lea rdi, [rip + .Lanon.HASH.2] + lea rcx, [rip + .Lanon.HASH.1] + lea r8, [rip + .Lanon.HASH.4] + mov rdx, rsp + mov esi, 54 + call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..03137756a8b5 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_mut_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1647 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 1.64 +IPC: 1.46 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rdx, qword ptr [rdi + 16] + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rcx, qword ptr [rdi + 8] + 1 1 0.33 mov esi, 1 + 1 1 0.33 movabs rdi, 4611686018427387902 + 1 1 0.33 cmp rdx, rdi + 1 1 1.00 jae .LBB8_3 + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB8_2 + 1 1 0.50 lea rdi, [2*rdx + 4] + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 jne .LBB8_3 + 1 1 1.00 U ret + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rsp, 24 + 1 1 1.00 * mov qword ptr [rsp], rsi + 1 1 1.00 * mov qword ptr [rsp + 8], rax + 1 1 1.00 * mov qword ptr [rsp + 16], rcx + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.2] + 1 1 0.50 lea rcx, [rip + .Lanon.HASH.1] + 1 1 0.50 lea r8, [rip + .Lanon.HASH.4] + 1 1 0.33 mov rdx, rsp + 1 1 0.33 mov esi, 54 + 4 7 1.00 * call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.15 5.74 4.00 6.11 3.98 4.02 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.71 0.29 mov rdx, qword ptr [rdi + 16] + - - - - - - 0.72 0.28 mov rax, qword ptr [rdi] + - - - - - - 0.29 0.71 mov rcx, qword ptr [rdi + 8] + - - 0.83 0.16 - 0.01 - - mov esi, 1 + - - 0.16 0.84 - - - - movabs rdi, 4611686018427387902 + - - 0.80 0.04 - 0.16 - - cmp rdx, rdi + - - - - - 1.00 - - jae .LBB8_3 + - - 0.13 0.74 - 0.13 - - test al, 1 + - - - - - 1.00 - - jne .LBB8_2 + - - 0.10 0.90 - - - - lea rdi, [2*rdx + 4] + - - 0.09 0.91 - - - - cmp rcx, rdi + - - - - - 1.00 - - jne .LBB8_3 + - - - - - 1.00 - - ret + - - - - - - - - xor esi, esi + - - 0.29 0.13 - 0.58 - - sub rsp, 24 + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp], rsi + - - - - 1.00 - 0.28 0.72 mov qword ptr [rsp + 8], rax + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp + 16], rcx + - - 0.76 0.24 - - - - lea rdi, [rip + .Lanon.HASH.2] + - - 0.82 0.18 - - - - lea rcx, [rip + .Lanon.HASH.1] + - - 0.92 0.08 - - - - lea r8, [rip + .Lanon.HASH.4] + - - 0.09 0.68 - 0.23 - - mov rdx, rsp + - - 0.16 0.84 - - - - mov esi, 54 + - - - - 1.00 1.00 0.54 1.46 call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_into_mut_static_size.rs b/rust/zerocopy/benches/ref_type_into_mut_static_size.rs new file mode 100644 index 000000000000..edab9abe0aa8 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_mut_static_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_into_mut_static_size( + source: Ref<&mut [u8], format::LocoPacket>, +) -> &mut format::LocoPacket { + Ref::into_mut(source) +} diff --git a/rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64 b/rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64 new file mode 100644 index 000000000000..0412b33d6cc3 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_ref_type_into_mut_static_size: + mov rax, qword ptr [rdi] + ret diff --git a/rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64.mca new file mode 100644 index 000000000000..2de0e6f9791c --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_mut_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.rs b/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.rs new file mode 100644 index 000000000000..9f5f3b618c2f --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_into_ref_dynamic_padding( + source: Ref<&[u8], format::LocoPacket>, +) -> &format::LocoPacket { + Ref::into_ref(source) +} diff --git a/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64 new file mode 100644 index 000000000000..ca16dbc03e8e --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64 @@ -0,0 +1,29 @@ +bench_ref_type_into_ref_dynamic_padding: + mov rdx, qword ptr [rdi + 16] + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 8] + mov esi, 1 + movabs rdi, 3074457345618258599 + cmp rdx, rdi + jae .LBB8_3 + test al, 3 + jne .LBB8_2 + lea rdi, [rdx + 2*rdx] + or rdi, 3 + add rdi, 9 + cmp rcx, rdi + jne .LBB8_3 + ret +.LBB8_2: + xor esi, esi +.LBB8_3: + sub rsp, 24 + mov qword ptr [rsp], rsi + mov qword ptr [rsp + 8], rax + mov qword ptr [rsp + 16], rcx + lea rdi, [rip + .Lanon.HASH.2] + lea rcx, [rip + .Lanon.HASH.1] + lea r8, [rip + .Lanon.HASH.4] + mov rdx, rsp + mov esi, 54 + call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..bf8eade92561 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 1812 +Total uOps: 2900 + +Dispatch Width: 4 +uOps Per Cycle: 1.60 +IPC: 1.43 +Block RThroughput: 7.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rdx, qword ptr [rdi + 16] + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rcx, qword ptr [rdi + 8] + 1 1 0.33 mov esi, 1 + 1 1 0.33 movabs rdi, 3074457345618258599 + 1 1 0.33 cmp rdx, rdi + 1 1 1.00 jae .LBB8_3 + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB8_2 + 1 1 0.50 lea rdi, [rdx + 2*rdx] + 1 1 0.33 or rdi, 3 + 1 1 0.33 add rdi, 9 + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 jne .LBB8_3 + 1 1 1.00 U ret + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rsp, 24 + 1 1 1.00 * mov qword ptr [rsp], rsi + 1 1 1.00 * mov qword ptr [rsp + 8], rax + 1 1 1.00 * mov qword ptr [rsp + 16], rcx + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.2] + 1 1 0.50 lea rcx, [rip + .Lanon.HASH.1] + 1 1 0.50 lea r8, [rip + .Lanon.HASH.4] + 1 1 0.33 mov rdx, rsp + 1 1 0.33 mov esi, 54 + 4 7 1.00 * call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.41 6.42 4.00 6.17 3.99 4.01 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.54 0.46 mov rdx, qword ptr [rdi + 16] + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - - 0.49 0.51 mov rcx, qword ptr [rdi + 8] + - - 0.52 0.47 - 0.01 - - mov esi, 1 + - - 0.47 0.53 - - - - movabs rdi, 3074457345618258599 + - - 0.47 0.53 - - - - cmp rdx, rdi + - - - - - 1.00 - - jae .LBB8_3 + - - 0.54 0.44 - 0.02 - - test al, 3 + - - - - - 1.00 - - jne .LBB8_2 + - - 0.44 0.56 - - - - lea rdi, [rdx + 2*rdx] + - - 0.48 0.52 - - - - or rdi, 3 + - - 0.48 0.48 - 0.04 - - add rdi, 9 + - - 0.51 0.39 - 0.10 - - cmp rcx, rdi + - - - - - 1.00 - - jne .LBB8_3 + - - - - - 1.00 - - ret + - - - - - - - - xor esi, esi + - - 0.53 0.47 - - - - sub rsp, 24 + - - - - 1.00 - 0.52 0.48 mov qword ptr [rsp], rsi + - - - - 1.00 - 0.48 0.52 mov qword ptr [rsp + 8], rax + - - - - 1.00 - 0.52 0.48 mov qword ptr [rsp + 16], rcx + - - 0.49 0.51 - - - - lea rdi, [rip + .Lanon.HASH.2] + - - 0.46 0.54 - - - - lea rcx, [rip + .Lanon.HASH.1] + - - 0.54 0.46 - - - - lea r8, [rip + .Lanon.HASH.4] + - - 0.04 0.06 - 0.90 - - mov rdx, rsp + - - 0.44 0.46 - 0.10 - - mov esi, 54 + - - - - 1.00 1.00 0.94 1.06 call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.rs b/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.rs new file mode 100644 index 000000000000..535c67985200 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_into_ref_dynamic_size( + source: Ref<&[u8], format::LocoPacket>, +) -> &format::LocoPacket { + Ref::into_ref(source) +} diff --git a/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..81624de69046 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64 @@ -0,0 +1,27 @@ +bench_ref_type_into_ref_dynamic_size: + mov rdx, qword ptr [rdi + 16] + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 8] + mov esi, 1 + movabs rdi, 4611686018427387902 + cmp rdx, rdi + jae .LBB8_3 + test al, 1 + jne .LBB8_2 + lea rdi, [2*rdx + 4] + cmp rcx, rdi + jne .LBB8_3 + ret +.LBB8_2: + xor esi, esi +.LBB8_3: + sub rsp, 24 + mov qword ptr [rsp], rsi + mov qword ptr [rsp + 8], rax + mov qword ptr [rsp + 16], rcx + lea rdi, [rip + .Lanon.HASH.2] + lea rcx, [rip + .Lanon.HASH.1] + lea r8, [rip + .Lanon.HASH.4] + mov rdx, rsp + mov esi, 54 + call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..03137756a8b5 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1647 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 1.64 +IPC: 1.46 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rdx, qword ptr [rdi + 16] + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov rcx, qword ptr [rdi + 8] + 1 1 0.33 mov esi, 1 + 1 1 0.33 movabs rdi, 4611686018427387902 + 1 1 0.33 cmp rdx, rdi + 1 1 1.00 jae .LBB8_3 + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB8_2 + 1 1 0.50 lea rdi, [2*rdx + 4] + 1 1 0.33 cmp rcx, rdi + 1 1 1.00 jne .LBB8_3 + 1 1 1.00 U ret + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rsp, 24 + 1 1 1.00 * mov qword ptr [rsp], rsi + 1 1 1.00 * mov qword ptr [rsp + 8], rax + 1 1 1.00 * mov qword ptr [rsp + 16], rcx + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.2] + 1 1 0.50 lea rcx, [rip + .Lanon.HASH.1] + 1 1 0.50 lea r8, [rip + .Lanon.HASH.4] + 1 1 0.33 mov rdx, rsp + 1 1 0.33 mov esi, 54 + 4 7 1.00 * call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.15 5.74 4.00 6.11 3.98 4.02 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.71 0.29 mov rdx, qword ptr [rdi + 16] + - - - - - - 0.72 0.28 mov rax, qword ptr [rdi] + - - - - - - 0.29 0.71 mov rcx, qword ptr [rdi + 8] + - - 0.83 0.16 - 0.01 - - mov esi, 1 + - - 0.16 0.84 - - - - movabs rdi, 4611686018427387902 + - - 0.80 0.04 - 0.16 - - cmp rdx, rdi + - - - - - 1.00 - - jae .LBB8_3 + - - 0.13 0.74 - 0.13 - - test al, 1 + - - - - - 1.00 - - jne .LBB8_2 + - - 0.10 0.90 - - - - lea rdi, [2*rdx + 4] + - - 0.09 0.91 - - - - cmp rcx, rdi + - - - - - 1.00 - - jne .LBB8_3 + - - - - - 1.00 - - ret + - - - - - - - - xor esi, esi + - - 0.29 0.13 - 0.58 - - sub rsp, 24 + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp], rsi + - - - - 1.00 - 0.28 0.72 mov qword ptr [rsp + 8], rax + - - - - 1.00 - 0.72 0.28 mov qword ptr [rsp + 16], rcx + - - 0.76 0.24 - - - - lea rdi, [rip + .Lanon.HASH.2] + - - 0.82 0.18 - - - - lea rcx, [rip + .Lanon.HASH.1] + - - 0.92 0.08 - - - - lea r8, [rip + .Lanon.HASH.4] + - - 0.09 0.68 - 0.23 - - mov rdx, rsp + - - 0.16 0.84 - - - - mov esi, 54 + - - - - 1.00 1.00 0.54 1.46 call qword ptr [rip + core::result::unwrap_failed@GOTPCREL] diff --git a/rust/zerocopy/benches/ref_type_into_ref_static_size.rs b/rust/zerocopy/benches/ref_type_into_ref_static_size.rs new file mode 100644 index 000000000000..a2bdf559a903 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_static_size.rs @@ -0,0 +1,19 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_into_ref_static_size( + source: Ref<&[u8], format::LocoPacket>, +) -> &format::LocoPacket { + Ref::into_ref(source) +} diff --git a/rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64 b/rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64 new file mode 100644 index 000000000000..173562c46561 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_ref_type_into_ref_static_size: + mov rax, qword ptr [rdi] + ret diff --git a/rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..2de0e6f9791c --- /dev/null +++ b/rust/zerocopy/benches/ref_type_into_ref_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_read_static_size.rs b/rust/zerocopy/benches/ref_type_read_static_size.rs new file mode 100644 index 000000000000..da7111261b80 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_read_static_size.rs @@ -0,0 +1,17 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_read_static_size(source: &Ref<&[u8], format::LocoPacket>) -> format::LocoPacket { + Ref::read(source) +} diff --git a/rust/zerocopy/benches/ref_type_read_static_size.x86-64 b/rust/zerocopy/benches/ref_type_read_static_size.x86-64 new file mode 100644 index 000000000000..c5ebb527f419 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_read_static_size.x86-64 @@ -0,0 +1,7 @@ +bench_ref_type_read_static_size: + mov rax, qword ptr [rdi] + mov ecx, dword ptr [rax] + movzx eax, word ptr [rax + 4] + shl rax, 32 + or rax, rcx + ret diff --git a/rust/zerocopy/benches/ref_type_read_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_read_static_size.x86-64.mca new file mode 100644 index 000000000000..37cfa368ce16 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_read_static_size.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 169 +Total uOps: 600 + +Dispatch Width: 4 +uOps Per Cycle: 3.55 +IPC: 3.55 +Block RThroughput: 1.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov ecx, dword ptr [rax] + 1 5 0.50 * movzx eax, word ptr [rax + 4] + 1 1 0.50 shl rax, 32 + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.91 0.90 - 1.19 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - - 0.31 0.69 mov ecx, dword ptr [rax] + - - - - - - 0.69 0.31 movzx eax, word ptr [rax + 4] + - - 0.91 - - 0.09 - - shl rax, 32 + - - - 0.90 - 0.10 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_type_write_static_size.rs b/rust/zerocopy/benches/ref_type_write_static_size.rs new file mode 100644 index 000000000000..6537cc4609e9 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_write_static_size.rs @@ -0,0 +1,20 @@ +// Copyright 2026 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use zerocopy::Ref; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_type_write_static_size( + destination: &mut Ref<&mut [u8], format::LocoPacket>, + value: format::LocoPacket, +) { + Ref::write(destination, value) +} diff --git a/rust/zerocopy/benches/ref_type_write_static_size.x86-64 b/rust/zerocopy/benches/ref_type_write_static_size.x86-64 new file mode 100644 index 000000000000..1b919d01409a --- /dev/null +++ b/rust/zerocopy/benches/ref_type_write_static_size.x86-64 @@ -0,0 +1,6 @@ +bench_ref_type_write_static_size: + mov rax, qword ptr [rdi] + mov dword ptr [rax], esi + shr rsi, 32 + mov word ptr [rax + 4], si + ret diff --git a/rust/zerocopy/benches/ref_type_write_static_size.x86-64.mca b/rust/zerocopy/benches/ref_type_write_static_size.x86-64.mca new file mode 100644 index 000000000000..a7089d3b1336 --- /dev/null +++ b/rust/zerocopy/benches/ref_type_write_static_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 208 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 2.40 +IPC: 2.40 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 1.00 * mov dword ptr [rax], esi + 1 1 0.50 shr rsi, 32 + 1 1 1.00 * mov word ptr [rax + 4], si + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 - 2.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - 1.00 - 0.33 0.67 mov dword ptr [rax], esi + - - 0.99 - - 0.01 - - shr rsi, 32 + - - - - 1.00 - 0.67 0.33 mov word ptr [rax + 4], si + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 index 03684cb09f8f..0a8a4f501a07 100644 --- a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 @@ -1,9 +1,8 @@ bench_split_via_runtime_check_dynamic_padding: mov rax, rdi mov rdx, qword ptr [rsi + 16] - mov ecx, edx - and ecx, 3 - cmp ecx, 1 + lea ecx, [rdx - 1] + test cl, 3 jne .LBB5_1 mov rcx, qword ptr [rsi] mov rsi, qword ptr [rsi + 8] diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca index 5034ab0583bd..7140b06e8842 100644 --- a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca @@ -1,11 +1,11 @@ Iterations: 100 -Instructions: 2000 +Instructions: 1900 Total Cycles: 708 -Total uOps: 2000 +Total uOps: 1900 Dispatch Width: 4 -uOps Per Cycle: 2.82 -IPC: 2.82 +uOps Per Cycle: 2.68 +IPC: 2.68 Block RThroughput: 5.0 @@ -20,9 +20,8 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 mov rax, rdi 1 5 0.50 * mov rdx, qword ptr [rsi + 16] - 1 1 0.33 mov ecx, edx - 1 1 0.33 and ecx, 3 - 1 1 0.33 cmp ecx, 1 + 1 1 0.50 lea ecx, [rdx - 1] + 1 1 0.33 test cl, 3 1 1 1.00 jne .LBB5_1 1 5 0.50 * mov rcx, qword ptr [rsi] 1 5 0.50 * mov rsi, qword ptr [rsi + 8] @@ -53,22 +52,21 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 3.00 3.02 5.00 4.98 4.00 4.00 + - - 2.99 3.00 5.00 4.01 4.00 4.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 0.99 - 0.01 - - mov rax, rdi + - - 0.99 - - 0.01 - - mov rax, rdi - - - - - - - 1.00 mov rdx, qword ptr [rsi + 16] - - - 0.99 0.01 - - - - mov ecx, edx - - - 0.99 0.01 - - - - and ecx, 3 - - - 0.97 0.03 - - - - cmp ecx, 1 + - - - 1.00 - - - - lea ecx, [rdx - 1] + - - - 0.98 - 0.02 - - test cl, 3 - - - - - 1.00 - - jne .LBB5_1 - - - - - - 1.00 - mov rcx, qword ptr [rsi] - - - - - - 0.99 0.01 mov rsi, qword ptr [rsi + 8] - - - 0.01 0.99 - - - - lea rdi, [rdx + 2*rdx] - - - - 0.96 - 0.04 - - add rdi, rcx - - - 0.03 - - 0.97 - - add rdi, 9 - - - 0.01 0.03 - 0.96 - - sub rsi, rdx + - - 1.00 - - - - - lea rdi, [rdx + 2*rdx] + - - 0.01 0.01 - 0.98 - - add rdi, rcx + - - - 1.00 - - - - add rdi, 9 + - - 0.99 0.01 - - - - sub rsi, rdx - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rdi - - - - 1.00 - 0.01 0.99 mov qword ptr [rax + 24], rsi diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64 b/rust/zerocopy/benches/try_read_from_bytes.x86-64 index 08088a08fd85..2135facc2655 100644 --- a/rust/zerocopy/benches/try_read_from_bytes.x86-64 +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64 @@ -1,23 +1,11 @@ bench_try_read_from_bytes_static_size: - mov ax, -16191 + mov eax, 49345 cmp rsi, 6 - jne .LBB5_1 - mov ecx, dword ptr [rdi] - movzx edx, cx - cmp edx, 49344 - jne .LBB5_4 - movzx eax, word ptr [rdi + 4] - shl rax, 32 - or rcx, rax - shr rcx, 16 - mov ax, -16192 -.LBB5_4: - shl rcx, 16 - movzx eax, ax - or rax, rcx - ret -.LBB5_1: - shl rcx, 16 - movzx eax, ax - or rax, rcx + jne .LBB5_3 + cmp word ptr [rdi], -16192 + jne .LBB5_3 + mov eax, dword ptr [rdi + 2] + shl rax, 16 + or rax, 49344 +.LBB5_3: ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca index 385e6a480253..f5568fef5ae7 100644 --- a/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2000 -Total Cycles: 608 -Total uOps: 2000 +Instructions: 900 +Total Cycles: 305 +Total uOps: 1000 Dispatch Width: 4 -uOps Per Cycle: 3.29 -IPC: 3.29 -Block RThroughput: 5.0 +uOps Per Cycle: 3.28 +IPC: 2.95 +Block RThroughput: 3.0 Instruction Info: @@ -18,25 +18,14 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 mov ax, -16191 + 1 1 0.33 mov eax, 49345 1 1 0.33 cmp rsi, 6 - 1 1 1.00 jne .LBB5_1 - 1 5 0.50 * mov ecx, dword ptr [rdi] - 1 1 0.33 movzx edx, cx - 1 1 0.33 cmp edx, 49344 - 1 1 1.00 jne .LBB5_4 - 1 5 0.50 * movzx eax, word ptr [rdi + 4] - 1 1 0.50 shl rax, 32 - 1 1 0.33 or rcx, rax - 1 1 0.50 shr rcx, 16 - 1 1 0.33 mov ax, -16192 - 1 1 0.50 shl rcx, 16 - 1 1 0.33 movzx eax, ax - 1 1 0.33 or rax, rcx - 1 1 1.00 U ret - 1 1 0.50 shl rcx, 16 - 1 1 0.33 movzx eax, ax - 1 1 0.33 or rax, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rdi], -16192 + 1 1 1.00 jne .LBB5_3 + 1 5 0.50 * mov eax, dword ptr [rdi + 2] + 1 1 0.50 shl rax, 16 + 1 1 0.33 or rax, 49344 1 1 1.00 U ret @@ -53,27 +42,16 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 5.99 5.99 - 6.02 1.00 1.00 + - - 2.50 2.49 - 3.01 1.00 1.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 0.99 - 0.01 - - mov ax, -16191 - - - - 0.01 - 0.99 - - cmp rsi, 6 - - - - - - 1.00 - - jne .LBB5_1 - - - - - - - - 1.00 mov ecx, dword ptr [rdi] - - - 0.98 - - 0.02 - - movzx edx, cx - - - 0.99 0.01 - - - - cmp edx, 49344 - - - - - - 1.00 - - jne .LBB5_4 - - - - - - - 1.00 - movzx eax, word ptr [rdi + 4] - - - 0.01 - - 0.99 - - shl rax, 32 - - - 0.02 0.98 - - - - or rcx, rax - - - 1.00 - - - - - shr rcx, 16 - - - 0.99 0.01 - - - - mov ax, -16192 - - - 1.00 - - - - - shl rcx, 16 - - - - 1.00 - - - - movzx eax, ax - - - - 1.00 - - - - or rax, rcx - - - - - - 1.00 - - ret - - - 1.00 - - - - - shl rcx, 16 - - - - 1.00 - - - - movzx eax, ax - - - - 0.99 - 0.01 - - or rax, rcx + - - 0.50 0.49 - 0.01 - - mov eax, 49345 + - - 0.49 0.51 - - - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.47 0.53 - - 0.43 0.57 cmp word ptr [rdi], -16192 + - - - - - 1.00 - - jne .LBB5_3 + - - - - - - 0.57 0.43 mov eax, dword ptr [rdi + 2] + - - 1.00 - - - - - shl rax, 16 + - - 0.04 0.96 - - - - or rax, 49344 - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64 b/rust/zerocopy/benches/try_read_from_prefix.x86-64 index d3e1edc3ea73..0f820efff5f8 100644 --- a/rust/zerocopy/benches/try_read_from_prefix.x86-64 +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64 @@ -1,16 +1,11 @@ bench_try_read_from_prefix_static_size: mov eax, 49345 cmp rsi, 6 - jb .LBB5_2 - mov eax, dword ptr [rdi] - movzx ecx, word ptr [rdi + 4] - shl rcx, 32 - or rcx, rax - movzx eax, cx - and rcx, -65536 - or rcx, 49344 - cmp eax, 49344 - mov eax, 49345 - cmove rax, rcx -.LBB5_2: + jb .LBB5_3 + cmp word ptr [rdi], -16192 + jne .LBB5_3 + mov eax, dword ptr [rdi + 2] + shl rax, 16 + or rax, 49344 +.LBB5_3: ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca index 40401d89e83d..e0ba4c8320e3 100644 --- a/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1400 -Total Cycles: 442 -Total uOps: 1500 +Instructions: 900 +Total Cycles: 305 +Total uOps: 1000 Dispatch Width: 4 -uOps Per Cycle: 3.39 -IPC: 3.17 -Block RThroughput: 3.8 +uOps Per Cycle: 3.28 +IPC: 2.95 +Block RThroughput: 3.0 Instruction Info: @@ -20,17 +20,12 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 mov eax, 49345 1 1 0.33 cmp rsi, 6 - 1 1 1.00 jb .LBB5_2 - 1 5 0.50 * mov eax, dword ptr [rdi] - 1 5 0.50 * movzx ecx, word ptr [rdi + 4] - 1 1 0.50 shl rcx, 32 - 1 1 0.33 or rcx, rax - 1 1 0.33 movzx eax, cx - 1 1 0.33 and rcx, -65536 - 1 1 0.33 or rcx, 49344 - 1 1 0.33 cmp eax, 49344 - 1 1 0.33 mov eax, 49345 - 2 2 0.67 cmove rax, rcx + 1 1 1.00 jb .LBB5_3 + 2 6 0.50 * cmp word ptr [rdi], -16192 + 1 1 1.00 jne .LBB5_3 + 1 5 0.50 * mov eax, dword ptr [rdi + 2] + 1 1 0.50 shl rax, 16 + 1 1 0.33 or rax, 49344 1 1 1.00 U ret @@ -47,21 +42,16 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 4.33 4.33 - 4.34 1.00 1.00 + - - 2.50 2.49 - 3.01 1.00 1.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.65 0.01 - 0.34 - - mov eax, 49345 - - - 0.01 0.33 - 0.66 - - cmp rsi, 6 - - - - - - 1.00 - - jb .LBB5_2 - - - - - - - - 1.00 mov eax, dword ptr [rdi] - - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] - - - 0.65 - - 0.35 - - shl rcx, 32 - - - - 0.67 - 0.33 - - or rcx, rax - - - 0.01 0.99 - - - - movzx eax, cx - - - 0.99 0.01 - - - - and rcx, -65536 - - - 0.01 0.99 - - - - or rcx, 49344 - - - 0.99 0.01 - - - - cmp eax, 49344 - - - 0.02 0.33 - 0.65 - - mov eax, 49345 - - - 1.00 0.99 - 0.01 - - cmove rax, rcx + - - 0.50 0.49 - 0.01 - - mov eax, 49345 + - - 0.49 0.51 - - - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_3 + - - 0.47 0.53 - - 0.43 0.57 cmp word ptr [rdi], -16192 + - - - - - 1.00 - - jne .LBB5_3 + - - - - - - 0.57 0.43 mov eax, dword ptr [rdi + 2] + - - 1.00 - - - - - shl rax, 16 + - - 0.04 0.96 - - - - or rax, 49344 - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64 b/rust/zerocopy/benches/try_read_from_suffix.x86-64 index 095e326f0467..a91c074e1806 100644 --- a/rust/zerocopy/benches/try_read_from_suffix.x86-64 +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64 @@ -1,18 +1,11 @@ bench_try_read_from_suffix_static_size: mov eax, 49345 cmp rsi, 6 - jb .LBB5_2 - mov eax, dword ptr [rdi + rsi - 6] - movzx ecx, word ptr [rdi + rsi - 2] - shl rcx, 32 - or rcx, rax - movzx edx, cx - xor eax, eax - cmp edx, 49344 - cmovne rcx, rsi - sete al - and rcx, -65536 - xor rax, 49345 - or rax, rcx -.LBB5_2: + jb .LBB5_3 + cmp word ptr [rdi + rsi - 6], -16192 + jne .LBB5_3 + mov eax, dword ptr [rdi + rsi - 4] + shl rax, 16 + or rax, 49344 +.LBB5_3: ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca index d3eaadbb8a81..e42f6b93effb 100644 --- a/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1600 -Total Cycles: 478 -Total uOps: 1700 +Instructions: 900 +Total Cycles: 305 +Total uOps: 1000 Dispatch Width: 4 -uOps Per Cycle: 3.56 -IPC: 3.35 -Block RThroughput: 4.3 +uOps Per Cycle: 3.28 +IPC: 2.95 +Block RThroughput: 3.0 Instruction Info: @@ -20,19 +20,12 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 mov eax, 49345 1 1 0.33 cmp rsi, 6 - 1 1 1.00 jb .LBB5_2 - 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] - 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] - 1 1 0.50 shl rcx, 32 - 1 1 0.33 or rcx, rax - 1 1 0.33 movzx edx, cx - 1 0 0.25 xor eax, eax - 1 1 0.33 cmp edx, 49344 - 2 2 0.67 cmovne rcx, rsi - 1 1 0.50 sete al - 1 1 0.33 and rcx, -65536 - 1 1 0.33 xor rax, 49345 - 1 1 0.33 or rax, rcx + 1 1 1.00 jb .LBB5_3 + 2 6 0.50 * cmp word ptr [rdi + rsi - 6], -16192 + 1 1 1.00 jne .LBB5_3 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 4] + 1 1 0.50 shl rax, 16 + 1 1 0.33 or rax, 49344 1 1 1.00 U ret @@ -49,23 +42,16 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 4.66 4.66 - 4.68 1.00 1.00 + - - 2.50 2.49 - 3.01 1.00 1.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.32 0.01 - 0.67 - - mov eax, 49345 - - - 0.62 0.02 - 0.36 - - cmp rsi, 6 - - - - - - 1.00 - - jb .LBB5_2 - - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] - - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] - - - 0.37 - - 0.63 - - shl rcx, 32 - - - 0.99 0.01 - - - - or rcx, rax - - - 1.00 - - - - - movzx edx, cx - - - - - - - - - xor eax, eax - - - 0.35 0.64 - 0.01 - - cmp edx, 49344 - - - 1.00 1.00 - - - - cmovne rcx, rsi - - - - - - 1.00 - - sete al - - - 0.01 0.99 - - - - and rcx, -65536 - - - - 1.00 - - - - xor rax, 49345 - - - - 0.99 - 0.01 - - or rax, rcx + - - 0.50 0.49 - 0.01 - - mov eax, 49345 + - - 0.49 0.51 - - - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_3 + - - 0.47 0.53 - - 0.43 0.57 cmp word ptr [rdi + rsi - 6], -16192 + - - - - - 1.00 - - jne .LBB5_3 + - - - - - - 0.57 0.43 mov eax, dword ptr [rdi + rsi - 4] + - - 1.00 - - - - - shl rax, 16 + - - 0.04 0.96 - - - - or rax, 49344 - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 index 217c5fc61796..5584e9054185 100644 --- a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 @@ -1,24 +1,22 @@ bench_try_ref_from_bytes_dynamic_padding: - test dil, 3 - jne .LBB5_4 - movabs rax, 9223372036854775804 - and rax, rsi - cmp rax, 9 - jb .LBB5_4 - add rax, -9 + cmp rsi, 12 + setae al + mov ecx, esi + or ecx, edi + test cl, 3 + sete cl + and cl, al + cmp cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + add rsi, -9 movabs rcx, -6148914691236517205 + mov rax, rsi mul rcx shr rdx - lea rax, [rdx + 2*rdx] - or rax, 3 - add rax, 9 - cmp rsi, rax - jne .LBB5_4 + xor eax, eax cmp word ptr [rdi], -16192 - je .LBB5_5 -.LBB5_4: - xor edi, edi - mov rdx, rsi -.LBB5_5: - mov rax, rdi + cmove rax, rdi ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca index 95b993c7e0a8..ccd610c7ef41 100644 --- a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca @@ -1,10 +1,10 @@ Iterations: 100 -Instructions: 2100 -Total Cycles: 709 +Instructions: 2000 +Total Cycles: 675 Total uOps: 2300 Dispatch Width: 4 -uOps Per Cycle: 3.24 +uOps Per Cycle: 3.41 IPC: 2.96 Block RThroughput: 5.8 @@ -18,26 +18,25 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 test dil, 3 - 1 1 1.00 jne .LBB5_4 - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rax, rsi - 1 1 0.33 cmp rax, 9 - 1 1 1.00 jb .LBB5_4 - 1 1 0.33 add rax, -9 + 1 1 0.33 cmp rsi, 12 + 1 1 0.50 setae al + 1 1 0.33 mov ecx, esi + 1 1 0.33 or ecx, edi + 1 1 0.33 test cl, 3 + 1 1 0.50 sete cl + 1 1 0.33 and cl, al + 1 1 0.33 cmp cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi 2 4 1.00 mul rcx 1 1 0.50 shr rdx - 1 1 0.50 lea rax, [rdx + 2*rdx] - 1 1 0.33 or rax, 3 - 1 1 0.33 add rax, 9 - 1 1 0.33 cmp rsi, rax - 1 1 1.00 jne .LBB5_4 + 1 0 0.25 xor eax, eax 2 6 0.50 * cmp word ptr [rdi], -16192 - 1 1 1.00 je .LBB5_5 - 1 0 0.25 xor edi, edi - 1 1 0.33 mov rdx, rsi - 1 1 0.33 mov rax, rdi + 2 2 0.67 cmove rax, rdi 1 1 1.00 U ret @@ -54,28 +53,27 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.98 6.99 - 7.03 0.50 0.50 + - - 6.67 6.66 - 6.67 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.48 0.51 - 0.01 - - test dil, 3 - - - - - - 1.00 - - jne .LBB5_4 - - - 0.51 0.49 - - - - movabs rax, 9223372036854775804 - - - 0.01 0.99 - - - - and rax, rsi - - - 0.51 0.49 - - - - cmp rax, 9 - - - - - - 1.00 - - jb .LBB5_4 - - - 0.98 - - 0.02 - - add rax, -9 - - - 0.98 0.02 - - - - movabs rcx, -6148914691236517205 - - - 1.00 1.00 - - - - mul rcx - - - 0.99 - - 0.01 - - shr rdx - - - - 1.00 - - - - lea rax, [rdx + 2*rdx] - - - - 0.51 - 0.49 - - or rax, 3 - - - 0.01 0.49 - 0.50 - - add rax, 9 - - - - 0.02 - 0.98 - - cmp rsi, rax - - - - - - 1.00 - - jne .LBB5_4 - - - 0.51 0.49 - - 0.50 0.50 cmp word ptr [rdi], -16192 - - - - - - 1.00 - - je .LBB5_5 - - - - - - - - - xor edi, edi - - - 0.50 0.50 - - - - mov rdx, rsi - - - 0.50 0.48 - 0.02 - - mov rax, rdi + - - 0.33 0.66 - 0.01 - - cmp rsi, 12 + - - 1.00 - - - - - setae al + - - 0.66 0.34 - - - - mov ecx, esi + - - - 0.99 - 0.01 - - or ecx, edi + - - - 0.67 - 0.33 - - test cl, 3 + - - 0.67 - - 0.33 - - sete cl + - - 0.33 0.34 - 0.33 - - and cl, al + - - 0.33 - - 0.67 - - cmp cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.67 0.33 - - - - add rsi, -9 + - - 0.99 0.01 - - - - movabs rcx, -6148914691236517205 + - - - 0.99 - 0.01 - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 0.01 - - 0.99 - - shr rdx + - - - - - - - - xor eax, eax + - - 0.67 0.33 - - 0.50 0.50 cmp word ptr [rdi], -16192 + - - 0.01 1.00 - 0.99 - - cmove rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 index cf67afd31ce0..39b12e14eec1 100644 --- a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 @@ -1,22 +1,21 @@ bench_try_ref_from_bytes_dynamic_size: - mov rdx, rsi - mov rax, rdi cmp rsi, 4 - setb cl - or cl, al + setae al + mov ecx, esi + or ecx, edi test cl, 1 - jne .LBB5_4 - lea rcx, [rdx - 4] - mov rsi, rcx - and rsi, -2 - add rsi, 4 - cmp rdx, rsi - jne .LBB5_4 - cmp word ptr [rax], -16192 - jne .LBB5_4 - shr rcx - mov rdx, rcx - ret -.LBB5_4: + sete cl + and cl, al + cmp cl, 1 + je .LBB5_2 xor eax, eax ret +.LBB5_2: + lea rdx, [rsi - 4] + shr rdx + movzx ecx, word ptr [rdi] + xor eax, eax + cmp ecx, 49344 + cmovne rdx, rsi + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca index ecd7a18f6d6d..eecf3a1554cb 100644 --- a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca @@ -1,10 +1,10 @@ Iterations: 100 -Instructions: 2000 -Total Cycles: 639 +Instructions: 1900 +Total Cycles: 607 Total uOps: 2100 Dispatch Width: 4 -uOps Per Cycle: 3.29 +uOps Per Cycle: 3.46 IPC: 3.13 Block RThroughput: 5.3 @@ -18,26 +18,25 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 mov rdx, rsi - 1 1 0.33 mov rax, rdi 1 1 0.33 cmp rsi, 4 - 1 1 0.50 setb cl - 1 1 0.33 or cl, al + 1 1 0.50 setae al + 1 1 0.33 mov ecx, esi + 1 1 0.33 or ecx, edi 1 1 0.33 test cl, 1 - 1 1 1.00 jne .LBB5_4 - 1 1 0.50 lea rcx, [rdx - 4] - 1 1 0.33 mov rsi, rcx - 1 1 0.33 and rsi, -2 - 1 1 0.33 add rsi, 4 - 1 1 0.33 cmp rdx, rsi - 1 1 1.00 jne .LBB5_4 - 2 6 0.50 * cmp word ptr [rax], -16192 - 1 1 1.00 jne .LBB5_4 - 1 1 0.50 shr rcx - 1 1 0.33 mov rdx, rcx - 1 1 1.00 U ret + 1 1 0.50 sete cl + 1 1 0.33 and cl, al + 1 1 0.33 cmp cl, 1 + 1 1 1.00 je .LBB5_2 1 0 0.25 xor eax, eax 1 1 1.00 U ret + 1 1 0.50 lea rdx, [rsi - 4] + 1 1 0.50 shr rdx + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 2 2 0.67 cmovne rdx, rsi + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret Resources: @@ -53,27 +52,26 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.32 6.32 - 6.36 0.50 0.50 + - - 5.99 5.99 - 6.02 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.33 0.66 - 0.01 - - mov rdx, rsi - - - 0.66 0.34 - - - - mov rax, rdi - - - 0.34 0.66 - - - - cmp rsi, 4 - - - 0.99 - - 0.01 - - setb cl - - - 0.01 0.99 - - - - or cl, al - - - - 1.00 - - - - test cl, 1 - - - - - - 1.00 - - jne .LBB5_4 - - - 0.66 0.34 - - - - lea rcx, [rdx - 4] - - - 0.33 0.66 - 0.01 - - mov rsi, rcx - - - 1.00 - - - - - and rsi, -2 - - - 0.66 0.34 - - - - add rsi, 4 - - - - 1.00 - - - - cmp rdx, rsi - - - - - - 1.00 - - jne .LBB5_4 - - - - - - 1.00 0.50 0.50 cmp word ptr [rax], -16192 - - - - - - 1.00 - - jne .LBB5_4 - - - 0.67 - - 0.33 - - shr rcx - - - 0.67 0.33 - - - - mov rdx, rcx - - - - - - 1.00 - - ret + - - 0.02 0.97 - 0.01 - - cmp rsi, 4 + - - 1.00 - - - - - setae al + - - - 1.00 - - - - mov ecx, esi + - - 0.93 0.05 - 0.02 - - or ecx, edi + - - 0.94 0.06 - - - - test cl, 1 + - - 0.99 - - 0.01 - - sete cl + - - 0.98 0.02 - - - - and cl, al + - - 0.98 0.02 - - - - cmp cl, 1 + - - - - - 1.00 - - je .LBB5_2 - - - - - - - - xor eax, eax - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - lea rdx, [rsi - 4] + - - - - - 1.00 - - shr rdx + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - - - - - - - xor eax, eax + - - 0.02 0.96 - 0.02 - - cmp ecx, 49344 + - - 0.05 0.97 - 0.98 - - cmovne rdx, rsi + - - 0.07 0.95 - 0.98 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 index d832cb7ecf7f..ff64bb659591 100644 --- a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 @@ -1,19 +1,18 @@ bench_try_ref_from_prefix_dynamic_padding: - xor edx, edx - mov eax, 0 test dil, 3 - je .LBB5_1 - ret -.LBB5_1: - movabs rax, 9223372036854775804 - and rsi, rax - cmp rsi, 9 - jae .LBB5_3 - mov edx, 1 + sete al + cmp rsi, 12 + setae cl + test al, cl + jne .LBB5_2 + xor edx, edx + test dil, 3 + sete dl xor eax, eax ret -.LBB5_3: - add rsi, -9 +.LBB5_2: + or rsi, 3 + add rsi, -12 movabs rcx, -6148914691236517205 mov rax, rsi mul rcx diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca index 482112a39b33..29225404a28f 100644 --- a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca @@ -18,19 +18,19 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 0 0.25 xor edx, edx - 1 1 0.33 mov eax, 0 1 1 0.33 test dil, 3 - 1 1 1.00 je .LBB5_1 - 1 1 1.00 U ret - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rsi, rax - 1 1 0.33 cmp rsi, 9 - 1 1 1.00 jae .LBB5_3 - 1 1 0.33 mov edx, 1 + 1 1 0.50 sete al + 1 1 0.33 cmp rsi, 12 + 1 1 0.50 setae cl + 1 1 0.33 test al, cl + 1 1 1.00 jne .LBB5_2 + 1 0 0.25 xor edx, edx + 1 1 0.33 test dil, 3 + 1 1 0.50 sete dl 1 0 0.25 xor eax, eax 1 1 1.00 U ret - 1 1 0.33 add rsi, -9 + 1 1 0.33 or rsi, 3 + 1 1 0.33 add rsi, -12 1 1 0.33 movabs rcx, -6148914691236517205 1 1 0.33 mov rax, rsi 2 4 1.00 mul rcx @@ -63,29 +63,29 @@ Resource pressure per iteration: Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - test dil, 3 + - - 0.98 - - 0.02 - - sete al + - - - 1.00 - - - - cmp rsi, 12 + - - - - - 1.00 - - setae cl + - - 0.01 0.03 - 0.96 - - test al, cl + - - - - - 1.00 - - jne .LBB5_2 - - - - - - - - xor edx, edx - - - 0.32 0.34 - 0.34 - - mov eax, 0 - - - 0.34 0.33 - 0.33 - - test dil, 3 - - - - - - 1.00 - - je .LBB5_1 - - - - - - 1.00 - - ret - - - 0.35 0.65 - - - - movabs rax, 9223372036854775804 - - - 0.96 0.03 - 0.01 - - and rsi, rax - - - 0.01 0.97 - 0.02 - - cmp rsi, 9 - - - - - - 1.00 - - jae .LBB5_3 - - - 0.67 0.01 - 0.32 - - mov edx, 1 + - - - 1.00 - - - - test dil, 3 + - - 0.99 - - 0.01 - - sete dl - - - - - - - - xor eax, eax - - - - - 1.00 - - ret - - - 0.02 0.34 - 0.64 - - add rsi, -9 - - - 0.33 0.66 - 0.01 - - movabs rcx, -6148914691236517205 - - - 0.66 0.34 - - - - mov rax, rsi + - - - 1.00 - - - - or rsi, 3 + - - 0.01 0.99 - - - - add rsi, -12 + - - 0.01 0.99 - - - - movabs rcx, -6148914691236517205 + - - 0.34 0.66 - - - - mov rax, rsi - - 1.00 1.00 - - - - mul rcx - - - 0.01 0.99 - - - - mov rax, rdx - - - 0.99 - - 0.01 - - shr rax + - - 0.99 - - 0.01 - - mov rax, rdx + - - 1.00 - - - - - shr rax - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] - - - 0.33 0.03 - 0.64 - - cmp cx, -16192 - - - 0.01 0.31 - 0.68 - - mov edx, 2 - - - 1.00 1.00 - - - - cmove rdx, rax + - - 0.34 0.32 - 0.34 - - cmp cx, -16192 + - - 0.33 0.01 - 0.66 - - mov edx, 2 + - - 0.99 0.01 - 1.00 - - cmove rdx, rax - - - - - - - - xor eax, eax - - 0.33 0.33 - 0.34 - - cmp ecx, 49344 - - - 1.00 1.00 - - - - cmove rax, rdi + - - 1.00 0.01 - 0.99 - - cmove rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 index be7f34b9f8f5..de31f571f3c9 100644 --- a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 @@ -1,14 +1,15 @@ bench_try_ref_from_prefix_dynamic_size: - xor edx, edx - mov eax, 0 - test dil, 1 - jne .LBB5_4 cmp rsi, 4 - jae .LBB5_3 - mov edx, 1 + setb al + or al, dil + test al, 1 + je .LBB5_2 + not edi + and edi, 1 xor eax, eax + mov rdx, rdi ret -.LBB5_3: +.LBB5_2: add rsi, -4 shr rsi movzx ecx, word ptr [rdi] @@ -18,5 +19,4 @@ bench_try_ref_from_prefix_dynamic_size: xor eax, eax cmp cx, -16192 cmove rax, rdi -.LBB5_4: ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca index 11706defe11e..cd0919d04c08 100644 --- a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1900 -Total Cycles: 573 -Total uOps: 2100 +Instructions: 2000 +Total Cycles: 643 +Total uOps: 2200 Dispatch Width: 4 -uOps Per Cycle: 3.66 -IPC: 3.32 -Block RThroughput: 5.3 +uOps Per Cycle: 3.42 +IPC: 3.11 +Block RThroughput: 5.5 Instruction Info: @@ -18,14 +18,15 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 0 0.25 xor edx, edx - 1 1 0.33 mov eax, 0 - 1 1 0.33 test dil, 1 - 1 1 1.00 jne .LBB5_4 1 1 0.33 cmp rsi, 4 - 1 1 1.00 jae .LBB5_3 - 1 1 0.33 mov edx, 1 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 je .LBB5_2 + 1 1 0.33 not edi + 1 1 0.33 and edi, 1 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rdi 1 1 1.00 U ret 1 1 0.33 add rsi, -4 1 1 0.50 shr rsi @@ -52,26 +53,27 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 5.66 5.67 - 5.67 0.50 0.50 + - - 6.33 6.33 - 6.34 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - - - - - - xor edx, edx - - - 0.30 0.37 - 0.33 - - mov eax, 0 - - - 0.35 0.32 - 0.33 - - test dil, 1 - - - - - - 1.00 - - jne .LBB5_4 - - - 0.32 0.33 - 0.35 - - cmp rsi, 4 - - - - - - 1.00 - - jae .LBB5_3 - - - 0.33 0.35 - 0.32 - - mov edx, 1 + - - 0.67 0.32 - 0.01 - - cmp rsi, 4 + - - 0.99 - - 0.01 - - setb al + - - 0.19 0.48 - 0.33 - - or al, dil + - - 0.50 0.16 - 0.34 - - test al, 1 + - - - - - 1.00 - - je .LBB5_2 + - - 0.50 0.50 - - - - not edi + - - 0.49 0.34 - 0.17 - - and edi, 1 - - - - - - - - xor eax, eax + - - 0.16 0.84 - - - - mov rdx, rdi - - - - - 1.00 - - ret - - - 0.34 0.64 - 0.02 - - add rsi, -4 - - - 1.00 - - - - - shr rsi + - - 0.68 0.32 - - - - add rsi, -4 + - - 0.85 - - 0.15 - - shr rsi - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] - - - 0.60 0.40 - - - - cmp ecx, 49344 - - - 0.05 0.95 - - - - mov edx, 2 - - - 1.00 1.00 - - - - cmove rdx, rsi + - - - 0.50 - 0.50 - - cmp ecx, 49344 + - - 0.63 0.37 - - - - mov edx, 2 + - - 0.17 1.00 - 0.83 - - cmove rdx, rsi - - - - - - - - xor eax, eax - - - 0.37 0.31 - 0.32 - - cmp cx, -16192 - - - 1.00 1.00 - - - - cmove rax, rdi + - - 0.50 0.50 - - - - cmp cx, -16192 + - - - 1.00 - 1.00 - - cmove rax, rdi - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 index b3e924442865..cee9a3179097 100644 --- a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 @@ -1,26 +1,23 @@ bench_try_ref_from_suffix_dynamic_padding: lea eax, [rsi + rdi] test al, 3 - jne .LBB5_1 - movabs rax, 9223372036854775804 - and rax, rsi - cmp rax, 9 - jae .LBB5_3 -.LBB5_1: + setne al + cmp rsi, 12 + setb cl + or cl, al + je .LBB5_2 xor eax, eax ret -.LBB5_3: - add rax, -9 +.LBB5_2: + mov rax, rsi + or rax, 3 + add rax, -12 movabs rcx, -6148914691236517205 mul rcx shr rdx - lea rcx, [rdx + 2*rdx] - sub rsi, rcx - or rcx, -4 - add rsi, rdi - lea rdi, [rcx + rsi] - add rdi, -8 + and esi, 3 + lea rcx, [rdi + rsi] xor eax, eax - cmp word ptr [rcx + rsi - 8], -16192 - cmove rax, rdi + cmp word ptr [rdi + rsi], -16192 + cmove rax, rcx ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca index d56ae56d854a..92df4484b823 100644 --- a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2300 -Total Cycles: 791 -Total uOps: 2600 +Instructions: 2100 +Total Cycles: 710 +Total uOps: 2400 Dispatch Width: 4 -uOps Per Cycle: 3.29 -IPC: 2.91 -Block RThroughput: 6.5 +uOps Per Cycle: 3.38 +IPC: 2.96 +Block RThroughput: 6.0 Instruction Info: @@ -20,26 +20,24 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.50 lea eax, [rsi + rdi] 1 1 0.33 test al, 3 - 1 1 1.00 jne .LBB5_1 - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rax, rsi - 1 1 0.33 cmp rax, 9 - 1 1 1.00 jae .LBB5_3 + 1 1 0.50 setne al + 1 1 0.33 cmp rsi, 12 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 1.00 je .LBB5_2 1 0 0.25 xor eax, eax 1 1 1.00 U ret - 1 1 0.33 add rax, -9 + 1 1 0.33 mov rax, rsi + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, -12 1 1 0.33 movabs rcx, -6148914691236517205 2 4 1.00 mul rcx 1 1 0.50 shr rdx - 1 1 0.50 lea rcx, [rdx + 2*rdx] - 1 1 0.33 sub rsi, rcx - 1 1 0.33 or rcx, -4 - 1 1 0.33 add rsi, rdi - 1 1 0.50 lea rdi, [rcx + rsi] - 1 1 0.33 add rdi, -8 + 1 1 0.33 and esi, 3 + 1 1 0.50 lea rcx, [rdi + rsi] 1 0 0.25 xor eax, eax - 2 6 0.50 * cmp word ptr [rcx + rsi - 8], -16192 - 2 2 0.67 cmove rax, rdi + 2 6 0.50 * cmp word ptr [rdi + rsi], -16192 + 2 2 0.67 cmove rax, rcx 1 1 1.00 U ret @@ -56,30 +54,28 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 7.70 7.58 - 7.72 0.50 0.50 + - - 7.00 7.00 - 7.00 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.26 0.74 - - - - lea eax, [rsi + rdi] - - - 0.19 0.28 - 0.53 - - test al, 3 - - - - - - 1.00 - - jne .LBB5_1 - - - 0.93 0.06 - 0.01 - - movabs rax, 9223372036854775804 - - - 0.81 0.14 - 0.05 - - and rax, rsi - - - 0.55 0.43 - 0.02 - - cmp rax, 9 - - - - - - 1.00 - - jae .LBB5_3 + - - 0.99 0.01 - - - - lea eax, [rsi + rdi] + - - 0.01 0.99 - - - - test al, 3 + - - 0.98 - - 0.02 - - setne al + - - 0.99 - - 0.01 - - cmp rsi, 12 + - - 0.02 - - 0.98 - - setb cl + - - - 0.03 - 0.97 - - or cl, al + - - - - - 1.00 - - je .LBB5_2 - - - - - - - - xor eax, eax - - - - - 1.00 - - ret - - - 0.42 0.56 - 0.02 - - add rax, -9 - - - 0.67 0.30 - 0.03 - - movabs rcx, -6148914691236517205 + - - - 1.00 - - - - mov rax, rsi + - - 0.03 0.97 - - - - or rax, 3 + - - - 1.00 - - - - add rax, -12 + - - 0.99 - - 0.01 - - movabs rcx, -6148914691236517205 - - 1.00 1.00 - - - - mul rcx - - - 0.71 - - 0.29 - - shr rdx - - - 0.32 0.68 - - - - lea rcx, [rdx + 2*rdx] - - - 0.57 0.04 - 0.39 - - sub rsi, rcx - - - 0.28 0.67 - 0.05 - - or rcx, -4 - - - 0.29 0.29 - 0.42 - - add rsi, rdi - - - 0.02 0.98 - - - - lea rdi, [rcx + rsi] - - - 0.02 0.41 - 0.57 - - add rdi, -8 + - - 0.01 - - 0.99 - - shr rdx + - - 1.00 - - - - - and esi, 3 + - - 0.97 0.03 - - - - lea rcx, [rdi + rsi] - - - - - - - - xor eax, eax - - - 0.57 0.01 - 0.42 0.50 0.50 cmp word ptr [rcx + rsi - 8], -16192 - - - 0.09 0.99 - 0.92 - - cmove rax, rdi + - - - 0.97 - 0.03 0.50 0.50 cmp word ptr [rdi + rsi], -16192 + - - 0.01 1.00 - 0.99 - - cmove rax, rcx - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 index c7530d8b6815..ca483dee63ba 100644 --- a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -1,32 +1,23 @@ bench_try_ref_from_suffix_with_elems_dynamic_padding: movabs rax, 3074457345618258598 cmp rdx, rax - ja .LBB5_1 - lea r8d, [rsi + rdi] - xor ecx, ecx - mov eax, 0 - test r8b, 3 - je .LBB5_3 - mov rdx, rcx - ret -.LBB5_3: + ja .LBB5_3 lea rax, [rdx + 2*rdx] or rax, 3 add rax, 9 + lea ecx, [rsi + rdi] + test cl, 3 + setne cl sub rsi, rax - jae .LBB5_4 -.LBB5_1: + setb al + or al, cl + je .LBB5_4 +.LBB5_3: xor eax, eax - mov edx, 1 ret .LBB5_4: - lea r8, [rdi + rsi] - movzx esi, word ptr [rdi + rsi] - cmp si, -16192 - mov ecx, 2 - cmove rcx, rdx + lea rcx, [rdi + rsi] xor eax, eax - cmp esi, 49344 - cmove rax, r8 - mov rdx, rcx + cmp word ptr [rdi + rsi], -16192 + cmove rax, rcx ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca index be736c00c250..c37a42c160f7 100644 --- a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2800 -Total Cycles: 878 -Total uOps: 3000 +Instructions: 2000 +Total Cycles: 643 +Total uOps: 2200 Dispatch Width: 4 uOps Per Cycle: 3.42 -IPC: 3.19 -Block RThroughput: 7.5 +IPC: 3.11 +Block RThroughput: 5.5 Instruction Info: @@ -20,31 +20,23 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 movabs rax, 3074457345618258598 1 1 0.33 cmp rdx, rax - 1 1 1.00 ja .LBB5_1 - 1 1 0.50 lea r8d, [rsi + rdi] - 1 0 0.25 xor ecx, ecx - 1 1 0.33 mov eax, 0 - 1 1 0.33 test r8b, 3 - 1 1 1.00 je .LBB5_3 - 1 1 0.33 mov rdx, rcx - 1 1 1.00 U ret + 1 1 1.00 ja .LBB5_3 1 1 0.50 lea rax, [rdx + 2*rdx] 1 1 0.33 or rax, 3 1 1 0.33 add rax, 9 + 1 1 0.50 lea ecx, [rsi + rdi] + 1 1 0.33 test cl, 3 + 1 1 0.50 setne cl 1 1 0.33 sub rsi, rax - 1 1 1.00 jae .LBB5_4 + 1 1 0.50 setb al + 1 1 0.33 or al, cl + 1 1 1.00 je .LBB5_4 1 0 0.25 xor eax, eax - 1 1 0.33 mov edx, 1 1 1 1.00 U ret - 1 1 0.50 lea r8, [rdi + rsi] - 1 5 0.50 * movzx esi, word ptr [rdi + rsi] - 1 1 0.33 cmp si, -16192 - 1 1 0.33 mov ecx, 2 - 2 2 0.67 cmove rcx, rdx + 1 1 0.50 lea rcx, [rdi + rsi] 1 0 0.25 xor eax, eax - 1 1 0.33 cmp esi, 49344 - 2 2 0.67 cmove rax, r8 - 1 1 0.33 mov rdx, rcx + 2 6 0.50 * cmp word ptr [rdi + rsi], -16192 + 2 2 0.67 cmove rax, rcx 1 1 1.00 U ret @@ -61,35 +53,27 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 8.65 8.65 - 8.70 0.50 0.50 + - - 6.32 6.33 - 6.35 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.67 0.30 - 0.03 - - movabs rax, 3074457345618258598 - - - 0.01 0.99 - - - - cmp rdx, rax - - - - - - 1.00 - - ja .LBB5_1 - - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] - - - - - - - - - xor ecx, ecx - - - 0.35 0.62 - 0.03 - - mov eax, 0 - - - 0.99 0.01 - - - - test r8b, 3 - - - - - - 1.00 - - je .LBB5_3 - - - 0.68 0.30 - 0.02 - - mov rdx, rcx - - - - - - 1.00 - - ret - - - 0.07 0.93 - - - - lea rax, [rdx + 2*rdx] - - - 0.06 0.35 - 0.59 - - or rax, 3 - - - 0.02 0.07 - 0.91 - - add rax, 9 - - - 0.01 0.04 - 0.95 - - sub rsi, rax - - - - - - 1.00 - - jae .LBB5_4 + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.02 0.98 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_3 + - - 0.66 0.34 - - - - lea rax, [rdx + 2*rdx] + - - 0.99 - - 0.01 - - or rax, 3 + - - 0.99 0.01 - - - - add rax, 9 + - - 0.01 0.99 - - - - lea ecx, [rsi + rdi] + - - - 0.99 - 0.01 - - test cl, 3 + - - 0.02 - - 0.98 - - setne cl + - - 0.97 0.02 - 0.01 - - sub rsi, rax + - - 0.99 - - 0.01 - - setb al + - - 0.32 0.67 - 0.01 - - or al, cl + - - - - - 1.00 - - je .LBB5_4 - - - - - - - - xor eax, eax - - - 0.92 0.01 - 0.07 - - mov edx, 1 - - - - - 1.00 - - ret - - - - 1.00 - - - - lea r8, [rdi + rsi] - - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] - - - 0.01 0.99 - - - - cmp si, -16192 - - - 0.88 0.04 - 0.08 - - mov ecx, 2 - - - 1.00 0.99 - 0.01 - - cmove rcx, rdx + - - - 1.00 - - - - lea rcx, [rdi + rsi] - - - - - - - - xor eax, eax - - - 0.99 0.01 - - - - cmp esi, 49344 - - - 1.00 1.00 - - - - cmove rax, r8 - - - - 0.99 - 0.01 - - mov rdx, rcx + - - 0.35 0.32 - 0.33 0.50 0.50 cmp word ptr [rdi + rsi], -16192 + - - 1.00 0.02 - 0.98 - - cmove rax, rcx - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 index 952eb12de8d6..748eb1166e13 100644 --- a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -1,28 +1,20 @@ bench_try_ref_from_suffix_with_elems_dynamic_size: movabs rax, 4611686018427387901 cmp rdx, rax - ja .LBB5_1 - lea r8d, [rsi + rdi] - xor ecx, ecx - mov eax, 0 - test r8b, 1 - jne .LBB5_5 + ja .LBB5_3 lea rax, [2*rdx + 4] + lea ecx, [rsi + rdi] sub rsi, rax - jae .LBB5_4 -.LBB5_1: + setb al + or al, cl + test al, 1 + je .LBB5_4 +.LBB5_3: xor eax, eax - mov edx, 1 ret .LBB5_4: - lea r8, [rdi + rsi] - movzx esi, word ptr [rdi + rsi] - cmp si, -16192 - mov ecx, 2 - cmove rcx, rdx + lea rcx, [rdi + rsi] xor eax, eax - cmp esi, 49344 - cmove rax, r8 -.LBB5_5: - mov rdx, rcx + cmp word ptr [rdi + rsi], -16192 + cmove rax, rcx ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca index d4f78f67a252..53ff06192696 100644 --- a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2400 -Total Cycles: 1107 -Total uOps: 2600 +Instructions: 1700 +Total Cycles: 544 +Total uOps: 1900 Dispatch Width: 4 -uOps Per Cycle: 2.35 -IPC: 2.17 -Block RThroughput: 6.5 +uOps Per Cycle: 3.49 +IPC: 3.13 +Block RThroughput: 4.8 Instruction Info: @@ -20,27 +20,20 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 movabs rax, 4611686018427387901 1 1 0.33 cmp rdx, rax - 1 1 1.00 ja .LBB5_1 - 1 1 0.50 lea r8d, [rsi + rdi] - 1 0 0.25 xor ecx, ecx - 1 1 0.33 mov eax, 0 - 1 1 0.33 test r8b, 1 - 1 1 1.00 jne .LBB5_5 + 1 1 1.00 ja .LBB5_3 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.50 lea ecx, [rsi + rdi] 1 1 0.33 sub rsi, rax - 1 1 1.00 jae .LBB5_4 + 1 1 0.50 setb al + 1 1 0.33 or al, cl + 1 1 0.33 test al, 1 + 1 1 1.00 je .LBB5_4 1 0 0.25 xor eax, eax - 1 1 0.33 mov edx, 1 1 1 1.00 U ret - 1 1 0.50 lea r8, [rdi + rsi] - 1 5 0.50 * movzx esi, word ptr [rdi + rsi] - 1 1 0.33 cmp si, -16192 - 1 1 0.33 mov ecx, 2 - 2 2 0.67 cmove rcx, rdx + 1 1 0.50 lea rcx, [rdi + rsi] 1 0 0.25 xor eax, eax - 1 1 0.33 cmp esi, 49344 - 2 2 0.67 cmove rax, r8 - 1 1 0.33 mov rdx, rcx + 2 6 0.50 * cmp word ptr [rdi + rsi], -16192 + 2 2 0.67 cmove rax, rcx 1 1 1.00 U ret @@ -57,31 +50,24 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.99 7.00 - 8.01 0.50 0.50 + - - 5.33 5.33 - 5.34 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.02 0.95 - 0.03 - - movabs rax, 4611686018427387901 - - - 0.93 0.04 - 0.03 - - cmp rdx, rax - - - - - - 1.00 - - ja .LBB5_1 - - - 0.96 0.04 - - - - lea r8d, [rsi + rdi] - - - - - - - - - xor ecx, ecx - - - 0.95 0.02 - 0.03 - - mov eax, 0 - - - 0.95 0.05 - - - - test r8b, 1 - - - - - - 1.00 - - jne .LBB5_5 - - - 0.06 0.94 - - - - lea rax, [2*rdx + 4] - - - 0.93 0.07 - - - - sub rsi, rax - - - - - - 1.00 - - jae .LBB5_4 + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.02 0.98 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_3 + - - 0.98 0.02 - - - - lea rax, [2*rdx + 4] + - - 0.66 0.34 - - - - lea ecx, [rsi + rdi] + - - 0.33 0.65 - 0.02 - - sub rsi, rax + - - 0.99 - - 0.01 - - setb al + - - 0.01 0.36 - 0.63 - - or al, cl + - - 0.01 0.96 - 0.03 - - test al, 1 + - - - - - 1.00 - - je .LBB5_4 - - - - - - - - xor eax, eax - - - 0.03 0.95 - 0.02 - - mov edx, 1 - - - - - 1.00 - - ret - - - 0.97 0.03 - - - - lea r8, [rdi + rsi] - - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] - - - 0.03 0.97 - - - - cmp si, -16192 - - - 0.05 0.94 - 0.01 - - mov ecx, 2 - - - 0.06 0.98 - 0.96 - - cmove rcx, rdx + - - 0.32 0.68 - - - - lea rcx, [rdi + rsi] - - - - - - - - xor eax, eax - - - 0.97 0.03 - - - - cmp esi, 49344 - - - 0.06 0.96 - 0.98 - - cmove rax, r8 - - - 0.02 0.03 - 0.95 - - mov rdx, rcx + - - 0.36 0.02 - 0.62 0.50 0.50 cmp word ptr [rdi + rsi], -16192 + - - 0.99 0.99 - 0.02 - - cmove rax, rcx - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 index c5abb17f7eba..714b68c51734 100644 --- a/rust/zerocopy/benches/write_to_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 @@ -3,8 +3,7 @@ bench_write_to_dynamic_size: push rbx push rax mov rbx, rcx - lea r14, [2*rsi + 5] - and r14, -2 + lea r14, [2*rsi + 4] cmp rcx, r14 jne .LBB5_2 mov rax, rdi diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca index 5b2c08a31a29..2cc0a58c70fc 100644 --- a/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1900 +Instructions: 1800 Total Cycles: 2890 -Total uOps: 2500 +Total uOps: 2400 Dispatch Width: 4 -uOps Per Cycle: 0.87 -IPC: 0.66 -Block RThroughput: 6.3 +uOps Per Cycle: 0.83 +IPC: 0.62 +Block RThroughput: 6.0 Instruction Info: @@ -22,8 +22,7 @@ Instruction Info: 2 5 1.00 * push rbx 2 5 1.00 * push rax 1 1 0.33 mov rbx, rcx - 1 1 0.50 lea r14, [2*rsi + 5] - 1 1 0.33 and r14, -2 + 1 1 0.50 lea r14, [2*rsi + 4] 1 1 0.33 cmp rcx, r14 1 1 1.00 jne .LBB5_2 1 1 0.33 mov rax, rdi @@ -52,25 +51,24 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 4.66 4.64 4.00 4.70 4.00 3.00 + - - 4.47 4.01 4.00 4.52 4.00 3.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 1.00 - - 1.00 push r14 - - - - 1.00 - 1.00 - push rbx - - - - 1.00 - - 1.00 push rax - - - 0.02 0.97 - 0.01 - - mov rbx, rcx - - - 0.97 0.03 - - - - lea r14, [2*rsi + 5] - - - 0.63 0.35 - 0.02 - - and r14, -2 - - - 0.31 0.34 - 0.35 - - cmp rcx, r14 + - - - 0.99 - 0.01 - - mov rbx, rcx + - - 0.99 0.01 - - - - lea r14, [2*rsi + 4] + - - 0.02 0.95 - 0.03 - - cmp rcx, r14 - - - - - 1.00 - - jne .LBB5_2 - - - 0.33 0.33 - 0.34 - - mov rax, rdi - - - 0.36 0.31 - 0.33 - - mov rdi, rdx - - - 0.33 0.35 - 0.32 - - mov rsi, rax - - - 0.35 0.63 - 0.02 - - mov rdx, rbx + - - 0.48 0.04 - 0.48 - - mov rax, rdi + - - 0.51 0.01 - 0.48 - - mov rdi, rdx + - - 0.01 0.98 - 0.01 - - mov rsi, rax + - - 0.98 0.02 - - - - mov rdx, rbx - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] - - - 0.65 0.35 - - - - cmp rbx, r14 - - - 0.69 - - 0.31 - - sete al + - - 0.94 0.03 - 0.03 - - cmp rbx, r14 + - - 0.52 - - 0.48 - - sete al - - 0.02 0.98 - - - - add rsp, 8 - - - - - - - 1.00 pop rbx - - - - - - 1.00 - pop r14 diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 index d7779c6c9178..63e5d5200962 100644 --- a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 @@ -3,8 +3,7 @@ bench_write_to_prefix_dynamic_size: push rbx push rax mov rbx, rcx - lea r14, [2*rsi + 5] - and r14, -2 + lea r14, [2*rsi + 4] cmp r14, rcx ja .LBB5_2 mov rax, rdi diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca index 4cebe24d4f3b..e623ab3f302c 100644 --- a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1900 +Instructions: 1800 Total Cycles: 2890 -Total uOps: 2600 +Total uOps: 2500 Dispatch Width: 4 -uOps Per Cycle: 0.90 -IPC: 0.66 -Block RThroughput: 6.5 +uOps Per Cycle: 0.87 +IPC: 0.62 +Block RThroughput: 6.3 Instruction Info: @@ -22,8 +22,7 @@ Instruction Info: 2 5 1.00 * push rbx 2 5 1.00 * push rax 1 1 0.33 mov rbx, rcx - 1 1 0.50 lea r14, [2*rsi + 5] - 1 1 0.33 and r14, -2 + 1 1 0.50 lea r14, [2*rsi + 4] 1 1 0.33 cmp r14, rcx 1 1 1.00 ja .LBB5_2 1 1 0.33 mov rax, rdi @@ -52,26 +51,25 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 5.47 4.49 4.00 5.04 4.00 3.00 + - - 4.09 4.00 4.00 5.91 4.00 3.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 1.00 - - 1.00 push r14 - - - - 1.00 - 1.00 - push rbx - - - - 1.00 - - 1.00 push rax - - - 0.48 0.51 - 0.01 - - mov rbx, rcx - - - 0.51 0.49 - - - - lea r14, [2*rsi + 5] - - - 0.48 0.05 - 0.47 - - and r14, -2 - - - 0.48 0.49 - 0.03 - - cmp r14, rcx + - - 0.02 0.97 - 0.01 - - mov rbx, rcx + - - 0.04 0.96 - - - - lea r14, [2*rsi + 4] + - - 0.02 0.04 - 0.94 - - cmp r14, rcx - - - - - 1.00 - - ja .LBB5_2 - - - 0.04 0.47 - 0.49 - - mov rax, rdi - - - 0.49 0.03 - 0.48 - - mov rdi, rdx - - - 0.03 0.48 - 0.49 - - mov rsi, rax - - - 0.48 0.51 - 0.01 - - mov rdx, r14 + - - 0.96 0.01 - 0.03 - - mov rax, rdi + - - 0.03 0.93 - 0.04 - - mov rdi, rdx + - - 0.94 0.06 - - - - mov rsi, rax + - - 0.06 0.93 - 0.01 - - mov rdx, r14 - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] - - - 0.51 0.49 - - - - cmp r14, rbx - - - 1.94 - - 0.06 - - setbe al - - - 0.03 0.97 - - - - add rsp, 8 + - - 0.93 0.05 - 0.02 - - cmp r14, rbx + - - 0.14 - - 1.86 - - setbe al + - - 0.95 0.05 - - - - add rsp, 8 - - - - - - - 1.00 pop rbx - - - - - - 1.00 - pop r14 - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 index 75f349562db6..d13a9bb3b83c 100644 --- a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 @@ -3,8 +3,7 @@ bench_write_to_suffix_dynamic_size: push rbx push rax mov rbx, rcx - lea r14, [2*rsi + 5] - and r14, -2 + lea r14, [2*rsi + 4] sub rcx, r14 jb .LBB5_2 mov rax, rdi diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca index 95cb9dfe2eb7..fb8fce317e0c 100644 --- a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2000 +Instructions: 1900 Total Cycles: 2890 -Total uOps: 2600 +Total uOps: 2500 Dispatch Width: 4 -uOps Per Cycle: 0.90 -IPC: 0.69 -Block RThroughput: 6.5 +uOps Per Cycle: 0.87 +IPC: 0.66 +Block RThroughput: 6.3 Instruction Info: @@ -22,8 +22,7 @@ Instruction Info: 2 5 1.00 * push rbx 2 5 1.00 * push rax 1 1 0.33 mov rbx, rcx - 1 1 0.50 lea r14, [2*rsi + 5] - 1 1 0.33 and r14, -2 + 1 1 0.50 lea r14, [2*rsi + 4] 1 1 0.33 sub rcx, r14 1 1 1.00 jb .LBB5_2 1 1 0.33 mov rax, rdi @@ -53,27 +52,26 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 4.98 4.98 4.00 5.04 4.00 3.00 + - - 4.65 4.64 4.00 4.71 4.00 3.00 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 1.00 - - 1.00 push r14 - - - - 1.00 - 1.00 - push rbx - - - - 1.00 - - 1.00 push rax - - - 0.94 0.05 - 0.01 - - mov rbx, rcx - - - 0.06 0.94 - - - - lea r14, [2*rsi + 5] - - - 0.93 0.02 - 0.05 - - and r14, -2 - - - 0.05 0.94 - 0.01 - - sub rcx, r14 + - - 0.02 0.97 - 0.01 - - mov rbx, rcx + - - 0.97 0.03 - - - - lea r14, [2*rsi + 4] + - - 0.64 0.35 - 0.01 - - sub rcx, r14 - - - - - 1.00 - - jb .LBB5_2 - - - 0.02 0.04 - 0.94 - - mov rax, rdi - - - 0.03 0.97 - - - - add rdx, rcx - - - 0.95 0.05 - - - - mov rdi, rdx - - - 0.94 0.03 - 0.03 - - mov rsi, rax - - - 0.01 0.03 - 0.96 - - mov rdx, r14 + - - 0.35 0.31 - 0.34 - - mov rax, rdi + - - 0.32 0.68 - - - - add rdx, rcx + - - 0.32 0.34 - 0.34 - - mov rdi, rdx + - - 0.67 0.32 - 0.01 - - mov rsi, rax + - - 0.34 0.35 - 0.31 - - mov rdx, r14 - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] - - - 0.05 0.94 - 0.01 - - cmp rbx, r14 - - - 0.97 - - 0.03 - - setae al - - - 0.03 0.97 - - - - add rsp, 8 + - - 0.35 0.31 - 0.34 - - cmp rbx, r14 + - - 0.65 - - 0.35 - - setae al + - - 0.02 0.98 - - - - add rsp, 8 - - - - - - - 1.00 pop rbx - - - - - - 1.00 - pop r14 - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64 b/rust/zerocopy/benches/zero_dynamic_size.x86-64 index 2b31ed644eaf..92e2748b1353 100644 --- a/rust/zerocopy/benches/zero_dynamic_size.x86-64 +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64 @@ -1,5 +1,4 @@ bench_zero_dynamic_size: - lea rdx, [2*rsi + 5] - and rdx, -2 + lea rdx, [2*rsi + 4] xor esi, esi jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca index 0b086a29b0ed..7dee3b1de141 100644 --- a/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 400 -Total Cycles: 142 -Total uOps: 500 +Instructions: 300 +Total Cycles: 108 +Total uOps: 400 Dispatch Width: 4 -uOps Per Cycle: 3.52 -IPC: 2.82 -Block RThroughput: 1.3 +uOps Per Cycle: 3.70 +IPC: 2.78 +Block RThroughput: 1.0 Instruction Info: @@ -18,8 +18,7 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.50 lea rdx, [2*rsi + 5] - 1 1 0.33 and rdx, -2 + 1 1 0.50 lea rdx, [2*rsi + 4] 1 0 0.25 xor esi, esi 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] @@ -37,11 +36,10 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 0.99 1.00 - 1.01 0.50 0.50 + - - 0.50 0.50 - 1.00 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.99 0.01 - - - - lea rdx, [2*rsi + 5] - - - - 0.99 - 0.01 - - and rdx, -2 + - - 0.50 0.50 - - - - lea rdx, [2*rsi + 4] - - - - - - - - xor esi, esi - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/src/byte_slice.rs b/rust/zerocopy/src/byte_slice.rs index b7f85098dbc4..26fcc088aecf 100644 --- a/rust/zerocopy/src/byte_slice.rs +++ b/rust/zerocopy/src/byte_slice.rs @@ -427,8 +427,12 @@ fn test_ref_mut_split_at_unchecked() { let borrow_mut = cell.borrow_mut(); let slice_ref_mut: cell::RefMut<'_, [u8]> = cell::RefMut::map(borrow_mut, |a| &mut a[..]); // SAFETY: 2 is within bounds of [1, 2, 3, 4] - let (l, r) = unsafe { slice_ref_mut.split_at_unchecked(2) }; + let (mut l, mut r) = unsafe { slice_ref_mut.split_at_unchecked(2) }; assert_eq!(*l, [1, 2]); assert_eq!(*r, [3, 4]); + l[0] = 5; + r[0] = 6; + drop((l, r)); + assert_eq!(*cell.borrow(), [5, 2, 6, 4]); } } diff --git a/rust/zerocopy/src/byteorder.rs b/rust/zerocopy/src/byteorder.rs index c761d5728320..6189a6e7f77c 100644 --- a/rust/zerocopy/src/byteorder.rs +++ b/rust/zerocopy/src/byteorder.rs @@ -87,7 +87,7 @@ pub trait ByteOrder: Copy + Clone + Debug + Display + Eq + PartialEq + Ord + PartialOrd + Hash + private::Sealed { - #[doc(hidden)] + /// A value-level representation of byte order. const ORDER: Order; } @@ -98,11 +98,12 @@ impl Sealed for super::BigEndian {} impl Sealed for super::LittleEndian {} } -#[allow(missing_copy_implementations, missing_debug_implementations)] -#[doc(hidden)] -#[derive(PartialEq)] +/// A value-level representation of [`ByteOrder`]. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum Order { + /// A value-level representation of [`BigEndian`]. BigEndian, + /// A value-level representation of [`LittleEndian`]. LittleEndian, } @@ -1235,7 +1236,7 @@ fn is_nan(self) -> bool { self.is_nan() } impl_traits!(I32, i32, signed); impl_traits!(I64, i64, signed); impl_traits!(I128, i128, signed); - impl_traits!(Isize, isize, unsigned); + impl_traits!(Isize, isize, signed); impl_traits!(F32, f32, signed, @float); impl_traits!(F64, f64, signed, @float); @@ -1383,6 +1384,18 @@ fn test_non_native() { call_for_all_types!(test_non_native, NonNativeEndian); } + #[test] + fn test_float_bit_patterns() { + for bits in [0u32, 0x8000_0000, 0x7F80_0000, 0xFF80_0000, 0x7FC0_1234] { + let be = F32::::new(f32::from_bits(bits)); + let le = F32::::new(f32::from_bits(bits)); + assert_eq!(be.to_bytes(), bits.to_be_bytes()); + assert_eq!(le.to_bytes(), bits.to_le_bytes()); + assert_eq!(be.get().to_bits(), bits); + assert_eq!(le.get().to_bits(), bits); + } + } + #[test] fn test_ops_impls() { // Test implementations of traits in `core::ops`. Some of these are @@ -1551,6 +1564,17 @@ fn t() test!(@unary Not, not, call_for_signed_types, call_for_unsigned_types); test!(@unary Neg, neg, call_for_signed_types, call_for_float_types); + + for shift in [0u64, 1, 31, 63] { + let n = 0x0123_4567_89AB_CDEFu64; + let shift_u32: u32 = shift.try_into().unwrap(); + let shl = n.checked_shl(shift_u32).unwrap(); + let shr = n.checked_shr(shift_u32).unwrap(); + assert_eq!(core::ops::Shl::shl(U64::::new(n), shift).get(), shl); + assert_eq!(core::ops::Shr::shr(U64::::new(n), shift).get(), shr); + assert_eq!(core::ops::Shl::shl(U64::::new(n), shift).get(), shl); + assert_eq!(core::ops::Shr::shr(U64::::new(n), shift).get(), shr); + } } #[test] @@ -1579,6 +1603,13 @@ fn test_byteorder_traits_coverage() { assert!(val_be <= val_be); assert_eq!(val_be.cmp(&val_be), core::cmp::Ordering::Equal); + let low = U16::::new(255); + let high = U16::::new(256); + assert!(low < high); + assert!(low < 256u16); + assert_eq!(low.cmp(&high), core::cmp::Ordering::Less); + assert!(I16::::new(-1) < I16::::new(0)); + // PartialOrd with native assert!(val_be == 1u16); assert!(val_be >= 1u16); diff --git a/rust/zerocopy/src/error.rs b/rust/zerocopy/src/error.rs index 5eb30de934f3..98399bab7e97 100644 --- a/rust/zerocopy/src/error.rs +++ b/rust/zerocopy/src/error.rs @@ -199,10 +199,8 @@ impl From, S, V>) -> ConvertError { match err { - ConvertError::Alignment(e) => { - #[allow(unreachable_code)] - return ConvertError::Alignment(Infallible::from(e)); - } + #[allow(unreachable_code)] + ConvertError::Alignment(e) => match Infallible::from(e) {}, ConvertError::Size(e) => ConvertError::Size(e), ConvertError::Validity(e) => ConvertError::Validity(e), } @@ -237,7 +235,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } #[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] -#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +#[cfg_attr(doc_cfg, doc(cfg(any(rust = "1.81.0", feature = "std"))))] impl Error for ConvertError where A: fmt::Display + fmt::Debug, @@ -410,7 +408,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } #[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] -#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +#[cfg_attr(doc_cfg, doc(cfg(any(rust = "1.81.0", feature = "std"))))] impl Error for AlignmentError where Src: Deref, @@ -571,7 +569,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } #[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] -#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +#[cfg_attr(doc_cfg, doc(cfg(any(rust = "1.81.0", feature = "std"))))] impl Error for SizeError where Src: Deref, @@ -707,7 +705,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } #[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] -#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +#[cfg_attr(doc_cfg, doc(cfg(any(rust = "1.81.0", feature = "std"))))] impl Error for ValidityError where Dst: KnownLayout + TryFromBytes {} impl From> diff --git a/rust/zerocopy/src/impls.rs b/rust/zerocopy/src/impls.rs index 62e234c5202b..01b509b98264 100644 --- a/rust/zerocopy/src/impls.rs +++ b/rust/zerocopy/src/impls.rs @@ -111,7 +111,7 @@ // pattern 0x01. const _: () = unsafe { unsafe_impl!(=> TryFromBytes for bool; |byte| { - let byte = byte.transmute_with::(); + let byte = byte.transmute_with::(); *byte.unaligned_as_ref() < 2 }) }; @@ -140,7 +140,7 @@ // `char`. const _: () = unsafe { unsafe_impl!(=> TryFromBytes for char; |c| { - let c = c.transmute_with::, invariant::Valid, CastSizedExact, BecauseImmutable>(); + let c = c.transmute_with::, invariant::Safe, CastSizedExact, BecauseImmutable>(); let c = c.read().into_inner(); char::from_u32(c).is_some() }); @@ -173,7 +173,7 @@ // Returns `Err` if the slice is not UTF-8. const _: () = unsafe { unsafe_impl!(=> TryFromBytes for str; |c| { - let c = c.transmute_with::<[u8], invariant::Valid, CastUnsized, BecauseImmutable>(); + let c = c.transmute_with::<[u8], invariant::Safe, CastUnsized, BecauseImmutable>(); let c = c.unaligned_as_ref(); core::str::from_utf8(c).is_ok() }) @@ -183,7 +183,7 @@ macro_rules! unsafe_impl_try_from_bytes_for_nonzero { ($($nonzero:ident[$prim:ty]),*) => { $( unsafe_impl!(=> TryFromBytes for $nonzero; |n| { - let n = n.transmute_with::, invariant::Valid, CastSizedExact, BecauseImmutable>(); + let n = n.transmute_with::, invariant::Safe, CastSizedExact, BecauseImmutable>(); $nonzero::new(n.read().into_inner()).is_some() }); )* @@ -421,15 +421,15 @@ macro_rules! unsafe_impl_transmute_from_for_atomic { ($($($tyvar:ident)? => $atomic:ty [$prim:ty]),*) => {{ crate::util::macros::__unsafe(); - use crate::pointer::{SizeEq, TransmuteFrom, invariant::Valid}; + use crate::pointer::{SizeEq, TransmuteFrom, invariant::Safe}; $( // SAFETY: The caller promised that `$atomic` and `$prim` have // the same size and bit validity. - unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for $prim {} + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Safe, Safe> for $prim {} // SAFETY: The caller promised that `$atomic` and `$prim` have // the same size and bit validity. - unsafe impl<$($tyvar)?> TransmuteFrom<$prim, Valid, Valid> for $atomic {} + unsafe impl<$($tyvar)?> TransmuteFrom<$prim, Safe, Safe> for $atomic {} impl<$($tyvar)?> SizeEq> for ReadOnly<$prim> { type CastFrom = $crate::pointer::cast::CastSizedExact; @@ -444,9 +444,9 @@ impl<$($tyvar)?> SizeEq> for ReadOnly<$prim> { // `UnsafeCell` has the same in-memory representation as // its inner type `T`. A consequence of this guarantee is that // it is possible to convert between `T` and `UnsafeCell`. - unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for core::cell::UnsafeCell<$prim> {} + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Safe, Safe> for core::cell::UnsafeCell<$prim> {} // SAFETY: See previous safety comment. - unsafe impl<$($tyvar)?> TransmuteFrom, Valid, Valid> for $atomic {} + unsafe impl<$($tyvar)?> TransmuteFrom, Safe, Safe> for $atomic {} )* }}; } @@ -758,7 +758,7 @@ mod atomic_ptr { pub enum value {} // SAFETY: See safety comment on `ProjectToTag`. - unsafe impl HasTag for ManuallyDrop { + unsafe impl HasTag for ManuallyDrop { #[inline] fn only_derive_is_allowed_to_implement_this_trait() where @@ -796,8 +796,8 @@ fn only_derive_is_allowed_to_implement_this_trait() // private field, and because it is the name it is referred to in the public // documentation of `ManuallyDrop::new`, `ManuallyDrop::into_inner`, // `ManuallyDrop::take` and `ManuallyDrop::drop`. - unsafe impl - HasField + unsafe impl + HasField for ManuallyDrop { #[inline] @@ -849,7 +849,7 @@ fn project(slf: PtrInner<'_, Self>) -> *mut T { const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for UnsafeCell) }; assert_unaligned!(UnsafeCell<()>, UnsafeCell); -// SAFETY: See safety comment in `is_bit_valid` impl. +// SAFETY: See safety comment in `is_safe` impl. unsafe impl TryFromBytes for UnsafeCell { #[allow(clippy::missing_inline_in_public_items)] fn only_derive_is_allowed_to_implement_this_trait() @@ -859,11 +859,11 @@ fn only_derive_is_allowed_to_implement_this_trait() } #[inline(always)] - fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool + fn is_safe(candidate: Maybe<'_, Self, A>) -> bool where A: invariant::Alignment, { - T::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + T::is_safe(candidate.transmute::<_, _, BecauseImmutable>()) } } @@ -897,10 +897,10 @@ fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool let c: Ptr<'_, ReadOnly<[T]>, _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); // Note that this call may panic, but it would still be sound even if it - // did. `is_bit_valid` does not promise that it will not panic (in fact, - // it explicitly warns that it's a possibility), and we have not - // violated any safety invariants that we must fix before returning. - <[T] as TryFromBytes>::is_bit_valid(c) + // did. `is_safe` does not promise that it will not panic (in fact, it + // explicitly warns that it's a possibility), and we have not violated + // any safety invariants that we must fix before returning. + <[T] as TryFromBytes>::is_safe(c) }); unsafe_impl!(const N: usize, T: FromZeros => FromZeros for [T; N]); unsafe_impl!(const N: usize, T: FromBytes => FromBytes for [T; N]); @@ -924,14 +924,14 @@ fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool // // In other words, the layout of a `[T] is a sequence of `T`s laid out // back-to-back with no bytes in between. If all elements in `candidate` - // are `is_bit_valid`, so too is `candidate`. + // are `is_safe`, so too is `candidate`. // // Note that any of the below calls may panic, but it would still be - // sound even if it did. `is_bit_valid` does not promise that it will - // not panic (in fact, it explicitly warns that it's a possibility), and - // we have not violated any safety invariants that we must fix before + // sound even if it did. `is_safe` does not promise that it will not + // panic (in fact, it explicitly warns that it's a possibility), and we + // have not violated any safety invariants that we must fix before // returning. - c.iter().all(::is_bit_valid) + c.iter().all(::is_safe) }); unsafe_impl!(T: FromZeros => FromZeros for [T]); unsafe_impl!(T: FromBytes => FromBytes for [T]); @@ -946,7 +946,7 @@ fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool // null pointers, so this is not a footgun. // - `TryFromBytes`: By the same reasoning as for `FromZeroes`, we can implement // `TryFromBytes` for thin pointers provided that -// [`TryFromByte::is_bit_valid`] only produces `true` for zeroed bytes. +// [`TryFromBytes::is_safe`] only produces `true` for zeroed bytes. // // NOTE(#170): Implementing `FromBytes` and `IntoBytes` for raw pointers would // be sound, but carries provenance footguns. We want to support `FromBytes` and @@ -1012,11 +1012,11 @@ macro_rules! impl_tuple { // SAFETY: If all fields of the tuple `Self` are `Immutable`, so too is `Self`. unsafe_impl!($($head_T: Immutable,)* $next_T: Immutable => Immutable for ($($head_T,)* $next_T,)); - // SAFETY: If all fields in `c` are `is_bit_valid`, so too is `c`. + // SAFETY: If all fields in `c` are `is_safe`, so too is `c`. unsafe_impl!($($head_T: TryFromBytes,)* $next_T: TryFromBytes => TryFromBytes for ($($head_T,)* $next_T,); |c| { let mut c = c; - $(TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($head_I) }>())) &&)* - TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($next_I) }>())) + $(TryFromBytes::is_safe(into_inner!(c.reborrow().project::())) &&)* + TryFromBytes::is_safe(into_inner!(c.reborrow().project::())) }); // SAFETY: If all fields in `Self` are `FromZeros`, so too is `Self`. @@ -1026,7 +1026,9 @@ macro_rules! impl_tuple { unsafe_impl!($($head_T: FromBytes,)* $next_T: FromBytes => FromBytes for ($($head_T,)* $next_T,)); // SAFETY: See safety comment on `ProjectToTag`. - unsafe impl<$($head_T,)* $next_T> crate::HasTag for ($($head_T,)* $next_T,) { + unsafe impl crate::HasTag + for ($($head_T,)* $next_T,) + { #[inline] fn only_derive_is_allowed_to_implement_this_trait() where @@ -1077,7 +1079,8 @@ fn only_derive_is_allowed_to_implement_this_trait() // - `()` has the same visibility as the `.$CurrI` field (ie, `.0`, // `.1`, etc) // - `Type` has the same type as `$CurrI`; i.e., `$CurrT`. - unsafe impl<$($AllT),+> crate::HasField< + unsafe impl crate::HasField< + Client, (), { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($CurrI)} @@ -1103,7 +1106,8 @@ fn project(slf: crate::PtrInner<'_, Self>) -> *mut Self::Type { } // SAFETY: See comments on items. - unsafe impl crate::ProjectField< + unsafe impl crate::ProjectField< + Client, (), (Aliasing, Alignment, crate::invariant::Uninit), { crate::STRUCT_VARIANT_ID }, @@ -1129,7 +1133,8 @@ fn only_derive_is_allowed_to_implement_this_trait() } // SAFETY: See comments on items. - unsafe impl crate::ProjectField< + unsafe impl crate::ProjectField< + Client, (), (Aliasing, Alignment, crate::invariant::Initialized), { crate::STRUCT_VARIANT_ID }, @@ -1155,9 +1160,10 @@ fn only_derive_is_allowed_to_implement_this_trait() } // SAFETY: See comments on items. - unsafe impl crate::ProjectField< + unsafe impl crate::ProjectField< + Client, (), - (Aliasing, Alignment, crate::invariant::Valid), + (Aliasing, Alignment, crate::invariant::Safe), { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($CurrI)} > for ($($AllT,)+) @@ -1174,7 +1180,7 @@ fn only_derive_is_allowed_to_implement_this_trait() // SAFETY: Tuples are product types whose fields are // well-aligned, so projection preserves both the alignment and // validity invariants of the outer pointer. - type Invariants = (Aliasing, Alignment, crate::invariant::Valid); + type Invariants = (Aliasing, Alignment, crate::invariant::Safe); // SAFETY: Tuples are product types and so projection is infallible; type Error = core::convert::Infallible; @@ -1389,10 +1395,9 @@ mod tests { #[test] fn test_impls() { - // A type that can supply test cases for testing - // `TryFromBytes::is_bit_valid`. All types passed to `assert_impls!` - // must implement this trait; that macro uses it to generate runtime - // tests for `TryFromBytes` impls. + // A type that can supply test cases for testing `TryFromBytes::is_safe`. + // All types passed to `assert_impls!` must implement this trait; that + // macro uses it to generate runtime tests for `TryFromBytes` impls. // // All `T: FromBytes` types are provided with a blanket impl. Other // types must implement `TryFromBytesTestable` directly (ie using @@ -1574,19 +1579,15 @@ mod autoref_trick { pub(super) struct AutorefWrapper(pub(super) PhantomData); - pub(super) trait TestIsBitValidShared { + pub(super) trait TestIsSafeShared { #[allow(clippy::needless_lifetimes)] - fn test_is_bit_valid_shared<'ptr>(&self, candidate: Maybe<'ptr, T>) - -> Option; + fn test_is_safe_shared<'ptr>(&self, candidate: Maybe<'ptr, T>) -> Option; } - impl TestIsBitValidShared for AutorefWrapper { + impl TestIsSafeShared for AutorefWrapper { #[allow(clippy::needless_lifetimes)] - fn test_is_bit_valid_shared<'ptr>( - &self, - candidate: Maybe<'ptr, T>, - ) -> Option { - Some(T::is_bit_valid(candidate)) + fn test_is_safe_shared<'ptr>(&self, candidate: Maybe<'ptr, T>) -> Option { + Some(T::is_safe(candidate)) } } @@ -1690,12 +1691,12 @@ macro_rules! assert_impls { #[allow(unused, non_local_definitions)] impl AutorefWrapper<$ty> { #[allow(clippy::needless_lifetimes)] - fn test_is_bit_valid_shared<'ptr>( + fn test_is_safe_shared<'ptr>( &mut self, candidate: Maybe<'ptr, $ty>, ) -> Option { assert_on_allowlist!( - test_is_bit_valid_shared($ty): + test_is_safe_shared($ty): ManuallyDrop>, ManuallyDrop<[UnsafeCell]>, ManuallyDrop<[UnsafeCell]>, @@ -1799,9 +1800,9 @@ fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { // necessarily `IntoBytes`, but that's the corner we've // backed ourselves into by using `Ptr::from_ref`. let c = unsafe { c.assume_initialized() }; - let res = w.test_is_bit_valid_shared(c); + let res = w.test_is_safe_shared(c); if let Some(res) = res { - assert!(res, "{}::is_bit_valid (shared `Ptr`): got false, expected true", stringify!($ty)); + assert!(res, "{}::is_safe (shared `Ptr`): got false, expected true", stringify!($ty)); } let c = Ptr::from_mut(&mut *val); @@ -1810,8 +1811,8 @@ fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { // necessarily `IntoBytes`, but that's the corner we've // backed ourselves into by using `Ptr::from_ref`. let mut c = unsafe { c.assume_initialized() }; - let res = <$ty as TryFromBytes>::is_bit_valid(c.reborrow_shared()); - assert!(res, "{}::is_bit_valid (exclusive `Ptr`): got false, expected true", stringify!($ty)); + let res = <$ty as TryFromBytes>::is_safe(c.reborrow_shared()); + assert!(res, "{}::is_safe (exclusive `Ptr`): got false, expected true", stringify!($ty)); // `bytes` is `Some(val.as_bytes())` if `$ty: IntoBytes + // Immutable` and `None` otherwise. @@ -2237,11 +2238,11 @@ fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { assert_impls!(ManuallyDrop: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); // This test is important because it allows us to test our hand-rolled - // implementation of ` as TryFromBytes>::is_bit_valid`. + // implementation of ` as TryFromBytes>::is_safe`. assert_impls!(ManuallyDrop: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); assert_impls!(ManuallyDrop<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); // This test is important because it allows us to test our hand-rolled - // implementation of ` as TryFromBytes>::is_bit_valid`. + // implementation of ` as TryFromBytes>::is_safe`. assert_impls!(ManuallyDrop<[bool]>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); assert_impls!(ManuallyDrop: !Immutable, !TryFromBytes, !KnownLayout, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); assert_impls!(ManuallyDrop<[NotZerocopy]>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); @@ -2255,14 +2256,14 @@ fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { assert_impls!(Wrapping: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); // This test is important because it allows us to test our hand-rolled - // implementation of ` as TryFromBytes>::is_bit_valid`. + // implementation of ` as TryFromBytes>::is_safe`. assert_impls!(Wrapping: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); assert_impls!(Wrapping: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); assert_impls!(Wrapping>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); assert_impls!(Unalign: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); // This test is important because it allows us to test our hand-rolled - // implementation of ` as TryFromBytes>::is_bit_valid`. + // implementation of ` as TryFromBytes>::is_safe`. assert_impls!(Unalign: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); assert_impls!(Unalign: KnownLayout, Unaligned, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes); diff --git a/rust/zerocopy/src/layout.rs b/rust/zerocopy/src/layout.rs index b1fa0cd436db..5d5598cd0cfb 100644 --- a/rust/zerocopy/src/layout.rs +++ b/rust/zerocopy/src/layout.rs @@ -30,6 +30,7 @@ #[allow(missing_debug_implementations, missing_copy_implementations)] #[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] #[derive(Copy, Clone)] +#[cfg_attr(kani, derive(kani::Arbitrary))] pub struct DstLayout { pub(crate) align: NonZeroUsize, pub(crate) size_info: SizeInfo, @@ -42,29 +43,680 @@ pub struct DstLayout { #[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] #[derive(Copy, Clone)] +#[cfg_attr(kani, derive(kani::Arbitrary))] pub(crate) enum SizeInfo { Sized { size: usize }, SliceDst(TrailingSliceLayout), } +/// The alignment and phase of a rounded size calculation. +/// +/// If `encoded` is the stored non-zero integer, then its highest set bit is a +/// power-of-two alignment `align`, and its remaining lower bits are a `phase` +/// in `0..align`. Thus every non-zero bit pattern is a valid encoding, and: +/// +/// ```text +/// encoded = align + phase +/// ``` +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[repr(transparent)] +#[derive(Copy, Clone)] +#[cfg_attr(kani, derive(kani::Arbitrary))] +pub(crate) struct RoundingAlignAndPhase(NonZeroUsize); + +impl RoundingAlignAndPhase { + /// Encodes `align` and `phase`. + /// + /// # Panics + /// + /// Panics if `align` is not a power of two and `phase` is not less than + /// `align`. + #[inline(always)] + #[cfg_attr(kani, kani::requires(align.is_power_of_two() && phase < align.get()))] + #[cfg_attr(kani, kani::ensures(|result| result.0.get() == align.get() + phase))] + pub(crate) const fn new(align: NonZeroUsize, phase: usize) -> Self { + #[cfg(kani)] + #[kani::proof_for_contract(RoundingAlignAndPhase::new)] + #[kani::solver(kissat)] + fn proof() { + RoundingAlignAndPhase::new(kani::any(), kani::any()); + } + + const_assert!(align.get().is_power_of_two()); + const_assert!(phase < align.get()); + + // Since `align` is a power of two and `phase < align`, their set bits + // are disjoint. The result is therefore non-zero and losslessly stores + // both components. + let encoded = align.get() | phase; + match NonZeroUsize::new(encoded) { + Some(encoded) => Self(encoded), + None => const_unreachable!(), + } + } + + /// Decodes the alignment and phase. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&(align, phase)| { + align.is_power_of_two() && phase < align.get() + && align.get().checked_add(phase) == Some(self.0.get()) + }))] + pub(crate) const fn components(self) -> (NonZeroUsize, usize) { + #[cfg(kani)] + #[kani::proof_for_contract(RoundingAlignAndPhase::components)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().components(); + } + // `leading_zeros <= POINTER_WIDTH_BITS - 1` because the encoded value + // is non-zero. Converting `leading_zeros` to `usize` cannot truncate + // because it is at most the number of bits in a `usize`. + #[allow( + clippy::arithmetic_side_effects, + clippy::as_conversions, + clippy::needless_nonzero_get + )] + let shift = (POINTER_WIDTH_BITS - 1) - (self.0.get().leading_zeros() as usize); + #[allow(clippy::arithmetic_side_effects)] + let align = 1usize << shift; + let align = match NonZeroUsize::new(align) { + Some(align) => align, + None => const_unreachable!(), + }; + // `align` is the highest set bit, so XOR removes exactly that bit and + // leaves a value in `0..align`. + let phase = self.0.get() ^ align.get(); + (align, phase) + } + + /// Decodes the alignment, which is the highest set bit of `self`. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|align| { + align.is_power_of_two() && align.get() <= self.0.get() + && self.0.get() - align.get() < align.get() + }))] + pub(crate) const fn align(self) -> NonZeroUsize { + #[cfg(kani)] + #[kani::proof_for_contract(RoundingAlignAndPhase::align)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().align(); + } + + self.components().0 + } +} + #[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] #[derive(Copy, Clone)] +#[cfg_attr(kani, derive(kani::Arbitrary))] pub(crate) struct TrailingSliceLayout { - // The offset of the first byte of the trailing slice field. Note that this - // is NOT the same as the minimum size of the type. For example, consider - // the following type: - // - // struct Foo { - // a: u16, - // b: u8, - // c: [u8], - // } - // - // In `Foo`, `c` is at byte offset 3. When `c.len() == 0`, `c` is followed - // by a padding byte. + /// The offset of the first byte of the trailing slice field. Note that this + /// is NOT the same as the minimum size of the type. For example, consider + /// the following type: + /// + /// ```ignore + /// #[repr(C, align(2))] + /// struct Foo { + /// header: u16, + /// tag: u8, + /// tail: [u8], + /// } + /// ``` + /// + /// In `Foo`, `tail` is at byte offset 3. When `tail.len() == 0`, `tail` is + /// followed by a padding byte. pub(crate) offset: usize, - // The size of the element type of the trailing slice field. + /// The size of the element type of the trailing slice field. pub(crate) elem_size: E, + /// The trailing slice's byte offset plus fixed trailing padding, minus the + /// rounding phase. + /// + /// Fixed trailing padding excludes padding added by the final alignment + /// rounding. The rounding alignment and phase are stored in + /// `size_rounding_align_and_phase`. This value is independent of the slice + /// length and can be smaller than, equal to, or larger than `offset`. + /// + /// For a completed layout, this value is a multiple of `DstLayout::align`. + /// The minimum object size equals this value when the phase is zero; + /// otherwise, it exceeds this value by one rounding-alignment unit. + pub(crate) size_base: usize, + /// The alignment and byte phase of the rounding applied to the trailing + /// slice's contribution to object size. + /// + /// The alignment `size_align` is a nonzero power of two. The phase + /// `size_phase` is the fixed byte contribution inside that rounding, in + /// `0..size_align`. Both are independent of the trailing slice's length. + /// For `elems` trailing elements, the enclosing object's size is: + /// + /// ```text + /// size_base + round_up(size_phase + elems * elem_size, size_align) + /// ``` + /// + /// For a completed layout, `size_align` is at least `DstLayout::align`. + /// Packing can reduce the enclosing type's alignment and the alignment + /// used to place its trailing field without removing padding required by + /// a nested DST's own layout. For example, a `repr(packed(2))` wrapper + /// around a DST with alignment 4 can have an object alignment of 2 and a + /// rounding alignment of 4. + /// + /// The rounding parameters account for padding preserved within nested + /// fields. Their rounding origin can differ from the physical start of + /// the trailing slice, so the phase need not equal `offset % size_align`. + pub(crate) size_rounding_align_and_phase: RoundingAlignAndPhase, +} + +impl TrailingSliceLayout { + /// Produces `self.size_base` rounded down to a multiple of `size_align`, + /// plus `size_phase`, where both components come from + /// `self.size_rounding_align_and_phase`. + /// + /// This value need not equal `self.offset`, the trailing slice's byte + /// offset. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&offset| { + let (align, phase) = self.size_rounding_align_and_phase.components(); + offset == util::round_down_to_next_multiple_of_alignment(self.size_base, align) + phase + }))] + const fn size_offset(&self) -> usize { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::::size_offset)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().size_offset(); + } + + let (size_align, size_phase) = self.size_rounding_align_and_phase.components(); + let aligned_base = + util::round_down_to_next_multiple_of_alignment(self.size_base, size_align); + // The aligned base and phase have disjoint set bits. + aligned_base | size_phase + } + + /// Produces the largest trailing-slice byte capacity whose complete, padded + /// object fits in `available_bytes`, or `None` if the object does not fit + /// even with an empty trailing slice. + /// + /// This capacity is independent of `elem_size`: it need not represent a + /// whole number of elements, even for a valid Rust layout. For example, a + /// slice of `[u8; 3]` elements has three-byte elements and alignment 1. + /// With `available_bytes = 4`, this method produces `Some(4)`, although + /// only one whole element, occupying three bytes, fits. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&result| { + let (align, phase) = self.size_rounding_align_and_phase.components(); + result == available_bytes.checked_sub(self.size_base) + .map(|bytes| util::round_down_to_next_multiple_of_alignment(bytes, align)) + .and_then(|rounded| rounded.checked_sub(phase)) + }))] + const fn max_trailing_bytes(&self, available_bytes: usize) -> Option { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::::max_trailing_bytes)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().max_trailing_bytes(kani::any()); + } + + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::::max_trailing_bytes)] + #[kani::solver(kissat)] + fn proof_nonzero() { + kani::any::>().max_trailing_bytes(kani::any()); + } + + let (size_align, size_phase) = self.size_rounding_align_and_phase.components(); + + // Interpret the normalized size formula over the nonnegative integers: + // + // object_size(trailing_bytes) + // = self.size_base + // + round_up(size_phase + trailing_bytes, size_align). + // + // We seek the largest byte count with + // `object_size(trailing_bytes) <= available_bytes`. + // + // At zero trailing bytes, the input to rounding is just `size_phase`. + // Since `0 <= size_phase < size_align`, the least alignment multiple at + // or above it is zero for a zero phase, and `size_align` otherwise. + // Thus `rounded_phase = round_up(size_phase, size_align)` is exactly + // the rounded contribution of an empty tail. + let rounded_phase = if size_phase == 0 { 0 } else { size_align.get() }; + + // Adding the fixed base gives `minimum_size = object_size(0)`. The size + // formula is nondecreasing in the trailing byte count, so this is the + // minimum object size. If this sum exceeds `usize::MAX`, it also + // exceeds `available_bytes`, and no tail can fit. + let minimum_size = match self.size_base.checked_add(rounded_phase) { + Some(size) => size, + None => return None, + }; + + // Subtracting the minimum size gives the budget for increasing the + // rounded contribution. Failure means even the minimum size exceeds the + // available space. Success establishes `minimum_size + extra_bytes = + // available_bytes`. + let extra_bytes = match available_bytes.checked_sub(minimum_size) { + Some(bytes) => bytes, + None => return None, + }; + + // The rounded contribution can increase only by multiples of + // `size_align`. Its largest permitted increase is therefore + // `aligned_bytes = round_down(extra_bytes, size_align)`, with: + // + // aligned_bytes <= extra_bytes < aligned_bytes + size_align. + // + // Hence `rounded_phase + aligned_bytes` is the largest alignment + // multiple that fits after the base: adding the base gives at most + // `minimum_size + extra_bytes = available_bytes`, while the next + // alignment multiple would exceed that budget. + let aligned_bytes = util::round_down_to_next_multiple_of_alignment(extra_bytes, size_align); + + // Rounding an empty tail adds `rounded_phase - size_phase` bytes. This + // is `initial_padding`: the tail can use these bytes without increasing + // the rounded contribution. The two cases for `rounded_phase` give `0 + // <= initial_padding < size_align`, so the subtraction is + // representable. + #[allow(clippy::arithmetic_side_effects)] + let initial_padding = rounded_phase - size_phase; + + // For any nonnegative candidate byte count, the aligned limit above + // gives the following equivalences: + // + // object_size(candidate_bytes) <= available_bytes + // iff round_up(size_phase + candidate_bytes, size_align) + // <= rounded_phase + aligned_bytes + // iff size_phase + candidate_bytes + // <= rounded_phase + aligned_bytes + // iff candidate_bytes <= aligned_bytes + initial_padding. + // + // Removing `round_up` is valid because its upper bound is an alignment + // multiple. Thus `aligned_bytes + initial_padding` is exactly the + // largest tail that fits. Since `aligned_bytes` is a multiple of the + // power-of-two `size_align` and `initial_padding < size_align`, their + // set bits are disjoint. Bitwise OR therefore equals their sum, with no + // carry or overflow. + let trailing_bytes = aligned_bytes | initial_padding; + Some(trailing_bytes) + } +} + +impl TrailingSliceLayout { + /// Returns the difference between the complete object size and the + /// trailing-slice end for `elems` elements, modulo `usize::MAX + 1`. + /// + /// If the object size fits in `usize` and contains the trailing slice, + /// this is the exact trailing padding. Other inputs need not describe a + /// Rust type; their result is still the modular difference. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&padding| { + let (align, phase) = self.size_rounding_align_and_phase.components(); + let trailing_bytes = elems.wrapping_mul(self.elem_size); + let mask = align.get() - 1; + let rounded = phase.wrapping_add(trailing_bytes).wrapping_add(mask) & !mask; + let object_size = self.size_base.wrapping_add(rounded); + padding == object_size.wrapping_sub(self.offset.wrapping_add(trailing_bytes)) + }))] + pub(crate) const fn padding_for_elems(self, elems: usize) -> usize { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::padding_for_elems)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().padding_for_elems(kani::any()); + } + + let (size_align, size_phase) = self.size_rounding_align_and_phase.components(); + + // Since `size_align` is a nonzero power of two, subtracting one gives + // a mask selecting precisely the remainder modulo `size_align`. + #[allow(clippy::arithmetic_side_effects)] + let size_mask = size_align.get() - 1; + + // Removing whole alignment multiples from each element's size + // preserves the trailing-byte count modulo `size_align`. If elements + // occupy whole alignment units, this is zero, making the padding's + // independence from the element count explicit to the optimizer. + let elem_remainder = self.elem_size & size_mask; + + // The product is congruent to `elems * self.elem_size` modulo + // `size_align`. Wrapping preserves that congruence because every + // power-of-two `size_align` divides `usize::MAX + 1`. Masking then + // gives the exact trailing-byte remainder in `0..size_align`. + let trailing_remainder = elems.wrapping_mul(elem_remainder) & size_mask; + + // Adding `size_phase` gives the same remainder as the mathematical + // `size_phase + elems * self.elem_size`. Reducing before adding the + // phase also exposes this periodicity to the optimizer. + let rounding_input = size_phase.wrapping_add(trailing_remainder); + + // `rounding_padding` is the unique value in `0..size_align` that + // rounds `rounding_input` up to an alignment multiple. Since only + // the remainder determines this value, it is also the padding added + // by `round_up(size_phase + elems * self.elem_size, size_align)`. + let rounding_padding = util::padding_needed_for(rounding_input, size_align); + + // Write `trailing_bytes = elems * self.elem_size` over mathematical + // integers. Subtracting the trailing-slice end from the complete + // object size cancels `trailing_bytes`: + // + // size_base + round_up(size_phase + trailing_bytes, size_align) + // - (offset + trailing_bytes) + // = size_base + size_phase - offset + rounding_padding. + // + // Wrapping computes this difference modulo `usize::MAX + 1`, even + // if an intermediate value or the difference is unrepresentable. + // If the object size fits in `usize` and contains the trailing slice, + // the difference is in `0..=usize::MAX`; its residue is then the + // exact number of padding bytes after the trailing slice. + self.size_base + .wrapping_add(size_phase) + .wrapping_sub(self.offset) + .wrapping_add(rounding_padding) + } + + /// Computes the size described by this layout for `elems` trailing slice + /// elements. + /// + /// Returns `None` if the element-byte count or complete rounded size + /// cannot be represented as a `usize`. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&size| { + size == elems.checked_mul(self.elem_size) + .and_then(|bytes| proofs::slice_dst_size_for_trailing_bytes(self, bytes)) + }))] + pub(crate) const fn size_for_elems(self, elems: usize) -> Option { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::size_for_elems)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().size_for_elems(kani::any()); + } + + // Let `size_align` and `size_phase` denote the components stored in + // `size_rounding_align_and_phase`. The normalized layout formula, + // evaluated over the nonnegative integers, is: + // + // object_size(trailing_bytes) + // = self.size_base + // + round_up(size_phase + trailing_bytes, size_align) + // + // Here, `round_up` gives the least multiple of `size_align` greater + // than or equal to its input. Thus `object_size` is nondecreasing. + // By the capacity helper's contract, `max_trailing_bytes` is the + // largest byte count for which `object_size` fits in a `usize`. + // Monotonicity then gives the equivalence: + // + // object_size(trailing_bytes) <= usize::MAX + // iff trailing_bytes <= max_trailing_bytes. + // + // If the helper returns `None`, even `object_size(0)` exceeds the + // limit, so no element count can produce a representable size. + let max_trailing_bytes = match self.max_trailing_bytes(usize::MAX) { + Some(bytes) => bytes, + None => return None, + }; + + // Each of the `elems` elements contributes `self.elem_size` bytes, + // so substitution into the formula requires their product. Checked + // multiplication yields that exact product, including zero for + // zero-sized elements. If it overflows, the complete size cannot fit + // either: `object_size(trailing_bytes) >= trailing_bytes` because + // the base, phase, and rounding increment are all nonnegative. + let trailing_size = match self.elem_size.checked_mul(elems) { + Some(bytes) => bytes, + None => return None, + }; + + // The equivalence above rejects exactly the remaining cases whose + // complete size exceeds `usize::MAX`. After this check we have + // `object_size(trailing_size) <= usize::MAX`. + if trailing_size > max_trailing_bytes { + return None; + } + + // The trailing slice starts at `self.offset` and occupies + // `trailing_size` bytes. Thus `trailing_end` is its end offset modulo + // `usize::MAX + 1`. This sum need not fit: a raw layout need not + // describe a Rust type, so its trailing slice need not lie within + // the complete object size bounded above. + let trailing_end = self.offset.wrapping_add(trailing_size); + + // `padding_for_elems` returns `object_size(trailing_size)` minus the + // trailing-slice end, modulo `usize::MAX + 1`. Adding the end back + // therefore gives `object_size(trailing_size)` modulo the same + // modulus. The bound above establishes that this size fits in + // `usize`, so its residue is the exact size specified by the layout. + let size = trailing_end.wrapping_add(self.padding_for_elems(elems)); + Some(size) + } + + /// Returns `true` only when `self` and `other` describe the same size for + /// every trailing slice length. + /// + /// This recognizes sufficient conditions for equality. A `false` result + /// also includes equivalent size sequences that this test cannot recognize. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&same| { + let (self_align, self_phase) = self.size_rounding_align_and_phase.components(); + let (other_align, other_phase) = other.size_rounding_align_and_phase.components(); + let initial_size = proofs::slice_dst_size_for_trailing_bytes(self, 0); + same == (self.elem_size == other.elem_size + && initial_size.is_some() + && initial_size == proofs::slice_dst_size_for_trailing_bytes(other, 0) + && ((self.elem_size % self_align.get() == 0 + && self.elem_size % other_align.get() == 0) + || (self_align == other_align && self_phase == other_phase))) + }))] + pub(crate) const fn has_same_size_sequence(self, other: Self) -> bool { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::has_same_size_sequence)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().has_same_size_sequence(kani::any()); + } + + // For either layout, interpret its normalized size formula over the + // nonnegative integers: + // + // object_size(elems) + // = size_base + // + round_up(size_phase + elems * elem_size, size_align). + // + // In two cases below, we establish equality of these mathematical sizes + // for every `elems`. Equal sizes also have the same `usize` + // representability bound. + // + // Both cases require a common byte contribution at each element count. + // Equal element sizes give that shared term, `elems * elem_size`. + if self.elem_size != other.elem_size { + return false; + } + + // At `elems == 0`, the element contribution vanishes. Require equal, + // representable initial sizes; each case below then shows that + // increasing `elems` adds the same amount to both initial sizes. + match (self.size_for_elems(0), other.size_for_elems(0)) { + (Some(self_size), Some(other_size)) if self_size == other_size => {} + _ => return false, + } + + let (self_align, self_phase) = self.size_rounding_align_and_phase.components(); + let (other_align, other_phase) = other.size_rounding_align_and_phase.components(); + let self_align = self_align.get(); + let other_align = other_align.get(); + + // Case 1: The element size is a multiple of both alignments. + // + // For two power-of-two alignments, the larger is divisible by the + // smaller. Thus `max_align` is their least common multiple, and a byte + // count is divisible by both alignments exactly when it is divisible by + // `max_align`. + let max_align = if self_align > other_align { self_align } else { other_align }; + + // If the element size is divisible by both alignments, so is `elems * + // elem_size`. Adding an alignment multiple commutes with rounding, + // giving the following identity for each layout: + // + // round_up(size_phase + elems * elem_size, size_align) + // = round_up(size_phase, size_align) + elems * elem_size. + // + // Consequently, `object_size(elems) = object_size(0) + elems * + // elem_size`. The initial sizes and element sizes already agree, so the + // complete size sequences agree. This also covers zero-sized elements, + // whose contribution is zero for every element count. + #[allow(clippy::arithmetic_side_effects)] + if self.elem_size % max_align == 0 { + return true; + } + + // Case 2: The rounding alignments and phases match. + // + // Require a common rounding alignment before comparing phases. If the + // alignments differ, neither case establishes equality. + if self_align != other_align { + return false; + } + + // If the phases also agree, both `round_up` operations have the same + // input and alignment at every element count. In particular, their + // rounded terms agree at zero. Subtracting that common term from the + // equal initial sizes proves `self.size_base == other.size_base`. Both + // the fixed and rounded contributions therefore agree for every + // `elems`, which establishes equality of the complete size formulas. + self_phase == other_phase + } + + /// Advances the size formula by `bytes` while replacing its element size. + /// + /// Let `(size_align, size_phase)` be decoded from + /// `self.size_rounding_align_and_phase`. For a trailing slice length + /// `elems`, the returned layout describes: + /// + /// ```text + /// self.size_base + /// + round_up(size_phase + bytes + elems * elem_size, size_align) + /// ``` + /// + /// Here, `elem_size` is the replacement element size passed to this + /// method. The returned layout preserves the physical slice offset + /// `self.offset` and the rounding alignment; advancing changes the size + /// calculation. + /// + /// Returns `None` if advancing cannot produce representable normalized + /// components. Even when those components fit, evaluating the rounded + /// object size may overflow for some or all trailing slice lengths. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|result| { + let expected_offset = self.size_offset().checked_add(bytes); + match result { + Some(advanced) => { + let align = self.size_rounding_align_and_phase.align(); + expected_offset == Some(advanced.size_offset()) + && advanced.offset == self.offset + && advanced.elem_size == elem_size + && advanced.size_rounding_align_and_phase.align() == align + && (advanced.size_base & (align.get() - 1)) == (self.size_base & (align.get() - 1)) + } + None => expected_offset.is_none(), + } + }))] + const fn advance(self, bytes: usize, elem_size: usize) -> Option { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(TrailingSliceLayout::advance)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().advance(kani::any(), kani::any()); + } + + let (size_align, size_phase) = self.size_rounding_align_and_phase.components(); + + // Advancing replaces the phase inside `round_up` with `size_phase + + // bytes`. To normalize this sum, separate it into whole alignment units + // and a remainder. The whole units can move outside `round_up` into the + // base; the remainder becomes the new phase. + // + // Since `size_align` is a nonzero power of two, `align_mask` selects + // exactly the bits representing a remainder modulo `size_align`. + // Nonzero `size_align` also makes this subtraction representable. + #[allow(clippy::arithmetic_side_effects)] + let align_mask = size_align.get() - 1; + + // The base can increase by at most `usize::MAX - self.size_base`. Since + // only whole alignment units move into the base, the largest + // transferable contribution is: + // + // whole_capacity + // = round_down(usize::MAX - self.size_base, size_align). + // + // A remainder up to `align_mask` stays in the phase and consumes no + // base capacity. Thus the largest advanced phase that can be split into + // representable normalized components is `phase_capacity = + // whole_capacity + align_mask`. Setting the low bits of the remaining + // base capacity computes exactly that sum. Its two terms have disjoint + // set bits, so it cannot overflow. Any larger sum would transfer more + // whole units than the base can hold. + #[allow(clippy::arithmetic_side_effects)] + let phase_capacity = (usize::MAX - self.size_base) | align_mask; + + // The existing phase already occupies `size_phase` bytes of this + // capacity. Subtracting it gives the greatest permitted increment: + // + // bytes <= max_advance + // iff size_phase + bytes <= phase_capacity. + // + // Since `phase_capacity >= align_mask >= size_phase`, the subtraction + // cannot underflow. + #[allow(clippy::arithmetic_side_effects)] + let max_advance = phase_capacity - size_phase; + if bytes > max_advance { + return None; + } + + // This is the fixed term inside rounding after the requested + // advancement, before normalization. The check above proves `size_phase + // + bytes <= phase_capacity <= usize::MAX`, so the addition is + // representable. + #[allow(clippy::arithmetic_side_effects)] + let advanced_phase = size_phase + bytes; + + // Taking the remainder retains precisely the part that cannot move + // outside rounding as whole alignment units. In particular, `0 <= + // normalized_phase < size_align`, as required for the new rounding + // parameters. + let normalized_phase = advanced_phase & align_mask; + + // Rounding down removes precisely the remainder retained in the + // phase, so `whole_bytes = advanced_phase - normalized_phase`. + let whole_bytes = + util::round_down_to_next_multiple_of_alignment(advanced_phase, size_align); + + // Because `advanced_phase <= phase_capacity`, we have + // `whole_bytes <= whole_capacity <= usize::MAX - self.size_base`. + // Adding this transferred contribution to the base therefore fits. + #[allow(clippy::arithmetic_side_effects)] + let size_base = self.size_base + whole_bytes; + + // To verify the returned formula, let `trailing_size = elems * + // elem_size` and evaluate the following over the nonnegative integers. + // Since `whole_bytes` is a multiple of `size_align`, it can move back + // inside `round_up` without changing the result: + // + // size_base + round_up(normalized_phase + trailing_size, size_align) + // = self.size_base + // + round_up(whole_bytes + normalized_phase + trailing_size, size_align) + // = self.size_base + // + round_up(size_phase + bytes + trailing_size, size_align). + // + // This is the requested advanced formula. The replacement element size + // supplies its new coefficient of `elems`; `offset` retains the + // caller's field-placement description. + Some(Self { + offset: self.offset, + elem_size, + size_base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(size_align, normalized_phase), + }) + } } impl SizeInfo { @@ -73,12 +725,42 @@ impl SizeInfo { #[allow(unused)] #[cfg_attr(not(zerocopy_inline_always), inline)] #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(kani, kani::ensures(|result| match (*self, *result) { + (SizeInfo::Sized { size }, Some(SizeInfo::Sized { size: converted })) => { + size == converted + } + (SizeInfo::SliceDst(original), Some(SizeInfo::SliceDst(converted))) => { + converted.elem_size.get() == original.elem_size + && converted.offset == original.offset + && converted.size_base == original.size_base + && converted.size_rounding_align_and_phase == original.size_rounding_align_and_phase + } + (SizeInfo::SliceDst(original), None) => original.elem_size == 0, + _ => false, + }))] const fn try_to_nonzero_elem_size(&self) -> Option> { + #[cfg(kani)] + #[kani::proof_for_contract(SizeInfo::try_to_nonzero_elem_size)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().try_to_nonzero_elem_size(); + } + Some(match *self { SizeInfo::Sized { size } => SizeInfo::Sized { size }, - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base, + size_rounding_align_and_phase, + }) => { if let Some(elem_size) = NonZeroUsize::new(elem_size) { - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base, + size_rounding_align_and_phase, + }) } else { return None; } @@ -87,6 +769,40 @@ const fn try_to_nonzero_elem_size(&self) -> Option> { } } +/// Returns the largest number of `elem_size`-byte elements which fit in +/// `bytes`, along with the number of bytes those elements occupy. +#[cfg_attr( + kani, + kani::ensures(|&(elems, used)| { + (match elems.checked_mul(elem_size.get()) { + Some(product) => { + product == used && used <= bytes && bytes - used < elem_size.get() + } + None => false, + }) && used + .checked_add(elem_size.get()) + .map(|next_used| next_used > bytes) + .unwrap_or(true) + }) +)] +#[inline(always)] +const fn max_elems_for_bytes(bytes: usize, elem_size: NonZeroUsize) -> (usize, usize) { + #[cfg(kani)] + #[kani::proof_for_contract(max_elems_for_bytes)] + #[kani::solver(kissat)] + fn proof() { + max_elems_for_bytes(kani::any(), kani::any()); + } + + #[allow(clippy::arithmetic_side_effects)] + let elems = bytes / elem_size.get(); + let used = match elems.checked_mul(elem_size.get()) { + Some(used) => used, + None => const_unreachable!(), + }; + (elems, used) +} + #[doc(hidden)] #[derive(Copy, Clone)] #[cfg_attr(test, derive(Debug))] @@ -97,6 +813,7 @@ pub enum CastType { } #[cfg_attr(test, derive(Debug))] +#[cfg_attr(kani, derive(kani::Arbitrary))] pub(crate) enum MetadataCastError { Alignment, Size, @@ -128,7 +845,7 @@ impl DstLayout { /// 229. #[cfg(not(kani))] #[cfg(not(target_pointer_width = "16"))] - pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 28) { + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 29) { Some(max_align) => max_align, None => const_unreachable!(), }; @@ -164,7 +881,18 @@ impl DstLayout { /// shallow padding, unsafe code may assume that the result of this method /// accurately reflects the size, alignment, and lack of static shallow /// padding of that type. + #[cfg_attr(kani, kani::ensures(|result| { + result.align == self.align && result.size_info == self.size_info + && result.statically_shallow_unpadded + }))] const fn assume_shallow_unpadded(self) -> Self { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::assume_shallow_unpadded)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().assume_shallow_unpadded(); + } + Self { statically_shallow_unpadded: true, ..self } } @@ -182,7 +910,20 @@ const fn assume_shallow_unpadded(self) -> Self { #[doc(hidden)] #[must_use] #[inline] + #[cfg_attr(kani, kani::requires(repr_align.map_or(true, |align| align.is_power_of_two())))] + #[cfg_attr(kani, kani::ensures(|result| { + result.align == repr_align.unwrap_or(Self::MIN_ALIGN) + && result.size_info == SizeInfo::Sized { size: 0 } + && result.statically_shallow_unpadded + }))] pub const fn new_zst(repr_align: Option) -> DstLayout { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::new_zst)] + #[kani::solver(kissat)] + fn proof() { + let _ = DstLayout::new_zst(kani::any()); + } + let align = match repr_align { Some(align) => align, None => Self::MIN_ALIGN, @@ -206,7 +947,24 @@ pub const fn new_zst(repr_align: Option) -> DstLayout { #[doc(hidden)] #[must_use] #[inline] + #[cfg_attr(kani, kani::ensures(|result| { + result.align.get() == mem::align_of::() + && result.size_info == SizeInfo::Sized { size: mem::size_of::() } + && result.statically_shallow_unpadded == false + }))] pub const fn for_type() -> DstLayout { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::for_type::<()>)] + fn proof_zst() { + let _ = DstLayout::for_type::<()>(); + } + + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::for_type::<(u8, u64)>)] + fn proof_padded() { + let _ = DstLayout::for_type::<(u8, u64)>(); + } + // SAFETY: `align` is correct by construction. `T: Sized`, and so it is // sound to initialize `size_info` to `SizeInfo::Sized { size }`; the // `size` field is also correct by construction. `unpadded` can safely @@ -221,16 +979,39 @@ pub const fn for_type() -> DstLayout { } } - /// Constructs a `DstLayout` which describes a `T` that does not contain - /// padding. + /// Produces `T`'s size and alignment with static shallow padding assumed + /// absent. + /// + /// This method records the assumption without checking `T` for padding. + /// It can be used to ignore padding within a field when computing padding + /// introduced by its enclosing struct. /// /// # Safety /// - /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + /// Unsafe code may rely on the result's size and alignment being correct + /// for `T`. Relying on its recorded absence of static shallow padding + /// requires an independent justification that `T` lacks such padding. #[doc(hidden)] #[must_use] #[inline] + #[cfg_attr(kani, kani::ensures(|result| { + result.align.get() == mem::align_of::() + && result.size_info == SizeInfo::Sized { size: mem::size_of::() } + && result.statically_shallow_unpadded == true + }))] pub const fn for_unpadded_type() -> DstLayout { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::for_unpadded_type::<()>)] + fn proof_zst() { + let _ = DstLayout::for_unpadded_type::<()>(); + } + + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::for_unpadded_type::<(u8, u64)>)] + fn proof_padded() { + let _ = DstLayout::for_unpadded_type::<(u8, u64)>(); + } + Self::for_type::().assume_shallow_unpadded() } @@ -239,7 +1020,31 @@ pub const fn for_unpadded_type() -> DstLayout { /// # Safety /// /// Unsafe code may assume that `DstLayout` is the correct layout for `[T]`. + #[cfg_attr(kani, kani::ensures(|result| { + result.align.get() == mem::align_of::() && result.statically_shallow_unpadded + && matches!(result.size_info, SizeInfo::SliceDst(trailing) + if trailing.offset == 0 && trailing.size_base == 0 + && trailing.elem_size == mem::size_of::() + && trailing.size_rounding_align_and_phase.components() == (result.align, 0)) + }))] pub(crate) const fn for_slice() -> DstLayout { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::for_slice::<()>)] + fn proof_zst() { + let _ = DstLayout::for_slice::<()>(); + } + + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::for_slice::<(u8, u64)>)] + fn proof_padded() { + let _ = DstLayout::for_slice::<(u8, u64)>(); + } + + let align = match NonZeroUsize::new(mem::align_of::()) { + Some(align) => align, + None => const_unreachable!(), + }; + // SAFETY: The alignment of a slice is equal to the alignment of its // element type, and so `align` is initialized correctly. // @@ -249,13 +1054,12 @@ pub(crate) const fn for_slice() -> DstLayout { // construction. Since `[T]` is a (degenerate case of a) slice DST, it // is correct to initialize `size_info` to `SizeInfo::SliceDst`. DstLayout { - align: match NonZeroUsize::new(mem::align_of::()) { - Some(align) => align, - None => const_unreachable!(), - }, + align, size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset: 0, elem_size: mem::size_of::(), + size_base: 0, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(align, 0), }), statically_shallow_unpadded: true, } @@ -273,15 +1077,42 @@ pub(crate) const fn for_slice() -> DstLayout { /// alignment modifiers and fields corresponding to that definition, the /// resulting `DstLayout` will correctly encode the layout of that struct. /// + /// This guarantee uses one explicit compatibility premise for generic + /// indirection: while Rust accepts an instantiation which transitively + /// places a `repr(align)` type inside a `repr(packed)` type, that + /// instantiation follows the otherwise-applicable `repr(C)`, + /// `repr(packed)`, and inner-type layout rules. The Reference prohibits the + /// direct form, and Rust's continued acceptance and layout behavior are + /// outside the algebra proved by this implementation. Re-audit if the + /// compiler rejects such an instantiation or assigns it a different + /// layout. + /// /// We make no guarantees to the behavior of this method when it is invoked /// with arguments that cannot correspond to a valid `repr(C)` struct. #[must_use] #[inline] + #[cfg_attr(kani, kani::requires(proofs::repr_c_layout(repr_align, repr_packed, fields).is_some()))] + #[cfg_attr(kani, kani::ensures(|&result| { + Some(result) == proofs::repr_c_layout(repr_align, repr_packed, fields) + }))] pub const fn for_repr_c_struct( repr_align: Option, repr_packed: Option, fields: &[DstLayout], ) -> DstLayout { + // This harness covers up to three fields, including an optional final + // DST. Arbitrary-length composition is not established by this bound. + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(DstLayout::for_repr_c_struct)] + #[kani::solver(kissat)] + #[kani::unwind(4)] + fn proof() { + let fields: [DstLayout; 3] = kani::any(); + let len: usize = kani::any(); + kani::assume(len <= fields.len()); + let _ = DstLayout::for_repr_c_struct(kani::any(), kani::any(), &fields[..len]); + } + let mut layout = DstLayout::new_zst(repr_align); let mut i = 0; @@ -338,7 +1169,18 @@ pub const fn for_repr_c_struct( #[doc(hidden)] #[must_use] #[inline] + #[cfg_attr(kani, kani::requires(proofs::extended_layout(self, field, repr_packed).is_some()))] + #[cfg_attr(kani, kani::ensures(|&result| { + Some(result) == proofs::extended_layout(self, field, repr_packed) + }))] pub const fn extend(self, field: DstLayout, repr_packed: Option) -> Self { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::extend)] + #[kani::solver(kissat)] + fn proof() { + let _ = kani::any::().extend(kani::any(), kani::any()); + } + use util::{max, min, padding_needed_for}; // If `repr_packed` is `None`, there are no alignment constraints, and @@ -427,6 +1269,8 @@ pub const fn extend(self, field: DstLayout, repr_packed: Option) - SizeInfo::SliceDst(TrailingSliceLayout { offset: trailing_offset, elem_size, + size_base, + size_rounding_align_and_phase, }) => { // If the trailing field is dynamically sized, so too // will the resulting layout. The offset of the trailing @@ -440,11 +1284,30 @@ pub const fn extend(self, field: DstLayout, repr_packed: Option) - // panic otherwise (e.g., combining or aligning the // components would create a size exceeding // `usize::MAX`). - let offset = match offset.checked_add(trailing_offset) { + let trailing_offset = match offset.checked_add(trailing_offset) { Some(offset) => offset, None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), }; - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + + // Before normalization, the composite's size is: + // + // offset + size_base + // + round_up(size_phase + elems * elem_size, size_align) + // + // `offset` does not participate in the rounding, + // so it can be added directly to the normalized + // base. + let size_base = match offset.checked_add(size_base) { + Some(base) => base, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + + SizeInfo::SliceDst(TrailingSliceLayout { + offset: trailing_offset, + elem_size, + size_base, + size_rounding_align_and_phase, + }) } }, ) @@ -459,18 +1322,11 @@ pub const fn extend(self, field: DstLayout, repr_packed: Option) - } /// Like `Layout::pad_to_align`, this routine rounds the size of this layout - /// up to the nearest multiple of this type's alignment or `repr_packed` - /// (whichever is less). This method leaves DST layouts unchanged, since the - /// trailing padding of DSTs is computed at runtime. - /// - /// The accompanying boolean is `true` if the resulting composition of - /// fields necessitated static (as opposed to dynamic) padding; otherwise - /// `false`. + /// up to the nearest multiple of this type's alignment. For DST layouts, + /// this updates the runtime size formula to account for trailing padding. /// /// In order to match the layout of a `#[repr(C)]` struct, this method - /// should be invoked after the invocations of [`DstLayout::extend`]. If - /// `self` corresponds to a type marked with `repr(packed(N))`, then - /// `repr_packed` should be set to `Some(N)`, otherwise `None`. + /// should be invoked after the invocations of [`DstLayout::extend`]. /// /// This method cannot be used to match the layout of a record with the /// default representation, as that representation is mostly unspecified. @@ -480,7 +1336,7 @@ pub const fn extend(self, field: DstLayout, repr_packed: Option) - /// If a (potentially hypothetical) valid `repr(C)` type begins with fields /// whose layout are `self` followed only by zero or more bytes of trailing /// padding (not included in `self`), then unsafe code may rely on - /// `self.pad_to_align(repr_packed)` producing a layout that correctly + /// `self.pad_to_align()` producing a layout that correctly /// encapsulates the layout of that type. /// /// We make no guarantees to the behavior of this method if `self` cannot @@ -489,7 +1345,16 @@ pub const fn extend(self, field: DstLayout, repr_packed: Option) - #[doc(hidden)] #[must_use] #[inline] + #[cfg_attr(kani, kani::requires(proofs::padded_layout(self).is_some()))] + #[cfg_attr(kani, kani::ensures(|&result| Some(result) == proofs::padded_layout(self)))] pub const fn pad_to_align(self) -> Self { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(DstLayout::pad_to_align)] + #[kani::solver(kissat)] + fn proof() { + let _ = kani::any::().pad_to_align(); + } + use util::padding_needed_for; let (static_padding, size_info) = match self.size_info { @@ -504,10 +1369,69 @@ pub const fn pad_to_align(self) -> Self { (padding, SizeInfo::Sized { size }) } // For DST layouts, trailing padding depends on the length of the - // trailing DST and is computed at runtime. This does not alter the - // offset or element size of the layout, so we leave `size_info` - // unchanged. - size_info @ SizeInfo::SliceDst(_) => (0, size_info), + // trailing DST and is computed at runtime. Normalize the composed + // rounding operation back into the representation documented on + // `TrailingSliceLayout`. + SizeInfo::SliceDst(mut trailing) => { + let (size_align, size_phase) = trailing.size_rounding_align_and_phase.components(); + if size_align.get() < self.align.get() { + // For a trailing-slice byte count `trailing_size`, the + // input layout describes: + // + // trailing.size_base + // + round_up(size_phase + trailing_size, size_align) + // + // Here, `size_align < self.align`. Both alignments are + // powers of two, so rounding the complete input size to + // `self.align` is equivalent to: + // + // rounded_base = round_up(trailing.size_base, size_align) + // fixed_contribution = rounded_base + size_phase + // new_phase = fixed_contribution % self.align + // new_base = fixed_contribution - new_phase + // new_base + round_up(new_phase + trailing_size, self.align) + // + // `prove_dst_layout_pad_to_align_dst_outer_alignment_transition` + // checks the exact field transition and preserved + // invariants. Its `_formula` companion proves this + // identity for every representable trailing byte count. + let base_padding = padding_needed_for(trailing.size_base, size_align); + let rounded_base = match trailing.size_base.checked_add(base_padding) { + Some(base) => base, + None => const_panic!("Adding padding caused size to overflow `usize`."), + }; + let fixed_contribution = match rounded_base.checked_add(size_phase) { + Some(bytes) => bytes, + None => const_panic!("Adding padding caused size to overflow `usize`."), + }; + #[allow(clippy::arithmetic_side_effects)] + let phase = fixed_contribution & (self.align.get() - 1); + // Rounding down removes `phase`, the fixed contribution + // modulo `self.align`, leaving the new base. + let size_base = util::round_down_to_next_multiple_of_alignment( + fixed_contribution, + self.align, + ); + trailing.size_base = size_base; + trailing.size_rounding_align_and_phase = + RoundingAlignAndPhase::new(self.align, phase); + } else { + // `size_align` is a multiple of `self.align`, so + // the inner rounded size is already aligned to + // `self.align`. Only `size_base` needs to be rounded. + // `prove_dst_layout_pad_to_align_dst_inner_rounding_transition` + // checks the exact field transition and preserved + // invariants. Its `_formula` companion proves size + // equivalence for every representable trailing byte count. + let padding = padding_needed_for(trailing.size_base, self.align); + trailing.size_base = match trailing.size_base.checked_add(padding) { + Some(base) => base, + None => const_panic!("Adding padding caused size to overflow `usize`."), + }; + } + + (0, SizeInfo::SliceDst(trailing)) + } }; let statically_shallow_unpadded = self.statically_shallow_unpadded && static_padding == 0; @@ -515,48 +1439,127 @@ pub const fn pad_to_align(self) -> Self { DstLayout { align: self.align, size_info, statically_shallow_unpadded } } - /// Produces `true` if `self` requires static padding; otherwise `false`. + /// Produces `false` when this layout is marked as having no static shallow + /// padding, and `true` otherwise. + /// + /// A `true` result is conservative: it does not establish that the type + /// contains padding. For example, this method produces `true` for + /// `DstLayout::for_type::()`, since `for_type` does not record whether + /// static shallow padding is absent. #[must_use] #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&padding| padding == !self.statically_shallow_unpadded))] pub const fn requires_static_padding(self) -> bool { + #[cfg(kani)] + #[kani::proof_for_contract(DstLayout::requires_static_padding)] + #[kani::solver(kissat)] + fn proof() { + let _ = kani::any::().requires_static_padding(); + } + !self.statically_shallow_unpadded } - /// Produces `true` if there exists any metadata for which a type of layout - /// `self` would require dynamic trailing padding; otherwise `false`. + /// Produces `false` only if every valid metadata describes an instance + /// which requires no dynamic trailing padding. + /// + /// A `true` result conservatively selects the dynamic-padding path; it does + /// not guarantee that any valid metadata actually requires padding. #[must_use] #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&padding| match self.size_info { + SizeInfo::Sized { .. } => !padding, + SizeInfo::SliceDst(trailing) => padding == ( + proofs::slice_dst_size_for_trailing_bytes(trailing, 0) != Some(trailing.offset) + || trailing.elem_size % trailing.size_rounding_align_and_phase.align().get() != 0 + ), + }))] pub const fn requires_dynamic_padding(self) -> bool { - // A `% self.align.get()` cannot panic, since `align` is non-zero. - #[allow(clippy::arithmetic_side_effects)] + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(DstLayout::requires_dynamic_padding)] + #[kani::solver(kissat)] + fn proof() { + let _ = kani::any::().requires_dynamic_padding(); + } + match self.size_info { SizeInfo::Sized { .. } => false, SizeInfo::SliceDst(trailing_slice_layout) => { - // SAFETY: This predicate is formally proved sound by - // `proofs::prove_requires_dynamic_padding`. - trailing_slice_layout.offset % self.align.get() != 0 - || trailing_slice_layout.elem_size % self.align.get() != 0 + // SAFETY: `proofs::prove_requires_dynamic_padding` formally + // proves the safety-critical direction: when this predicate + // returns false, no valid metadata requires dynamic padding. + match trailing_slice_layout.size_for_elems(0) { + Some(initial_size) => { + #[allow(clippy::arithmetic_side_effects)] + let has_static_stride = trailing_slice_layout.elem_size + % trailing_slice_layout.size_rounding_align_and_phase.align().get() + == 0; + initial_size != trailing_slice_layout.offset || !has_static_stride + } + None => true, + } } } } - /// Validates that a cast is sound from a layout perspective. + /// Returns the largest trailing slice length whose object size is exactly + /// `size`. /// - /// Validates that the size and alignment requirements of a type with the - /// layout described in `self` would not be violated by performing a - /// `cast_type` cast from a pointer with address `addr` which refers to a - /// memory region of size `bytes_len`. + /// Returns `None` if there is no such length, if this describes a sized + /// type, or if the trailing slice element is zero-sized. + #[inline(always)] + #[cfg_attr(kani, kani::ensures(|&result| { + result == match self.size_info { + SizeInfo::SliceDst(trailing) if trailing.elem_size != 0 => { + proofs::max_trailing_bytes(trailing, size) + .map(|bytes| bytes / trailing.elem_size) + .filter(|&elems| trailing.size_for_elems(elems) == Some(size)) + } + _ => None, + } + }))] + const fn metadata_for_exact_size(&self, size: usize) -> Option { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(DstLayout::metadata_for_exact_size)] + #[kani::solver(kissat)] + fn proof() { + kani::any::().metadata_for_exact_size(kani::any()); + } + + match self.size_info { + SizeInfo::Sized { .. } + | SizeInfo::SliceDst(TrailingSliceLayout { elem_size: 0, .. }) => None, + SizeInfo::SliceDst(_) => { + // Address zero is aligned to every supported alignment, and + // `0 + size` cannot overflow. The validator returns the + // largest metadata whose object fits in `size`; accepting it + // only when the resulting object consumes all `size` bytes + // turns that bounded query into an exact-size query. + match self.validate_cast_and_convert_metadata(0, size, CastType::Prefix) { + Ok((elems, object_size)) if object_size == size => Some(elems), + _ => None, + } + } + } + } + + /// Given a completed type layout, produces metadata and a split index for + /// a prefix or suffix cast that satisfies the type's size and alignment + /// requirements. + /// + /// `addr` and `bytes_len` describe the address and size of the source + /// memory region, and `cast_type` selects its prefix or suffix. /// /// If the cast is valid, `validate_cast_and_convert_metadata` returns - /// `(elems, split_at)`. If `self` describes a dynamically-sized type, then - /// `elems` is the maximum number of trailing slice elements for which a - /// cast would be valid (for sized types, `elem` is meaningless and should - /// be ignored). `split_at` is the index at which to split the memory region - /// in order for the prefix (suffix) to contain the result of the cast, and - /// in order for the remaining suffix (prefix) to contain the leftover - /// bytes. + /// `Ok((elems, split_at))`. For a dynamically-sized type, `elems` is the + /// maximum number of trailing slice elements for which the cast is valid. + /// For sized types, `elems` is meaningless and should be ignored. + /// `split_at` is the index at which to split the memory region so that the + /// prefix (suffix) contains the result of the cast and the remaining + /// suffix (prefix) contains the leftover bytes. /// /// There are three conditions under which a cast can fail: + /// /// - The smallest possible value for the type is larger than the provided /// memory region /// - A prefix cast is requested, and `addr` does not satisfy `self`'s @@ -568,10 +1571,10 @@ pub const fn requires_dynamic_padding(self) -> bool { /// /// # Safety /// - /// The caller may assume that this implementation is correct, and may rely - /// on that assumption for the soundness of their code. In particular, the - /// caller may assume that, if `validate_cast_and_convert_metadata` returns - /// `Some((elems, split_at))`, then: + /// When `self` accurately describes a completed type layout, including + /// trailing padding, and `addr + bytes_len` fits in `usize`, callers may + /// rely on the following guarantees for `Ok((elems, split_at))`: + /// /// - A pointer to the type (for dynamically sized types, this includes /// `elems` as its pointer metadata) describes an object of size `size <= /// bytes_len` @@ -582,6 +1585,11 @@ pub const fn requires_dynamic_padding(self) -> bool { /// - `split_at == bytes_len - size` /// - `addr + split_at` satisfies `self`'s alignment /// + /// These guarantees require the completed-layout premise; `DstLayout` + /// also permits unfinished layout fragments and arbitrary field values. + /// In particular, suffix alignment relies on every object size being a + /// multiple of the type's alignment. + /// /// Note that this method does *not* ensure that a pointer constructed from /// its return values will be a valid pointer. In particular, this method /// does not reason about `isize` overflow, which is a requirement of many @@ -606,12 +1614,57 @@ pub const fn requires_dynamic_padding(self) -> bool { /// condition, even if `debug_assertions` are enabled. #[allow(unused)] #[inline(always)] + #[cfg_attr(kani, kani::requires(addr.checked_add(bytes_len).is_some()))] + #[cfg_attr(kani, kani::requires(!matches!(self.size_info, + SizeInfo::SliceDst(TrailingSliceLayout { elem_size: 0, .. }))))] + #[cfg_attr(kani, kani::ensures(|result| { + let endpoint = match cast_type { + CastType::Prefix => addr, + CastType::Suffix => addr + bytes_len, + }; + if endpoint % self.align.get() != 0 { + return matches!(result, Err(MetadataCastError::Alignment)); + } + let candidate = match self.size_info { + SizeInfo::Sized { size } => { + if size <= bytes_len { Some((0, size)) } else { None } + } + SizeInfo::SliceDst(trailing) => { + proofs::max_trailing_bytes(trailing, bytes_len).and_then(|bytes| { + let elems = bytes / trailing.elem_size; + trailing.size_for_elems(elems).map(|size| (elems, size)) + }) + } + }; + match (result, candidate) { + (Ok((elems, split_at)), Some((expected_elems, size))) => { + *elems == expected_elems && *split_at == match cast_type { + CastType::Prefix => size, + CastType::Suffix => bytes_len - size, + } + } + (Err(MetadataCastError::Size), None) => true, + _ => false, + } + }))] pub(crate) const fn validate_cast_and_convert_metadata( &self, addr: usize, bytes_len: usize, cast_type: CastType, ) -> Result<(usize, usize), MetadataCastError> { + #[cfg(all(kani, kani_slow))] + #[kani::proof_for_contract(DstLayout::validate_cast_and_convert_metadata)] + #[kani::solver(kissat)] + fn proof() { + let cast_type = if kani::any() { CastType::Prefix } else { CastType::Suffix }; + let _ = kani::any::().validate_cast_and_convert_metadata( + kani::any(), + kani::any(), + cast_type, + ); + } + // `debug_assert!`, but with `#[allow(clippy::arithmetic_side_effects)]`. macro_rules! __const_debug_assert { ($e:expr $(, $msg:expr)?) => { @@ -651,9 +1704,8 @@ macro_rules! __const_debug_assert { // of this check is trivial - `addr` is the address the object will // live at. // - // For a suffix cast, we know that all valid sizes for the type are - // a multiple of the alignment (and by safety precondition, we know - // `DstLayout` may only describe valid Rust types). Thus, a + // Under the documented completed-layout premise, all valid sizes + // for the type are multiples of its alignment. Thus, a // validly-sized instance which lives at a validly-aligned address // must also end at a validly-aligned address. Thus, if the end // address for a suffix cast (`addr + bytes_len`) is not aligned, @@ -666,7 +1718,7 @@ macro_rules! __const_debug_assert { // Addition is guaranteed not to overflow because `offset <= // bytes_len`, and `addr + bytes_len <= usize::MAX` is a // precondition of this method. Modulus is guaranteed not to divide - // by 0 because `align` is non-zero. + // by zero because `self.align` is non-zero. #[allow(clippy::arithmetic_side_effects)] if (addr + offset) % self.align.get() != 0 { return Err(MetadataCastError::Alignment); @@ -680,59 +1732,50 @@ macro_rules! __const_debug_assert { } (0, size) } - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { - // Calculate the maximum number of bytes that could be consumed - // - any number of bytes larger than this will either not be a - // multiple of the alignment, or will be larger than - // `bytes_len`. - let max_total_bytes = - util::round_down_to_next_multiple_of_alignment(bytes_len, self.align); - // Calculate the maximum number of bytes that could be consumed - // by the trailing slice. - // - // FIXME(#67): Once our MSRV is 1.65, use let-else: - // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements - let max_slice_and_padding_bytes = match max_total_bytes.checked_sub(offset) { - Some(max) => max, - // `bytes_len` too small even for 0 trailing slice elements. + SizeInfo::SliceDst(trailing) => { + let max_slice_bytes = match trailing.max_trailing_bytes(bytes_len) { + Some(bytes) => bytes, None => return Err(MetadataCastError::Size), }; - // Calculate the number of elements that fit in - // `max_slice_and_padding_bytes`; any remaining bytes will be - // considered padding. + // `max_slice_bytes`. Any remaining input bytes stay outside + // the selected object unless the object's own rounding + // operation consumes them as padding. // // Guaranteed not to divide by zero: `elem_size` is non-zero. - #[allow(clippy::arithmetic_side_effects)] - let elems = max_slice_and_padding_bytes / elem_size.get(); - // Guaranteed not to overflow on multiplication: `usize::MAX >= - // max_slice_and_padding_bytes >= (max_slice_and_padding_bytes / - // elem_size) * elem_size`. + let (elems, _) = max_elems_for_bytes(max_slice_bytes, trailing.elem_size); + // The remainder is `max_slice_bytes - elems * elem_size`. + // Writing it as a remainder exposes its bound to the + // optimizer: when `elem_size <= size_align`, rounding the + // remainder down to `size_align` always gives zero. + let size_align = trailing.size_rounding_align_and_phase.align(); + // `size_phase + max_slice_bytes == max_rounded_bytes`, an + // alignment multiple. The selected tail removes `unused_bytes` + // from this boundary; rounding back up removes only whole + // alignment units. Thus its normalized rounded size is: // - // Guaranteed not to overflow on addition: - // - max_slice_and_padding_bytes == max_total_bytes - offset - // - elems * elem_size <= max_slice_and_padding_bytes == max_total_bytes - offset - // - elems * elem_size + offset <= max_total_bytes <= usize::MAX - #[allow(clippy::arithmetic_side_effects)] - let without_padding = offset + elems * elem_size.get(); - // `self_bytes` is equal to the offset bytes plus the bytes - // consumed by the trailing slice plus any padding bytes - // required to satisfy the alignment. Note that we have computed - // the maximum number of trailing slice elements that could fit - // in `self_bytes`, so any padding is guaranteed to be less than - // the size of an extra element. + // round_up(size_phase + elems * elem_size, size_align) + // = max_rounded_bytes - round_down(unused_bytes, size_align) // - // Guaranteed not to overflow: - // - By previous comment: without_padding == elems * elem_size + - // offset <= max_total_bytes - // - By construction, `max_total_bytes` is a multiple of - // `self.align`. - // - At most, adding padding needed to round `without_padding` - // up to the next multiple of the alignment will bring - // `self_bytes` up to `max_total_bytes`. + // `prove_slice_dst_validator_selected_size_fits` checks this + // identity and its intermediate bounds, using the relations + // in `prove_max_trailing_bytes_components`. The capacity is + // exact by `prove_max_trailing_bytes`; the element helper's + // contract establishes the largest whole-element count. #[allow(clippy::arithmetic_side_effects)] - let self_bytes = - without_padding + util::padding_needed_for(without_padding, self.align); + let unused_bytes = max_slice_bytes % trailing.elem_size.get(); + let unused_aligned_bytes = + util::round_down_to_next_multiple_of_alignment(unused_bytes, size_align); + #[allow(clippy::arithmetic_side_effects)] + let max_rounded_bytes = util::round_down_to_next_multiple_of_alignment( + bytes_len - trailing.size_base, + size_align, + ); + // `unused_aligned_bytes <= unused_bytes <= max_slice_bytes + // <= max_rounded_bytes`, and `size_base + max_rounded_bytes + // <= bytes_len`, so neither operation can overflow. + #[allow(clippy::arithmetic_side_effects)] + let self_bytes = trailing.size_base + (max_rounded_bytes - unused_aligned_bytes); (elems, self_bytes) } }; @@ -745,7 +1788,7 @@ macro_rules! __const_debug_assert { // - In the `Sized` branch, only returns `size` if `size <= // bytes_len`. // - In the `SliceDst` branch, calculates `self_bytes <= - // max_toatl_bytes`, which is upper-bounded by `bytes_len`. + // bytes_len`. #[allow(clippy::arithmetic_side_effects)] CastType::Suffix => bytes_len - self_bytes, }; @@ -754,6 +1797,95 @@ macro_rules! __const_debug_assert { } } +/// Computes the affine destination-metadata map. +/// +/// # Safety +/// +/// `base + metadata * multiple`, evaluated over the mathematical nonnegative +/// integers, must not exceed `usize::MAX`. +#[inline(always)] +#[allow(unstable_name_collisions)] +#[cfg_attr(kani, kani::requires(metadata.checked_mul(multiple) + .and_then(|scaled| base.checked_add(scaled)).is_some()))] +#[cfg_attr(kani, kani::ensures(|&result| { + metadata.checked_mul(multiple).and_then(|scaled| base.checked_add(scaled)) == Some(result) +}))] +unsafe fn add_scaled_metadata(base: usize, metadata: usize, multiple: usize) -> usize { + // The call-site safety argument establishes that both operations are + // representable for valid source metadata. This harness verifies the + // production implementation selected by Kani's modern rustc against + // checked arithmetic under precisely that precondition. The distributive + // natural-number argument which establishes the precondition is kept + // beside the unsafe pointer projection where it can be reviewed with the + // layout premises that it consumes. + #[cfg(kani)] + #[kani::proof_for_contract(add_scaled_metadata)] + #[kani::solver(kissat)] + fn proof() { + let base: usize = kani::any(); + let metadata: usize = kani::any(); + let multiple: usize = kani::any(); + let Some(scaled) = metadata.checked_mul(multiple) else { + kani::assume(false); + loop {} + }; + let Some(expected) = base.checked_add(scaled) else { + kani::assume(false); + loop {} + }; + + // SAFETY: The checked operations above returned `Some`, so the + // operation is representable in `usize`. The standard-library + // contracts say that this rules out undefined behavior for both + // unchecked operations [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.93.1/std/primitive.usize.html#method.unchecked_mul: + // + // This results in undefined behavior [..] when `checked_mul` would + // return `None`. + // + // [2] Per https://doc.rust-lang.org/1.93.1/std/primitive.usize.html#method.unchecked_add: + // + // This results in undefined behavior [..] when `checked_add` would + // return `None`. + assert_eq!(unsafe { add_scaled_metadata(base, metadata, multiple) }, expected); + } + + #[allow(unused_imports)] + use crate::util::polyfills::*; + + // SAFETY: The caller promises that `base + metadata * multiple` is + // representable over the nonnegative integers. Thus `metadata * multiple` + // is representable too. On toolchains with inherent unchecked arithmetic, + // this is exactly the condition under which `usize::unchecked_mul` does + // not have undefined behavior [1]. On older supported toolchains, method + // resolution selects `polyfills::NumExt::unchecked_mul`, whose safety + // contract requires the same no-overflow condition. Kani checks the two + // implementations in separate harnesses. + // + // [1] Per https://doc.rust-lang.org/1.93.1/std/primitive.usize.html#method.unchecked_mul: + // + // This results in undefined behavior when `self * rhs > usize::MAX` or + // `self * rhs < usize::MIN`, i.e. when `checked_mul` would return `None`. + // + let scaled = unsafe { metadata.unchecked_mul(multiple) }; + + // SAFETY: The caller promises that `base + metadata * multiple` is + // representable in `usize`, and `scaled == metadata * multiple`. On + // toolchains with inherent unchecked arithmetic, that is exactly the + // condition under which `usize::unchecked_add` does not have undefined + // behavior [1]. On older supported toolchains, method resolution selects + // `polyfills::NumExt::unchecked_add`, whose safety contract requires the + // same no-overflow condition. Kani checks the two implementations in + // separate harnesses. + // + // [1] Per https://doc.rust-lang.org/1.93.1/std/primitive.usize.html#method.unchecked_add: + // + // This results in undefined behavior when `self + rhs > usize::MAX` or + // `self + rhs < usize::MIN`, i.e. when `checked_add` would return `None`. + unsafe { base.unchecked_add(scaled) } +} + pub(crate) use cast_from::CastFrom; mod cast_from { use crate::*; @@ -823,52 +1955,19 @@ enum CastParamsInner { // - If this is possible, any information necessary to perform // the `Src`->`Dst` metadata conversion at runtime. // - // Assume that `Src` and `Dst` are slice DSTs, and define: - // - `S_OFF = Src::LAYOUT.size_info.offset` - // - `S_ELEM = Src::LAYOUT.size_info.elem_size` - // - `D_OFF = Dst::LAYOUT.size_info.offset` - // - `D_ELEM = Dst::LAYOUT.size_info.elem_size` + // For slice DSTs, destination metadata is an affine function + // of source metadata: // - // We are trying to solve the following equation: + // dst_meta = offset_delta_elems + src_meta * elem_multiple // - // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM - // - // At runtime, we will be attempting to compute `d_meta`, given - // `s_meta` (a runtime value) and all other parameters (which - // are compile-time values). We can solve like so: - // - // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM - // - // d_meta * D_ELEM = S_OFF - D_OFF + s_meta * S_ELEM - // - // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM - // - // Since `d_meta` will be a `usize`, we need the right-hand side - // to be an integer, and this needs to hold for *any* value of - // `s_meta` (in order for our conversion to be infallible - ie, - // to not have to reject certain values of `s_meta` at runtime). - // This means that: - // - // - `s_meta * S_ELEM` must be a multiple of `D_ELEM` - // - Since this must hold for any value of `s_meta`, `S_ELEM` - // must be a multiple of `D_ELEM` - // - `S_OFF - D_OFF` must be a multiple of `D_ELEM` - // - // Thus, let `OFFSET_DELTA_ELEMS = (S_OFF - D_OFF)/D_ELEM` and - // `ELEM_MULTIPLE = S_ELEM/D_ELEM`. We can rewrite the above - // expression as: - // - // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM - // - // d_meta = OFFSET_DELTA_ELEMS + s_meta * ELEM_MULTIPLE - // - // Thus, we just need to compute the following and confirm that - // they have integer solutions in order to both a) determine - // whether infallible `Src` -> `Dst` casts are possible and, b) - // pre-compute the parameters necessary to perform those casts - // at runtime. These parameters are encapsulated in - // `CastParams`, which acts as a witness that such infallible - // casts are possible. + // `elem_multiple` scales the destination's trailing element + // size to the source's. `offset_delta_elems` selects a + // destination size equal to the source's zero-element size. + // We then prove that the two complete rounded size formulas + // produce the same sequence for all metadata. This is + // necessary because a packed outer DST can retain a nested + // field's rounding operation; comparing only physical + // trailing-slice offsets is not sufficient. /// The parameters required in order to perform an /// unsized-to-unsized pointer cast from `Src` to `Dst` as /// described above. @@ -910,15 +2009,60 @@ fn clone(&self) -> Self { } impl CastParams { + /// Given a nonzero `dst.elem_size` that exactly divides + /// `src.elem_size`, produces `true` only when the following + /// metadata map preserves the complete object size for every + /// source element count `src_meta`: + /// + /// ```text + /// dst_meta = dst_base + src_meta * (src.elem_size / dst.elem_size) + /// ``` + /// + /// This recognizes sufficient conditions for equality. A + /// `false` result also includes equivalent size sequences + /// that this test cannot recognize, or an adjustment whose + /// normalized components cannot be represented in `usize`. + const fn size_sequences_match( + src: TrailingSliceLayout, + dst: TrailingSliceLayout, + dst_base: usize, + ) -> bool { + let base_bytes = match dst_base.checked_mul(dst.elem_size) { + Some(bytes) => bytes, + None => return false, + }; + let shifted_dst = match dst.advance(base_bytes, src.elem_size) { + Some(layout) => layout, + None => return false, + }; + src.has_same_size_sequence(shifted_dst) + } + + /// Given the complete layouts of `Src` and `Dst`, produces + /// parameters for a cast that preserves object size. + /// + /// A `Some` result establishes that `Src`'s alignment is at + /// least `Dst`'s and that the metadata map preserves size for + /// every valid source metadata value. + /// + /// Supports casts between sized types of equal size, from a + /// sized type to a slice DST, and between slice DSTs. A slice + /// DST destination must have nonzero-sized elements; between + /// slice DSTs, its element size must exactly divide the + /// source's nonzero element size. + /// + /// Produces `None` for unsupported casts or when this method + /// cannot establish a size-preserving metadata map. Rejection + /// does not establish that no such map exists. const fn try_compute( - src: &DstLayout, - dst: &DstLayout, + src_layout: &DstLayout, + dst_layout: &DstLayout, ) -> Option> { - if src.align.get() < dst.align.get() { + if src_layout.align.get() < dst_layout.align.get() { return None; } - let inner = match (src.size_info, dst.size_info) { + let inner = match (src_layout.size_info, dst_layout.size_info) { ( SizeInfo::Sized { size: src_size }, SizeInfo::Sized { size: dst_size }, @@ -931,82 +2075,79 @@ const fn try_compute( // dst_size`. CastParamsInner::SizedToSized } - (SizeInfo::Sized { size: src_size }, SizeInfo::SliceDst(dst)) => { - let offset_delta = if let Some(od) = src_size.checked_sub(dst.offset) { - od - } else { - return None; + (SizeInfo::Sized { size: src_size }, SizeInfo::SliceDst(_)) => { + let dst_meta = match dst_layout.metadata_for_exact_size(src_size) { + Some(meta) => meta, + None => return None, }; - let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { - e - } else { - return None; - }; - - // PANICS: `dst_elem_size: NonZeroUsize`, so this won't - // divide by zero. - #[allow(clippy::arithmetic_side_effects)] - let delta_mod_other_elem = offset_delta % dst_elem_size.get(); - - if delta_mod_other_elem != 0 { - return None; - } - - // PANICS: `dst_elem_size: NonZeroUsize`, so this won't - // divide by zero. - #[allow(clippy::arithmetic_side_effects)] - let dst_meta = offset_delta / dst_elem_size.get(); - // SAFETY: The preceding math ensures that a `Dst` // with `dst_meta` addresses `src_size` bytes. CastParamsInner::SizedToUnsized { dst_meta } } (SizeInfo::SliceDst(src), SizeInfo::SliceDst(dst)) => { - let offset_delta = if let Some(od) = src.offset.checked_sub(dst.offset) - { - od - } else { - return None; - }; - let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { e } else { return None; }; - // PANICS: `dst_elem_size: NonZeroUsize`, so this won't - // divide by zero. - #[allow(clippy::arithmetic_side_effects)] - let delta_mod_other_elem = offset_delta % dst_elem_size.get(); - - // PANICS: `dst_elem_size: NonZeroUsize`, so this won't - // divide by zero. - #[allow(clippy::arithmetic_side_effects)] - let elem_remainder = src.elem_size % dst_elem_size.get(); - - if delta_mod_other_elem != 0 - || src.elem_size < dst.elem_size - || elem_remainder != 0 - { + if src.elem_size < dst.elem_size { return None; } - // PANICS: `dst_elem_size: NonZeroUsize`, so this won't - // divide by zero. - #[allow(clippy::arithmetic_side_effects)] - let offset_delta_elems = offset_delta / dst_elem_size.get(); + let (elem_multiple, described_src_elem_size) = + super::max_elems_for_bytes(src.elem_size, dst_elem_size); + if described_src_elem_size != src.elem_size { + return None; + } - // PANICS: `dst_elem_size: NonZeroUsize`, so this won't - // divide by zero. - #[allow(clippy::arithmetic_side_effects)] - let elem_multiple = src.elem_size / dst_elem_size.get(); + // Prefer an exact destination-element adjustment + // between `src.size_offset()` and + // `dst.size_offset()`. This preserves offset-based + // cast compatibility and metadata selection when + // several lengths have the same rounded size. It + // also avoids choosing a length whose subsequent + // padding sequence differs from the source's. + let exact_offset_delta_elems = + match src.size_offset().checked_sub(dst.size_offset()) { + Some(delta) => { + let (elems, described_delta) = + super::max_elems_for_bytes(delta, dst_elem_size); + if described_delta == delta { + Some(elems) + } else { + None + } + } + None => None, + }; + + let offset_delta_elems = match exact_offset_delta_elems { + Some(elems) if Self::size_sequences_match(src, dst, elems) => elems, + _ => { + let src_zero_size = match src.size_for_elems(0) { + Some(size) => size, + None => return None, + }; + let elems = + match dst_layout.metadata_for_exact_size(src_zero_size) { + Some(elems) => elems, + None => return None, + }; + if !Self::size_sequences_match(src, dst, elems) { + return None; + } + elems + } + }; CastParamsInner::UnsizedToUnsized { - // SAFETY: We checked above that this is an exact ratio. + // SAFETY: `size_sequences_match` proves that + // this affine metadata map preserves size. offset_delta_elems, - // SAFETY: We checked above that this is an exact ratio. + // SAFETY: We checked above that this is an exact + // ratio of source to destination element size. elem_multiple, } } @@ -1019,40 +2160,77 @@ const fn try_compute( } impl CastParams { + /// Produces `Dst` metadata describing an object of the same + /// size as the `Src` described by `src_meta`. + /// /// # Safety /// /// `src_meta` describes a `Src` whose size is no larger than /// `isize::MAX`. - /// - /// The returned metadata describes a `Dst` of the same size as - /// the original `Src`. #[inline(always)] unsafe fn cast_metadata( self, src_meta: Src::PointerMetadata, ) -> Dst::PointerMetadata { - #[allow(unused)] - use crate::util::polyfills::*; - let dst_meta = match self.inner { CastParamsInner::UnsizedToUnsized { offset_delta_elems, elem_multiple } => { let src_meta = src_meta.to_elem_count(); - #[allow( - unstable_name_collisions, - clippy::multiple_unsafe_ops_per_block - )] - // SAFETY: `self` is a witness that the following - // equation holds: + // SAFETY: `self` witnesses that this affine map + // makes `Src` and `Dst`'s complete rounded size + // formulas equal for every source metadata value. + // Interpret the arithmetic in this proof over the + // mathematical nonnegative integers. Since the + // caller promises that `src_meta` is valid `Src` + // metadata, the source object size `src_size` it + // describes is at most `isize::MAX`. Let + // `src_elem_size` and `dst_elem_size` be the + // respective trailing-slice element sizes. Then: // - // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // src_meta * src_elem_size + // <= src_size <= isize::MAX. // - // Since the caller promises that `src_meta` is - // valid `Src` metadata, this math will not - // overflow, and the returned value will describe a - // `Dst` of the same size. + // Since `elem_multiple` is the exact ratio + // `src_elem_size / dst_elem_size` and + // `dst_elem_size >= 1`, + // + // src_meta * elem_multiple + // <= src_meta * src_elem_size + // <= isize::MAX <= usize::MAX. + // + // Thus `scaled_elems = src_meta * elem_multiple` + // is representable, and its byte contribution is + // `scaled_elems * dst_elem_size = src_meta * + // src_elem_size`. Let `base_bytes = + // offset_delta_elems * dst_elem_size`. + // `size_sequences_match` compares the source + // formula with the destination formula advanced by + // `base_bytes`. The latter's unrounded term + // contains `base_bytes + src_meta * src_elem_size`, + // and its complete size is at least that large + // (formally checked by Kani in + // `prove_size_formula_bounds_size_offset`). + // Sequence equality therefore gives, over the + // mathematical nonnegative integers, + // + // base_bytes + src_meta * src_elem_size + // = (offset_delta_elems + scaled_elems) + // * dst_elem_size + // <= src_size. + // + // Since `dst_elem_size >= 1`, the metadata sum + // `offset_delta_elems + scaled_elems` is also at + // most `src_size`, and thus at most `isize::MAX <= + // usize::MAX`. The sum is therefore representable, + // and the returned metadata describes a `Dst` of + // the same size. + // SAFETY: The bounds above establish the helper's + // complete no-overflow precondition. unsafe { - offset_delta_elems - .unchecked_add(src_meta.unchecked_mul(elem_multiple)) + super::add_scaled_metadata( + offset_delta_elems, + src_meta, + elem_multiple, + ) } } CastParamsInner::SizedToUnsized { dst_meta } => dst_meta, @@ -1095,6 +2273,55 @@ impl Params for Dst } } +#[cfg(any(test, all(kani, feature = "derive")))] +mod padding_testutil { + use crate::KnownLayout; + + #[derive(KnownLayout)] + #[repr(C, align(4))] + pub(super) struct Aligned { + pub(super) prefix: Prefix, + pub(super) tail: Tail, + } + + /// Compare direct padding with the complete size minus the slice end. + /// Counts whose complete object size exceeds `usize::MAX` are omitted. + pub(super) fn check_padding_for_elems(elems: usize) -> Option + where + Target: ?Sized + KnownLayout, + { + let layout = crate::trailing_slice_layout::(); + let trailing_size = elems.checked_mul(layout.elem_size)?; + + // Evaluate the complete rounded size independently of both helpers: + // `size_for_elems` delegates to the padding method being tested. + let (alignment, phase) = layout.size_rounding_align_and_phase.components(); + #[allow(clippy::arithmetic_side_effects)] + let mask = alignment.get() - 1; + let unrounded = phase.checked_add(trailing_size)?; + let rounded = unrounded.checked_add(mask)? & !mask; + let object_size = layout.size_base.checked_add(rounded)?; + let trailing_end = layout.offset.checked_add(trailing_size).unwrap(); + let expected = object_size.checked_sub(trailing_end).unwrap(); + + let padding = layout.padding_for_elems(elems); + assert_eq!(padding, expected); + Some(padding) + } + + pub(super) fn check_layouts(elems: usize) { + // Slices have no trailing padding. The aligned fixtures cover + // zero and nonzero phases, a three-byte element whose padding + // cycles with the element count, and zero-sized elements. + let _ = check_padding_for_elems::<[u8]>(elems); + let _ = check_padding_for_elems::<[()]>(elems); + let _ = check_padding_for_elems::>(elems); + let _ = check_padding_for_elems::>(elems); + let _ = check_padding_for_elems::>(elems); + let _ = check_padding_for_elems::>(elems); + } +} + // FIXME(#67): For some reason, on our MSRV toolchain, this `allow` isn't // enforced despite having `#![allow(unknown_lints)]` at the crate root, but // putting it here works. Once our MSRV is high enough that this bug has been @@ -1103,12 +2330,560 @@ impl Params for Dst #[cfg(test)] mod tests { use super::*; + use crate::PointerMetadata as _; + + /// Models the size of nested `repr(C)` structs ending in a slice. + /// + /// `leading` lists structs from outermost to innermost. Each tuple gives + /// the packing factor, minimum alignment, and prefix size before padding + /// for the final field. `trailing_elem_size` and `trailing_alignment` give + /// the slice element size and alignment. An empty `leading` describes the + /// slice itself. + /// + /// The returned closure maps a slice length to the complete size, including + /// padding at every nesting level. It returns `None` only on `usize` + /// overflow; sizes above `isize::MAX` are permitted by this arithmetic model. + /// Packing affects field placement, preserving each field's internal + /// padding, even for descriptors that Rust would reject as types. + /// + /// # Panics + /// + /// Panics if any alignment or packing factor is not a power of two, a + /// minimum alignment exceeds its packing factor, or the element size is + /// not a multiple of the slice alignment. + fn size_for_metadata_model( + leading: &[(NonZeroUsize, NonZeroUsize, usize)], + trailing_elem_size: usize, + trailing_alignment: NonZeroUsize, + ) -> impl Fn(usize) -> Option + '_ { + // Rust requires power-of-two alignments and sizes divisible by their + // alignment [1]; alignment modifiers also take powers of two [2]. + // + // [1] https://doc.rust-lang.org/1.93.0/reference/type-layout.html#size-and-alignment + // [2] https://doc.rust-lang.org/1.93.0/reference/type-layout.html#the-alignment-modifiers + assert!(trailing_alignment.get().is_power_of_two()); + assert_eq!(trailing_elem_size % trailing_alignment, 0); + for &(packed, align, _) in leading { + assert!(packed.get().is_power_of_two()); + assert!(align.get().is_power_of_two()); + // The descriptor requires its minimum to fit under the packing + // cap. It also models combinations that Rust forbids, including + // simultaneous `align` and `packed` modifiers [2]. + assert!(align <= packed); + } + + move |elems| { + // A slice shares its array section's layout [3]. Array size is + // element size times length, with the element's alignment [4]. + // + // [3] https://doc.rust-lang.org/1.93.0/reference/type-layout.html#slice-layout + // [4] https://doc.rust-lang.org/1.93.0/reference/type-layout.html#array-layout + let mut size = trailing_elem_size.checked_mul(elems)?; + let mut alignment = trailing_alignment; + + // Work outward with each field's complete size, including its + // trailing padding: an enclosing representation "does not change + // the layout of the fields themselves" [5]. Thus packing a nested + // field preserves the padding already included in `size`. + // + // [5] https://doc.rust-lang.org/1.93.0/reference/type-layout.html#representations + for &(packed, align, prefix) in leading.iter().rev() { + // Packing caps the field's alignment for placement [2]. + let field_align = alignment.min(packed); + + // `repr(C)` places the next field at an aligned offset [6]; + // packing requires the least sufficient inter-field padding + // [2]. Here `prefix` is the end of the preceding fields. + // + // [6] https://doc.rust-lang.org/1.93.0/reference/type-layout.html#reprc-structs + let offset = prefix.checked_add(util::padding_needed_for(prefix, field_align))?; + + // `repr(C)` uses the greatest field alignment [6], and an + // explicit alignment can only raise it [2]. Here `align` + // summarizes the minimum from the prefix and any explicit + // alignment. Both operands are <= `packed`, so their maximum + // also respects the packing cap. + alignment = align.max(field_align); + + // Advance past the complete field, then round to the struct's + // alignment, following `repr(C)`'s final sizing steps [6]. + size = offset.checked_add(size)?; + size = size.checked_add(util::padding_needed_for(size, alignment))?; + } + // Each enclosing size is at least its field's size. Thus any + // checked multiplication or addition that overflowed above would + // also make the final size unrepresentable in `usize`. + Some(size) + } + } + + /// Computes [`size_for_metadata_model`]'s size function using `DstLayout`. + /// + /// Layout construction uses [`DstLayout::for_repr_c_struct`], and the + /// returned closure uses [`crate::PointerMetadata::size_for_metadata`]. + /// + /// # Panics + /// + /// Panics for invalid descriptors as in [`size_for_metadata_model`], or if + /// `DstLayout` cannot construct the layout. This includes static size + /// overflow and, in debug builds, alignments or packing factors above + /// [`DstLayout::CURRENT_MAX_ALIGN`]. + fn size_for_metadata_via_dst_layout( + leading: &[(NonZeroUsize, NonZeroUsize, usize)], + trailing_elem_size: usize, + trailing_alignment: NonZeroUsize, + ) -> impl Fn(usize) -> Option + '_ { + assert_eq!(trailing_elem_size % trailing_alignment, 0); + let mut layout = DstLayout { + align: trailing_alignment, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 0, + elem_size: trailing_elem_size, + size_base: 0, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(trailing_alignment, 0), + }), + statically_shallow_unpadded: true, + }; + for &(packed, align, prefix) in leading.iter().rev() { + assert!(align <= packed); + let prefix = DstLayout { + align: DstLayout::MIN_ALIGN, + size_info: SizeInfo::Sized { size: prefix }, + statically_shallow_unpadded: true, + }; + layout = DstLayout::for_repr_c_struct(Some(align), Some(packed), &[prefix, layout]); + } + move |elems: usize| elems.size_for_metadata(layout) + } + + #[test] + fn test_model_via_dst_layout() { + let nz = |n| NonZeroUsize::new(n).unwrap(); + let cases: &[&[_]] = &[ + &[], + &[(nz(8), nz(1), 3)], + &[(nz(8), nz(8), 3)], + &[(nz(2), nz(1), 1), (nz(8), nz(8), 3)], + &[(nz(8), nz(8), 1), (nz(2), nz(1), 3), (nz(8), nz(8), 5)], + &[(nz(1), nz(1), 1), (nz(2), nz(2), 3), (nz(8), nz(8), 5)], + ]; + for &leading in cases { + for &(elem_size, alignment) in &[(0, 1), (0, 8), (1, 1), (2, 2), (3, 1), (8, 8)] { + let alignment = nz(alignment); + let expected = size_for_metadata_model(leading, elem_size, alignment); + let actual = size_for_metadata_via_dst_layout(leading, elem_size, alignment); + let max_elems = usize::MAX / elem_size.max(1); + for elems in (0..32).chain([ + max_elems - 1, + max_elems, + max_elems.saturating_add(1), + usize::MAX, + ]) { + assert_eq!( + actual(elems), + expected(elems), + "leading: {:?}, trailing: {:?}, elems: {}", + leading, + (elem_size, alignment), + elems, + ); + } + } + } + + for prefix in [usize::MAX - 1, usize::MAX] { + let leading = [(nz(1), nz(1), prefix)]; + let expected = size_for_metadata_model(&leading, 1, nz(1)); + let actual = size_for_metadata_via_dst_layout(&leading, 1, nz(1)); + for elems in [0, 1, 2] { + assert_eq!(actual(elems), expected(elems)); + } + } + } + + // This exercises arithmetic rather than unsafe operations; keep the + // smaller comparison test enabled under Miri instead of this large search. + #[test] + #[cfg_attr(miri, ignore)] + fn test_model_via_dst_layout_combinations() { + use rand::{rngs::SmallRng, Rng as _, SeedableRng as _}; + + fn check( + leading: &[(NonZeroUsize, NonZeroUsize, usize)], + trailing_elem_size: usize, + trailing_alignment: NonZeroUsize, + ) { + let expected = size_for_metadata_model(leading, trailing_elem_size, trailing_alignment); + assert!( + expected(0).is_some(), + "unrepresentable test layout: {:?}, {:?}", + leading, + (trailing_elem_size, trailing_alignment) + ); + let actual = std::panic::catch_unwind(|| { + size_for_metadata_via_dst_layout(leading, trailing_elem_size, trailing_alignment) + }) + .unwrap_or_else(|_| { + panic!( + "DstLayout rejected leading: {:?}, trailing: {:?}", + leading, + (trailing_elem_size, trailing_alignment) + ) + }); + let compare = |elems| { + let expected = expected(elems); + assert_eq!( + actual(elems), + expected, + "leading: {:?}, trailing: {:?}, elems: {}", + leading, + (trailing_elem_size, trailing_alignment), + elems, + ); + expected + }; + + let first_sixteen = 0..=16; + let pointer_width = (0..POINTER_WIDTH_BITS).step_by(4).flat_map(|shift| { + let elems = 1usize << shift; + [elems - 1, elems, elems + 1] + }); + let last_two = (usize::MAX - 1)..=usize::MAX; + let elemses = first_sixteen.chain(pointer_width).chain(last_two); + for elems in elemses { + let _ = compare(elems); + } + + // The modeled size is nondecreasing. Find the last fitting length, + // comparing every probe, then check both sides of that boundary. + // Zero-sized elements can remain representable at usize::MAX. + let (mut low, mut high) = (0, usize::MAX); + while low < high { + let distance = high - low; + let mid = low + distance / 2 + distance % 2; + if compare(mid).is_some() { + low = mid; + } else { + high = mid - 1; + } + } + assert!(compare(low).is_some()); + if let Some(previous) = low.checked_sub(1) { + assert!(compare(previous).is_some()); + } + if let Some(next) = low.checked_add(1) { + assert_eq!(compare(next), None); + } + } + + let nz = |n| NonZeroUsize::new(n).unwrap(); + let check_small_tails = |leading: &[_]| { + for alignment in [1, 2, 4, 8] { + for multiple in [0, 1, 2, 3, 7] { + check(leading, multiple * alignment, nz(alignment)); + } + } + }; + let mut fragments = Vec::new(); + for packed in [1, 2, 4] { + for align in [1, 2, 4].iter().copied().filter(|&align| align <= packed) { + for prefix in 0..=4 { + fragments.push((nz(packed), nz(align), prefix)); + } + } + } + + // Exhaust every combination in this small domain at depths 0, 1, 2. + check_small_tails(&[]); + for &outer in &fragments { + check_small_tails(&[outer]); + for &inner in &fragments { + check_small_tails(&[outer, inner]); + } + } + + let mut rng = SmallRng::seed_from_u64(0); + for depth in (0usize..=128).chain([256, 512, 1024]) { + // A small prefix adds at most 4 * max_align bytes per layer. + // Reserve half of usize::MAX for a large outermost prefix so + // construction remains representable even at the largest depth. + let align_limit = + DstLayout::CURRENT_MAX_ALIGN.get().min(usize::MAX / (8 * depth.max(1))); + let alignments: Vec<_> = (0..POINTER_WIDTH_BITS) + .map(|shift| 1usize << shift) + .take_while(|&align| align <= align_limit) + .collect(); + let last = alignments.len() - 1; + let max_align = alignments[last]; + for case in 0..40 { + let mut leading = Vec::with_capacity(depth); + for level in 0..depth { + // Mix arbitrary modifiers, increasing and decreasing + // alignments, alternating packing, and fully packed trees. + let (packed, align) = match case / 8 { + 0 => { + let packed = rng.gen_range(0..=last); + (packed, rng.gen_range(0..=packed)) + } + 1 => (last, level % alignments.len()), + 2 => (last, last - level % alignments.len()), + 3 => { + let index = if level % 2 == 0 { 0 } else { last }; + (index, index) + } + _ => (0, 0), + }; + let (packed, align) = (alignments[packed], alignments[align]); + let prefix = if level == 0 && case % 4 == 3 { + usize::MAX / 2 + rng.gen_range(0..=2) - 1 + } else { + match rng.gen_range(0..8) { + 0 => 0, + 1 => 1, + 2 => align - 1, + 3 => align, + 4 => align + 1, + 5 => packed - 1, + 6 => packed + 1, + _ => rng.gen_range(0..=2 * max_align), + } + }; + leading.push((nz(packed), nz(align), prefix)); + } + let alignment = alignments[rng.gen_range(0..=last)]; + let elem_size = match case % 8 { + 0 => 0, + 1 => alignment, + 2 => 2 * alignment, + 3 => 3 * alignment, + 4 => 7 * alignment, + 5 => 15 * alignment, + 6 => (usize::MAX / 2 / alignment) * alignment, + _ => (usize::MAX / alignment) * alignment, + }; + check(&leading, elem_size, nz(alignment)); + } + } + } + + const TEST_SIZE_ROUNDING_ALIGN_AND_PHASE_ALIGN: NonZeroUsize = match NonZeroUsize::new(8) { + Some(align) => align, + None => const_unreachable!(), + }; + const TEST_SIZE_ROUNDING_ALIGN_AND_PHASE: RoundingAlignAndPhase = + RoundingAlignAndPhase::new(TEST_SIZE_ROUNDING_ALIGN_AND_PHASE_ALIGN, 3); + const TEST_SIZE_ROUNDING_ALIGN_AND_PHASE_DECODED_ALIGN: usize = + TEST_SIZE_ROUNDING_ALIGN_AND_PHASE.align().get(); + const TEST_SIZE_ROUNDING_ALIGN_AND_PHASE_DECODED_PHASE: usize = + TEST_SIZE_ROUNDING_ALIGN_AND_PHASE.components().1; + + #[test] + #[allow(clippy::arithmetic_side_effects)] + fn test_size_rounding_align_and_phase_encoding() { + // These constants ensure that encoding and decoding remain evaluable + // on every supported compiler, including the MSRV. + assert_eq!(TEST_SIZE_ROUNDING_ALIGN_AND_PHASE_DECODED_ALIGN, 8); + assert_eq!(TEST_SIZE_ROUNDING_ALIGN_AND_PHASE_DECODED_PHASE, 3); + + for shift in 0..POINTER_WIDTH_BITS { + #[allow(clippy::arithmetic_side_effects)] + let align = NonZeroUsize::new(1usize << shift).unwrap(); + for phase in [0, align.get() / 2, align.get() - 1] { + let rounding = RoundingAlignAndPhase::new(align, phase); + assert_eq!(rounding.align(), align); + assert_eq!(rounding.components().1, phase); + assert_eq!(rounding.0.get(), align.get() | phase); + } + } + + let max_align = DstLayout::THEORETICAL_MAX_ALIGN; + let max_phase = max_align.get() - 1; + let max = RoundingAlignAndPhase::new(max_align, max_phase); + assert_eq!(max.0.get(), usize::MAX); + assert_eq!(max.align(), max_align); + assert_eq!(max.components().1, max_phase); + + assert_eq!(mem::size_of::(), 4 * mem::size_of::()); + } + + #[test] + #[allow(clippy::arithmetic_side_effects)] + fn test_trailing_slice_layout_normal_form() { + // Compare the stored base-and-phase formula with + // `prefix + round_up(offset + elems * elem_size, align)`, where + // `prefix < align`. Move whole alignment-sized chunks of `offset` + // into `size_base` and retain the remainder as the phase. Also verify + // that `size_offset()` recovers the exact `offset` used by casts to + // choose between metadata values with the same rounded object size. + for align in [1, 2, 4, 8, 16] { + let align = NonZeroUsize::new(align).unwrap(); + for prefix in 0..align.get() { + for offset in 0..32 { + let phase = offset % align; + let size_base = (offset - phase) | prefix; + let layout = TrailingSliceLayout { + offset: 0, + elem_size: 3, + size_base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(align, phase), + }; + assert_eq!(layout.size_offset(), offset); + + for elems in 0..32 { + let unrounded = offset + elems * layout.elem_size; + let old_size = + prefix + unrounded + util::padding_needed_for(unrounded, align); + assert_eq!(layout.size_for_elems(elems), Some(old_size)); + } + } + } + } + } + + #[test] + #[allow(clippy::arithmetic_side_effects)] + fn test_normalized_size_bounds() { + // Use checked evaluation as an independent oracle, including layouts + // whose empty object is already too large to represent. + let checked_size = |layout: TrailingSliceLayout, trailing_bytes: usize| { + let (alignment, phase) = layout.size_rounding_align_and_phase.components(); + let unrounded = phase.checked_add(trailing_bytes)?; + let rounded = unrounded.checked_add(util::padding_needed_for(unrounded, alignment))?; + layout.size_base.checked_add(rounded) + }; + for exponent in 0..POINTER_WIDTH_BITS { + let alignment = NonZeroUsize::new(1usize << exponent).unwrap(); + for phase in [0, alignment.get() / 2, alignment.get() - 1] { + for base in [0, 1, alignment.get() - 1, alignment.get(), usize::MAX - 1, usize::MAX] + { + let mut layout = TrailingSliceLayout { + offset: 0, + elem_size: 1, + size_base: base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(alignment, phase), + }; + for available in [0, 1, alignment.get() - 1, alignment.get(), usize::MAX] { + match layout.max_trailing_bytes(available) { + Some(max_bytes) => { + assert!(checked_size(layout, max_bytes).unwrap() <= available); + if let Some(next_bytes) = max_bytes.checked_add(1) { + assert!(checked_size(layout, next_bytes) + .map(|size| size > available) + .unwrap_or(true)); + } + } + None => { + assert!(checked_size(layout, 0) + .map(|size| size > available) + .unwrap_or(true)); + } + } + } + + for elem_size in [0, 1, 3, alignment.get(), usize::MAX] { + layout.elem_size = elem_size; + let max_elems = layout + .max_trailing_bytes(usize::MAX) + .and_then(|bytes| bytes.checked_div(elem_size)) + .unwrap_or(usize::MAX); + for elems in [ + 0, + 1, + max_elems.saturating_sub(1), + max_elems, + max_elems.saturating_add(1), + usize::MAX, + ] { + let expected = elems + .checked_mul(elem_size) + .and_then(|bytes| checked_size(layout, bytes)); + // The physical slice offset does not affect + // the size formula. Raw layouts may place the + // slice beyond the object, and adding its byte + // count to the offset may overflow. + for offset in [0, usize::MAX] { + layout.offset = offset; + assert_eq!(layout.size_for_elems(elems), expected); + } + } + } + } + } + } + } + + #[test] + fn test_padding_for_elems_for_rust_layouts() { + use super::padding_testutil::{check_layouts, check_padding_for_elems, Aligned}; + + for elems in 0..256 { + check_layouts(elems); + } + let size_limit = usize::MAX; + for remaining in 0..16 { + check_layouts(size_limit - remaining); + check_layouts(size_limit / 3 - remaining); + } + check_layouts(usize::MAX); + + for (elems, expected) in [(0, 3), (1, 0), (2, 1), (3, 2)] { + assert_eq!( + check_padding_for_elems::>(elems), + Some(expected) + ); + } + // Every count is valid for zero-sized trailing elements. + assert_eq!(check_padding_for_elems::>(usize::MAX), Some(3)); + } + + #[test] + fn test_padding_for_elems() { + for exponent in 0..POINTER_WIDTH_BITS { + let alignment = NonZeroUsize::new(1usize << exponent).unwrap(); + let mask = alignment.get() - 1; + for phase in [0, alignment.get() / 2, mask] { + // Include zero-sized elements, a negative fixed term, fixed + // padding larger than the alignment, and overflowing sums + // and products. Raw layouts need not describe Rust types. + for (offset, elem_size, size_base) in [ + (0, 0, 0), + (0, 1, 1), + (9, 3, 8), + (3, 3, 0), + (9, 3, 12), + (usize::MAX, usize::MAX, usize::MAX), + (usize::MAX, 0, usize::MAX), + ] { + let layout = TrailingSliceLayout { + offset, + elem_size, + size_base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(alignment, phase), + }; + for elems in [0, 1, 2, 3, alignment.get(), usize::MAX] { + // Independently reconstruct the complete rounded + // size and slice end, then subtract. Round up by + // adding the mask and clearing the low bits. + let trailing_bytes = elems.wrapping_mul(elem_size); + let unrounded = phase.wrapping_add(trailing_bytes); + let rounded = unrounded.wrapping_add(mask) & !mask; + let object_size = size_base.wrapping_add(rounded); + let trailing_end = offset.wrapping_add(trailing_bytes); + assert_eq!( + layout.padding_for_elems(elems), + object_size.wrapping_sub(trailing_end), + ); + } + } + } + } + } #[test] fn test_dst_layout_for_slice() { let layout = DstLayout::for_slice::(); match layout.size_info { - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size, .. }) => { assert_eq!(offset, 0); assert_eq!(elem_size, 4); } @@ -1121,17 +2896,18 @@ fn test_dst_layout_for_slice() { #[allow(clippy::decimal_literal_representation)] #[test] fn test_dst_layout_extend_sized_with_sized() { - // This macro constructs a layout corresponding to a `u8` and extends it - // with a zero-sized trailing field of given alignment `n`. The macro - // tests that the resulting layout has both size and alignment `min(n, - // P)` for all valid values of `repr(packed(P))`. + // This macro constructs a layout corresponding to a `u8` and extends + // it with a zero-sized trailing field of the specified alignment. The + // resulting size and alignment must both be the field's alignment + // capped by the packing limit. Exercise every supported packing limit + // as well as the absence of a packing limit. macro_rules! test_align_is_size { ($n:expr) => { let base = DstLayout::for_type::(); let trailing_field = DstLayout::for_type::>(); - let packs = - core::iter::once(None).chain((0..29).map(|p| NonZeroUsize::new(2usize.pow(p)))); + let packs = core::iter::once(None) + .chain((0..=29).map(|p| NonZeroUsize::new(2usize.pow(p)))); for pack in packs { let composite = base.extend(trailing_field, pack); @@ -1179,6 +2955,7 @@ macro_rules! test_align_is_size { test_align_is_size!(33554432); test_align_is_size!(134217728); test_align_is_size!(268435456); + test_align_is_size!(536870912); } /// Tests of when a sized `DstLayout` is extended with a DST field. @@ -1192,15 +2969,29 @@ fn test_dst_layout_extend_sized_with_dst() { let aligns = (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()); let packs = core::iter::once(None).chain(aligns.clone().map(Some)); - for align in aligns { + for field_type_align in aligns { for pack in packs.clone() { let base = DstLayout::for_type::(); let elem_size = 42; let trailing_field_offset = 11; + #[allow(clippy::arithmetic_side_effects)] + let trailing_phase = trailing_field_offset % field_type_align; + let trailing_base = util::round_down_to_next_multiple_of_alignment( + trailing_field_offset, + field_type_align, + ); let trailing_field = DstLayout { - align, - size_info: SizeInfo::SliceDst(TrailingSliceLayout { elem_size, offset: 11 }), + align: field_type_align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + elem_size, + offset: trailing_field_offset, + size_base: trailing_base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + field_type_align, + trailing_phase, + ), + }), statically_shallow_unpadded: false, }; @@ -1208,7 +2999,9 @@ fn test_dst_layout_extend_sized_with_dst() { let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN).get(); - let align = align.get().min(max_align); + let align = field_type_align.get().min(max_align); + let field_offset = align; + let size_base = field_offset + trailing_base; assert_eq!( composite, @@ -1216,7 +3009,12 @@ fn test_dst_layout_extend_sized_with_dst() { align: NonZeroUsize::new(align).unwrap(), size_info: SizeInfo::SliceDst(TrailingSliceLayout { elem_size, - offset: align + trailing_field_offset, + offset: field_offset + trailing_field_offset, + size_base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + field_type_align, + trailing_phase, + ), }), statically_shallow_unpadded: false, } @@ -1292,7 +3090,8 @@ macro_rules! test { => padded { size: current_max_align * 2, align: current_max_align }); } - /// Tests that calling `pad_to_align` on a DST `DstLayout` is a no-op. + /// Tests that calling `pad_to_align` on a DST `DstLayout` incorporates the + /// type's trailing-padding operation into its runtime size formula. #[test] fn test_dst_layout_pad_to_align_with_dst() { for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { @@ -1300,13 +3099,138 @@ fn test_dst_layout_pad_to_align_with_dst() { for elem_size in 0..10 { let layout = DstLayout { align, - size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base: offset, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + DstLayout::MIN_ALIGN, + 0, + ), + }), statically_shallow_unpadded: false, }; - assert_eq!(layout.pad_to_align(), layout); + assert_eq!( + layout.pad_to_align(), + DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base: offset - offset % align, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + align, + offset % align + ), + }), + statically_shallow_unpadded: false, + } + ); } } } + + // Exercise the branch where the existing inner rounding alignment is + // at least the enclosing alignment. The inner rounded term is already + // aligned to two, so padding only advances `size_base` from three to + // four. + let inner_rounding_dominates = DstLayout { + align: NonZeroUsize::new(2).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 3, + elem_size: 1, + size_base: 3, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(4).unwrap(), + 1, + ), + }), + statically_shallow_unpadded: false, + }; + assert_eq!( + inner_rounding_dominates.pad_to_align(), + DstLayout { + align: NonZeroUsize::new(2).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 3, + elem_size: 1, + size_base: 4, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(4).unwrap(), + 1 + ), + }), + statically_shallow_unpadded: false, + } + ); + + // Exercise the branch where the enclosing alignment is larger. For + // a trailing-slice byte count `trailing_size`, rounding + // `6 + round_up(1 + trailing_size, 4)` to eight gives the equivalent + // formula `8 + round_up(1 + trailing_size, 8)`. + let outer_rounding_dominates = DstLayout { + align: NonZeroUsize::new(8).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 7, + elem_size: 1, + size_base: 6, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(4).unwrap(), + 1, + ), + }), + statically_shallow_unpadded: false, + }; + assert_eq!( + outer_rounding_dominates.pad_to_align(), + DstLayout { + align: NonZeroUsize::new(8).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 7, + elem_size: 1, + size_base: 8, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(8).unwrap(), + 1 + ), + }), + statically_shallow_unpadded: false, + } + ); + } + + #[test] + fn test_trailing_slice_layout_size_sequence_equivalence() { + // These are the two normalized descriptions produced by a + // `repr(C, packed(2))` wrapper around slice DSTs whose element types + // both have size four but alignments four and two respectively. Their + // formulas differ structurally, but both simplify to `2 + 4 * elems`. + let align4 = TrailingSliceLayout { + offset: 2, + elem_size: 4, + size_base: 2, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(4).unwrap(), + 0, + ), + }; + let align2 = TrailingSliceLayout { + offset: 2, + elem_size: 4, + size_base: 2, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(2).unwrap(), + 0, + ), + }; + assert!(align4.has_same_size_sequence(align2)); + for elems in 0..8 { + assert_eq!(align4.size_for_elems(elems), Some(2 + 4 * elems)); + assert_eq!(align4.size_for_elems(elems), align2.size_for_elems(elems)); + } + + let dynamically_padded = TrailingSliceLayout { elem_size: 1, ..align4 }; + assert!(!dynamically_padded.has_same_size_sequence(align2)); } // This test takes a long time when running under Miri, so we skip it in @@ -1325,16 +3249,32 @@ fn from(size: usize) -> SizeInfo { #[allow(non_local_definitions)] impl From<(usize, usize)> for SizeInfo { fn from((offset, elem_size): (usize, usize)) -> SizeInfo { - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base: offset, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + DstLayout::MIN_ALIGN, + 0, + ), + }) } } fn layout>(s: S, align: usize) -> DstLayout { - DstLayout { - size_info: s.into(), - align: NonZeroUsize::new(align).unwrap(), - statically_shallow_unpadded: false, - } + let align = NonZeroUsize::new(align).unwrap(); + let size_info = match s.into() { + SizeInfo::SliceDst(mut trailing) => { + let phase = trailing.size_base % align; + trailing.size_base = + util::round_down_to_next_multiple_of_alignment(trailing.size_base, align); + trailing.size_rounding_align_and_phase = + RoundingAlignAndPhase::new(align, phase); + SizeInfo::SliceDst(trailing) + } + size_info => size_info, + }; + DstLayout { size_info, align, statically_shallow_unpadded: false } } /// This macro accepts arguments in the form of: @@ -1531,10 +3471,18 @@ fn validate_behavior( let resulting_size = match layout.size_info { SizeInfo::Sized { size } => size, - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + SizeInfo::SliceDst(TrailingSliceLayout { + elem_size, + size_base, + size_rounding_align_and_phase, + .. + }) => { + let (size_align, size_phase) = size_rounding_align_and_phase.components(); let padded_size = |elems| { - let without_padding = offset + elems * elem_size; - without_padding + util::padding_needed_for(without_padding, align) + let without_padding = size_phase + elems * elem_size; + size_base + + without_padding + + util::padding_needed_for(without_padding, size_align) }; let resulting_size = padded_size(elems); @@ -1562,8 +3510,13 @@ fn validate_behavior( } else { let min_size = match layout.size_info { SizeInfo::Sized { size } => size, - SizeInfo::SliceDst(TrailingSliceLayout { offset, .. }) => { - offset + util::padding_needed_for(offset, layout.align) + SizeInfo::SliceDst(TrailingSliceLayout { + size_base, + size_rounding_align_and_phase, + .. + }) => { + let (size_align, size_phase) = size_rounding_align_and_phase.components(); + size_base + size_phase + util::padding_needed_for(size_phase, size_align) } }; @@ -1592,6 +3545,35 @@ fn validate_behavior( .map(|(size_info, align)| layout(size_info, align)); itertools::iproduct!(layouts, 0..8, 0..8, [CastType::Prefix, CastType::Suffix]) .for_each(validate_behavior); + + // Exercise a normalized formula which cannot be represented as + // `round_up(offset + elems * elem_size, align)`: the physical trailing + // slice begins at offset 7, while an inner aligned DST contributes the + // size formula `2 + round_up(5 + elems, 4)` to a packed outer type. + let nested_packed = DstLayout { + align: NonZeroUsize::new(2).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 7, + elem_size: 1, + size_base: 6, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(4).unwrap(), + 1, + ), + }), + statically_shallow_unpadded: false, + }; + itertools::iproduct!([nested_packed], 0..8, 0..24, [CastType::Prefix, CastType::Suffix]) + .for_each(validate_behavior); + + assert!(matches!( + nested_packed.validate_cast_and_convert_metadata(0, 8, CastType::Prefix), + Err(MetadataCastError::Size) + )); + assert!(matches!( + nested_packed.validate_cast_and_convert_metadata(0, 10, CastType::Prefix), + Ok((3, 10)) + )); } #[test] @@ -1631,9 +3613,18 @@ fn test NonNull>( let dst = args.elem_size.is_some(); let layout = { let size_info = match args.elem_size { - Some(elem_size) => { - SizeInfo::SliceDst(TrailingSliceLayout { offset: args.offset, elem_size }) - } + Some(elem_size) => SizeInfo::SliceDst(TrailingSliceLayout { + offset: args.offset, + elem_size, + size_base: util::round_down_to_next_multiple_of_alignment( + args.offset, + args.align, + ), + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + args.align, + args.offset % args.align, + ), + }), None => SizeInfo::Sized { // Rust only supports types whose sizes are a multiple // of their alignment. If the macro created a type like @@ -1951,70 +3942,748 @@ macro_rules! validate_against_rust { #[cfg(kani)] mod proofs { + //! Arithmetic models and input generators for the layout proofs. + //! + //! The contract models specify expected results without calling the + //! operation being checked. The composition models (`extended_layout`, + //! `padded_layout`, and `repr_c_layout`) also define the contracts' input + //! domains: `Some` supplies an expected layout, while `None` excludes an + //! input from the corresponding non-panicking contract. These models check + //! representability in `usize`, without imposing Rust's `isize::MAX` bound. + use core::alloc::Layout; use super::*; - impl kani::Arbitrary for DstLayout { - fn any() -> Self { - let align: NonZeroUsize = kani::any(); - let size_info: SizeInfo = kani::any(); + /// Evaluates the normalized object size for a trailing slice of + /// `trailing_size` bytes: + /// + /// ```text + /// size_base + round_up(size_phase + trailing_size, size_align) + /// ``` + /// + /// Here, `size_align` and `size_phase` come from `layout.size_rounding_align_and_phase`. + /// `round_up` rounds its input up to a multiple of its alignment. Returns + /// `None` if the size is not representable in `usize`. + /// + /// Using byte counts separates rounding from element-count multiplication + /// and permits proofs over byte counts that are not whole elements. + pub(super) fn slice_dst_size_for_trailing_bytes( + layout: TrailingSliceLayout, + trailing_size: usize, + ) -> Option { + let without_padding = + layout.size_rounding_align_and_phase.components().1.checked_add(trailing_size)?; + let padding = + util::padding_needed_for(without_padding, layout.size_rounding_align_and_phase.align()); + let rounded = without_padding.checked_add(padding)?; + layout.size_base.checked_add(rounded) + } - kani::assume(align.is_power_of_two()); - kani::assume(align < DstLayout::THEORETICAL_MAX_ALIGN); + /// Returns the least multiple of `align` greater than or equal to `bytes`, + /// or `None` if that multiple exceeds `usize::MAX`. + /// + /// Requires a power-of-two `align`. Composition models use this checked + /// operation to identify inputs for which adding padding would overflow. + fn round_up(bytes: usize, align: NonZeroUsize) -> Option { + bytes.checked_add(util::padding_needed_for(bytes, align)) + } - // For testing purposes, we most care about instantiations of - // `DstLayout` that can correspond to actual Rust types. We use - // `Layout` to verify that our `DstLayout` satisfies the validity - // conditions of Rust layouts. - kani::assume( - match size_info { - SizeInfo::Sized { size } => Layout::from_size_align(size, align.get()), - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size: _ }) => { - // `SliceDst` cannot encode an exact size, but we know - // it is at least `offset` bytes. - Layout::from_size_align(offset, align.get()) + /// Models the largest trailing-slice byte count whose object size fits in + /// `available` bytes. Returns `None` if even an empty trailing slice cannot + /// fit. The result need not be a multiple of the element size. + /// + /// Subtracting `size_base` leaves the budget for the rounded contribution. + /// Rounding that budget down to `size_align` and subtracting `size_phase` + /// gives the largest byte count whose `slice_dst_size_for_trailing_bytes` + /// result is at most `available`. Metadata and casting contracts use + /// this bound without calling the production capacity calculation. + pub(super) fn max_trailing_bytes( + trailing: TrailingSliceLayout, + available: usize, + ) -> Option { + let (align, phase) = trailing.size_rounding_align_and_phase.components(); + let capacity = available.checked_sub(trailing.size_base)?; + util::round_down_to_next_multiple_of_alignment(capacity, align).checked_sub(phase) + } + + /// Models appending `field` to a sized `prefix` for `DstLayout::extend`. + /// + /// Places the field at the first offset after the prefix that satisfies the + /// field's alignment, capped by `packed` when present. The resulting + /// alignment is the maximum of the prefix's alignment and this effective + /// field alignment. A DST field retains its inner rounding operation, with + /// its slice offset and size base shifted by the field's placement. + /// Trailing padding for the enclosing layout is left to `padded_layout`. + /// + /// Returns `None` for a non-power-of-two alignment, an unsized prefix, or + /// arithmetic overflow. The `statically_shallow_unpadded` flag is true only + /// if both inputs have it set and field placement introduces no gap. + pub(super) fn extended_layout( + prefix: DstLayout, + field: DstLayout, + packed: Option, + ) -> Option { + if !prefix.align.is_power_of_two() + || !field.align.is_power_of_two() + || packed.map_or(false, |align| !align.is_power_of_two()) + { + return None; + } + let SizeInfo::Sized { size: prefix_size } = prefix.size_info else { return None }; + let field_align = field.align.min(packed.unwrap_or(DstLayout::THEORETICAL_MAX_ALIGN)); + let field_offset = round_up(prefix_size, field_align)?; + let size_info = match field.size_info { + SizeInfo::Sized { size } => SizeInfo::Sized { size: field_offset.checked_add(size)? }, + SizeInfo::SliceDst(trailing) => SizeInfo::SliceDst(TrailingSliceLayout { + offset: field_offset.checked_add(trailing.offset)?, + size_base: field_offset.checked_add(trailing.size_base)?, + ..trailing + }), + }; + Some(DstLayout { + align: prefix.align.max(field_align), + size_info, + statically_shallow_unpadded: prefix.statically_shallow_unpadded + && field.statically_shallow_unpadded + && field_offset == prefix_size, + }) + } + + /// Models adding trailing padding for `DstLayout::pad_to_align`. + /// + /// A sized layout rounds its size up to a multiple of `layout.align`. A DST + /// instead composes that rounding with its existing size expression: + /// + /// ```text + /// round_up(size_base + round_up(size_phase + trailing_size, size_align), + /// layout.align) + /// ``` + /// + /// Here, `size_align` and `size_phase` come from the DST's + /// `size_rounding_align_and_phase`. The result encodes this expression with + /// one base, phase, and rounding alignment, preserving the physical slice + /// offset and element size. The `statically_shallow_unpadded` flag is + /// cleared only when a sized layout gains padding; DST padding is accounted + /// for by the size expression. + /// + /// Returns `None` if `layout.align` is not a power of two or a checked + /// intermediate overflows. The contract compares this expected encoding + /// with the production result; the separate padding formula proofs check + /// that the encoding preserves the rounded size. + pub(super) fn padded_layout(layout: DstLayout) -> Option { + if !layout.align.is_power_of_two() { + return None; + } + let (size_info, no_static_padding) = match layout.size_info { + SizeInfo::Sized { size } => { + let rounded = round_up(size, layout.align)?; + (SizeInfo::Sized { size: rounded }, rounded == size) + } + SizeInfo::SliceDst(trailing) => { + let (inner_align, phase) = trailing.size_rounding_align_and_phase.components(); + let (size_base, size_rounding_align_and_phase) = if inner_align >= layout.align { + ( + round_up(trailing.size_base, layout.align)?, + trailing.size_rounding_align_and_phase, + ) + } else { + let fixed = round_up(trailing.size_base, inner_align)?.checked_add(phase)?; + let new_phase = fixed & (layout.align.get() - 1); + (fixed - new_phase, RoundingAlignAndPhase::new(layout.align, new_phase)) + }; + ( + SizeInfo::SliceDst(TrailingSliceLayout { + size_base, + size_rounding_align_and_phase, + ..trailing + }), + true, + ) + } + }; + Some(DstLayout { + size_info, + statically_shallow_unpadded: layout.statically_shallow_unpadded && no_static_padding, + ..layout + }) + } + + /// Models a complete layout for `DstLayout::for_repr_c_struct` by composing + /// the field-placement and trailing-padding specifications. + /// + /// Starts with an empty prefix of alignment `align`, or one if absent. + /// Appends fields in order, with `packed` as an optional cap on field + /// alignment; the final layout is padded to its resulting alignment. Only + /// the last field may be unsized, since field extension requires a sized + /// prefix. + /// + /// Returns `None` if the initial alignment is not a power of two or any + /// field extension or final padding step returns `None`. + pub(super) fn repr_c_layout( + align: Option, + packed: Option, + fields: &[DstLayout], + ) -> Option { + let align = align.unwrap_or(DstLayout::MIN_ALIGN); + if !align.is_power_of_two() { + return None; + } + let mut layout = DstLayout { + align, + size_info: SizeInfo::Sized { size: 0 }, + statically_shallow_unpadded: true, + }; + for &field in fields { + layout = extended_layout(layout, field, packed)?; + } + padded_layout(layout) + } + + // The inline contract harnesses use unrestricted `Arbitrary` values. + // These generators retain the narrower domains of the Rust-layout proofs. + + /// Generates any power-of-two alignment representable in `usize`, including + /// `DstLayout::THEORETICAL_MAX_ALIGN`. This restricts the alignment alone; + /// callers must impose any constraints relating it to a layout's size. + fn any_layout_align() -> NonZeroUsize { + let exponent: u8 = kani::any(); + kani::assume(usize::from(exponent) < POINTER_WIDTH_BITS); + + // `exponent < POINTER_WIDTH_BITS`, so the shift is in range and its + // result is non-zero. This construction is surjective over every + // power-of-two alignment up to and including + // `THEORETICAL_MAX_ALIGN`. `Layout` admits that maximum alignment for + // zero-sized layouts, so the validity assumptions below—not this + // generator—filter combinations that cannot describe Rust types. + NonZeroUsize::new(1usize << exponent).unwrap() + } + + /// Generates layouts satisfying the arithmetic restrictions used by the + /// standalone Rust-layout proofs. + /// + /// The fixed size, or the zero-element DST size, must be accepted by + /// `Layout::from_size_align`. For a DST, `size_base + size_phase` must also + /// be representable and at least the physical slice offset. Component + /// bounds come from `any_bounded_size_info`; `statically_shallow_unpadded` + /// remains arbitrary. These conditions do not establish that a Rust type + /// with this layout exists. + fn any_valid_dst_layout() -> DstLayout { + let align = any_layout_align(); + let size_info = any_bounded_size_info(); + + kani::assume( + match size_info { + SizeInfo::Sized { size } => Layout::from_size_align(size, align.get()), + SizeInfo::SliceDst(trailing) => { + // `SliceDst` cannot encode one exact size. Validate its + // minimum size and the invariants required of the size + // formula for layouts of real Rust types. + let min_size = trailing.size_for_elems(0); + let unrounded_min_size = trailing + .size_base + .checked_add(trailing.size_rounding_align_and_phase.components().1); + + kani::assume(matches!( + unrounded_min_size, + Some(size) if size >= trailing.offset + )); + + match min_size { + Some(min_size) => Layout::from_size_align(min_size, align.get()), + None => Layout::from_size_align(usize::MAX, align.get()), } } - .is_ok(), - ); - - Self { align: align, size_info: size_info, statically_shallow_unpadded: kani::any() } - } - } - - impl kani::Arbitrary for SizeInfo { - fn any() -> Self { - let is_sized: bool = kani::any(); - - match is_sized { - true => { - let size: usize = kani::any(); - - kani::assume(size <= DstLayout::MAX_SIZE); - - SizeInfo::Sized { size } - } - false => SizeInfo::SliceDst(kani::any()), } + .is_ok(), + ); + + DstLayout { align: align, size_info: size_info, statically_shallow_unpadded: kani::any() } + } + + /// Generates either a fixed size at most `DstLayout::MAX_SIZE`, or a DST + /// description from `any_bounded_trailing_layout`. + /// + /// Supplies bounded size components to `any_valid_dst_layout`, which adds + /// constraints involving the enclosing layout's alignment. + fn any_bounded_size_info() -> SizeInfo { + let is_sized: bool = kani::any(); + + match is_sized { + true => { + let size: usize = kani::any(); + + kani::assume(size <= DstLayout::MAX_SIZE); + + SizeInfo::Sized { size } + } + false => SizeInfo::SliceDst(any_bounded_trailing_layout()), } } - impl kani::Arbitrary for TrailingSliceLayout { - fn any() -> Self { - let elem_size: usize = kani::any(); - let offset: usize = kani::any(); + /// Generates a trailing-slice description whose element size, slice offset, + /// and size base are each at most `DstLayout::MAX_SIZE` (`isize::MAX`). + /// + /// The rounding alignment spans all representable powers of two, and the + /// phase spans all values below that alignment. No further relationships + /// between these components are imposed: their combined size may overflow + /// or fail to contain the slice. Standalone proofs add the relationships + /// they require; contract harnesses use unrestricted `Arbitrary` values + /// instead. + fn any_bounded_trailing_layout() -> TrailingSliceLayout { + let elem_size: usize = kani::any(); + let offset: usize = kani::any(); + let size_base: usize = kani::any(); + let size_align = any_layout_align(); + let raw_size_phase: usize = kani::any(); + // Since `size_align` is a power of two, masking is surjective over + // precisely the values in `0..size_align`. + #[allow(clippy::arithmetic_side_effects)] + let size_phase = raw_size_phase & (size_align.get() - 1); - kani::assume(elem_size < DstLayout::MAX_SIZE); - kani::assume(offset < DstLayout::MAX_SIZE); + kani::assume(elem_size <= DstLayout::MAX_SIZE); + kani::assume(offset <= DstLayout::MAX_SIZE); + kani::assume(size_base <= DstLayout::MAX_SIZE); - TrailingSliceLayout { elem_size, offset } + TrailingSliceLayout { + elem_size, + offset, + size_base, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(size_align, size_phase), + } + } + + #[cfg(feature = "derive")] + #[kani::proof] + fn prove_padding_for_elems_for_rust_layouts() { + // For each fixture, check every element count whose complete size + // fits in `usize`, including all counts for zero-sized elements. + padding_testutil::check_layouts(kani::any()); + } + + #[cfg(kani_slow)] + #[kani::proof] + #[kani::solver(kissat)] + fn prove_padding_for_elems() { + // Check the modular result for every field value and element count, + // including layouts that cannot describe a Rust type. + let layout = TrailingSliceLayout { + offset: kani::any(), + elem_size: kani::any(), + size_base: kani::any(), + size_rounding_align_and_phase: RoundingAlignAndPhase(kani::any()), + }; + let elems: usize = kani::any(); + let (size_align, size_phase) = layout.size_rounding_align_and_phase.components(); + let mask = size_align.get() - 1; + + // Evaluate the complete size and trailing-slice end separately in + // modular arithmetic, without reducing the trailing bytes first or + // canceling them from the difference. + let trailing_bytes = elems.wrapping_mul(layout.elem_size); + let unrounded = size_phase.wrapping_add(trailing_bytes); + let rounded = unrounded.wrapping_add(mask) & !mask; + let object_size = layout.size_base.wrapping_add(rounded); + let trailing_end = layout.offset.wrapping_add(trailing_bytes); + assert_eq!(layout.padding_for_elems(elems), object_size.wrapping_sub(trailing_end)); + } + + #[kani::proof] + fn prove_size_rounding_align_and_phase_encoding() { + let encoded: usize = kani::any(); + kani::assume(encoded != 0); + let encoded = NonZeroUsize::new(encoded).unwrap(); + let rounding = RoundingAlignAndPhase(encoded); + + let align = rounding.align(); + let phase = rounding.components().1; + assert!(align.is_power_of_two()); + assert!(phase < align.get()); + assert_eq!(RoundingAlignAndPhase::new(align, phase), rounding); + } + + #[kani::proof] + fn prove_size_rounding_align_and_phase_decoding() { + let align: NonZeroUsize = kani::any(); + let phase: usize = kani::any(); + kani::assume(align.is_power_of_two()); + kani::assume(phase < align.get()); + + let rounding = RoundingAlignAndPhase::new(align, phase); + assert_eq!(rounding.components(), (align, phase)); + } + + #[cfg(kani_slow)] + #[kani::proof] + fn prove_trailing_slice_layout_normal_form() { + let rounding_word: usize = kani::any(); + kani::assume(rounding_word != 0); + let align = RoundingAlignAndPhase(NonZeroUsize::new(rounding_word).unwrap()).align(); + + let raw_prefix: usize = kani::any(); + let offset: usize = kani::any(); + let trailing_size: usize = kani::any(); + #[allow(clippy::arithmetic_side_effects)] + let prefix = raw_prefix & (align.get() - 1); + #[allow(clippy::arithmetic_side_effects)] + let phase = offset & (align.get() - 1); + #[allow(clippy::arithmetic_side_effects)] + let aligned_offset = offset - phase; + let layout = TrailingSliceLayout { + offset: 0, + elem_size: 1, + size_base: aligned_offset | prefix, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(align, phase), + }; + + assert_eq!(layout.size_offset(), offset); + + let old_size = offset + .checked_add(trailing_size) + .and_then(|without_padding| { + without_padding.checked_add(util::padding_needed_for(without_padding, align)) + }) + .and_then(|rounded| prefix.checked_add(rounded)); + assert_eq!(layout.size_for_elems(trailing_size), old_size); + assert_eq!(slice_dst_size_for_trailing_bytes(layout, trailing_size), old_size); + } + + #[cfg(kani_slow)] + #[kani::proof] + #[kani::solver(kissat)] + fn prove_element_product_preserves_alignment() { + let elem_size: usize = kani::any(); + let elems: usize = kani::any(); + let align = any_layout_align(); + #[allow(clippy::arithmetic_side_effects)] + let align_mask = align.get() - 1; + + kani::assume(elem_size & align_mask == 0); + let Some(bytes) = elem_size.checked_mul(elems) else { + kani::assume(false); + loop {} + }; + assert_eq!(bytes & align_mask, 0); + } + + #[kani::proof] + fn prove_trailing_slice_layout_advance() { + let layout: TrailingSliceLayout = any_bounded_trailing_layout(); + let bytes: usize = kani::any(); + let elem_size: usize = kani::any(); + let trailing_size: usize = kani::any(); + + let size_offset = layout.size_offset(); + let advanced_offset = size_offset.checked_add(bytes); + let advanced = layout.advance(bytes, elem_size); + assert_eq!(advanced.is_some(), advanced_offset.is_some()); + let Some(advanced_offset) = advanced_offset else { + kani::assume(false); + loop {} + }; + let Some(advanced) = advanced else { unreachable!() }; + + assert_eq!(advanced.offset, layout.offset); + assert_eq!(advanced.size_offset(), advanced_offset); + assert_eq!(advanced.elem_size, elem_size); + assert_eq!( + advanced.size_rounding_align_and_phase.align(), + layout.size_rounding_align_and_phase.align() + ); + #[allow(clippy::arithmetic_side_effects)] + let size_align_mask = advanced.size_rounding_align_and_phase.align().get() - 1; + assert_eq!(advanced.size_base & size_align_mask, layout.size_base & size_align_mask); + + let size_align = layout.size_rounding_align_and_phase.align(); + #[allow(clippy::arithmetic_side_effects)] + let size_prefix = layout.size_base & (size_align.get() - 1); + let expected_size = advanced_offset + .checked_add(trailing_size) + .and_then(|without_padding| { + without_padding.checked_add(util::padding_needed_for(without_padding, size_align)) + }) + .and_then(|rounded| size_prefix.checked_add(rounded)); + assert_eq!(slice_dst_size_for_trailing_bytes(advanced, trailing_size), expected_size); + } + + #[cfg(kani_slow)] + #[kani::proof] + fn prove_has_same_size_sequence() { + let left: TrailingSliceLayout = any_bounded_trailing_layout(); + let right: TrailingSliceLayout = any_bounded_trailing_layout(); + let trailing_size: usize = kani::any(); + + kani::assume(left.has_same_size_sequence(right)); + let left_align = left.size_rounding_align_and_phase.align().get(); + let right_align = right.size_rounding_align_and_phase.align().get(); + let max_align = if left_align > right_align { left_align } else { right_align }; + #[allow(clippy::arithmetic_side_effects)] + let max_align_mask = max_align - 1; + if left.elem_size & max_align_mask == 0 { + // In this branch `has_same_size_sequence` relies on every actual + // trailing byte count being a multiple of both alignments. Prove + // the formula for that entire byte class, which includes every + // representable `elems * elem_size`, without introducing a + // nonlinear symbolic multiplication into the harness. + kani::assume(trailing_size & max_align_mask == 0); + } + assert_eq!( + slice_dst_size_for_trailing_bytes(left, trailing_size), + slice_dst_size_for_trailing_bytes(right, trailing_size) + ); + } + + #[kani::proof] + #[kani::solver(kissat)] + fn prove_max_trailing_bytes() { + let trailing: TrailingSliceLayout = any_bounded_trailing_layout(); + let available_bytes: usize = kani::any(); + let trailing_bytes: usize = kani::any(); + let fits = slice_dst_size_for_trailing_bytes(trailing, trailing_bytes) + .map(|size| size <= available_bytes) + .unwrap_or(false); + match trailing.max_trailing_bytes(available_bytes) { + Some(max_bytes) => assert_eq!(trailing_bytes <= max_bytes, fits), + None => assert!(!fits), } } + #[kani::proof] + #[kani::solver(kissat)] + fn prove_max_trailing_bytes_components() { + let trailing: TrailingSliceLayout = any_bounded_trailing_layout(); + let bytes_len: usize = kani::any(); + let Some(max_slice_bytes) = trailing.max_trailing_bytes(bytes_len) else { + kani::assume(false); + loop {} + }; + let (size_align, size_phase) = trailing.size_rounding_align_and_phase.components(); + let Some(bytes_after_base) = bytes_len.checked_sub(trailing.size_base) else { + unreachable!() + }; + let max_rounded_bytes = + util::round_down_to_next_multiple_of_alignment(bytes_after_base, size_align); + assert_eq!(size_phase.checked_add(max_slice_bytes), Some(max_rounded_bytes)); + assert!(trailing.size_base.checked_add(max_rounded_bytes).unwrap() <= bytes_len); + } + + #[kani::proof] + #[kani::solver(kissat)] + fn prove_slice_dst_validator_selected_size_fits() { + let size_align = any_layout_align(); + #[allow(clippy::arithmetic_side_effects)] + let align_mask = size_align.get() - 1; + let raw_phase: usize = kani::any(); + let size_phase = raw_phase & align_mask; + let raw_capacity: usize = kani::any(); + let max_rounded_bytes = raw_capacity & !align_mask; + let size_base: usize = kani::any(); + let bytes_len: usize = kani::any(); + let trailing_size: usize = kani::any(); + + // These are exactly the relations established for every successful + // capacity query by `prove_max_trailing_bytes_components`. Isolate + // them from the helper's implementation to keep this arithmetic + // identity independent of how that helper computes its bound. + let Some(max_slice_bytes) = max_rounded_bytes.checked_sub(size_phase) else { + kani::assume(false); + loop {} + }; + let Some(max_object_size) = size_base.checked_add(max_rounded_bytes) else { + kani::assume(false); + loop {} + }; + kani::assume(max_object_size <= bytes_len); + kani::assume(trailing_size <= max_slice_bytes); + + // For a whole-element count selected by the validator, this unused + // byte count is also `max_slice_bytes % elem_size`. The identity here + // holds for every byte count satisfying the capacity bound. + #[allow(clippy::arithmetic_side_effects)] + let unused_bytes = max_slice_bytes - trailing_size; + let unused_aligned_bytes = + util::round_down_to_next_multiple_of_alignment(unused_bytes, size_align); + #[allow(clippy::arithmetic_side_effects)] + let object_size = size_base + (max_rounded_bytes - unused_aligned_bytes); + + let unrounded = size_phase.checked_add(trailing_size).unwrap(); + let rounded = + unrounded.checked_add(util::padding_needed_for(unrounded, size_align)).unwrap(); + assert_eq!(Some(object_size), size_base.checked_add(rounded)); + assert!(object_size <= bytes_len); + } + + #[kani::proof] + #[kani::solver(kissat)] + fn prove_slice_dst_validator_rejects_larger_trailing_size() { + let trailing: TrailingSliceLayout = any_bounded_trailing_layout(); + let bytes_len: usize = kani::any(); + let larger_trailing_size: usize = kani::any(); + let Some(max_slice_bytes) = trailing.max_trailing_bytes(bytes_len) else { + kani::assume(false); + loop {} + }; + kani::assume(larger_trailing_size > max_slice_bytes); + if let Some(object_size) = slice_dst_size_for_trailing_bytes(trailing, larger_trailing_size) + { + assert!(object_size > bytes_len); + } + } + + #[kani::proof] + fn prove_slice_formula_contains_trailing_slice() { + let trailing: TrailingSliceLayout = any_bounded_trailing_layout(); + let trailing_size: usize = kani::any(); + let (_, size_phase) = trailing.size_rounding_align_and_phase.components(); + + let Some(unrounded_min) = trailing.size_base.checked_add(size_phase) else { + kani::assume(false); + loop {} + }; + kani::assume(unrounded_min >= trailing.offset); + let Some(object_size) = slice_dst_size_for_trailing_bytes(trailing, trailing_size) else { + kani::assume(false); + loop {} + }; + + let Some(trailing_end) = trailing.offset.checked_add(trailing_size) else { unreachable!() }; + assert!(trailing_end <= object_size); + } + + #[kani::proof] + fn prove_finalized_slice_formula_is_aligned() { + let trailing: TrailingSliceLayout = any_bounded_trailing_layout(); + let outer_align = any_layout_align(); + let trailing_size: usize = kani::any(); + kani::assume(trailing.size_rounding_align_and_phase.align() >= outer_align); + #[allow(clippy::arithmetic_side_effects)] + let outer_align_mask = outer_align.get() - 1; + kani::assume(trailing.size_base & outer_align_mask == 0); + let Some(object_size) = slice_dst_size_for_trailing_bytes(trailing, trailing_size) else { + kani::assume(false); + loop {} + }; + + assert_eq!(object_size & outer_align_mask, 0); + } + + #[kani::proof] + fn prove_validator_checks_selected_endpoint_alignment() { + let align = any_layout_align(); + let addr: usize = kani::any(); + let bytes_len: usize = kani::any(); + let Some(end) = addr.checked_add(bytes_len) else { + kani::assume(false); + loop {} + }; + let suffix: bool = kani::any(); + let cast_type = if suffix { CastType::Suffix } else { CastType::Prefix }; + let selected_endpoint = if suffix { end } else { addr }; + #[allow(clippy::arithmetic_side_effects)] + let align_mask = align.get() - 1; + + // The endpoint check precedes and is independent of the size-info + // branch. A zero-sized layout makes every aligned case succeed and + // isolates that control flow, including prefix/suffix split selection. + let layout = DstLayout { + align, + size_info: SizeInfo::Sized { size: 0 }, + statically_shallow_unpadded: false, + }; + match layout.validate_cast_and_convert_metadata(addr, bytes_len, cast_type) { + Ok((elems, split_at)) => { + assert_eq!(selected_endpoint & align_mask, 0); + assert_eq!(elems, 0); + if suffix { + assert_eq!(split_at, bytes_len); + } else { + assert_eq!(split_at, 0); + } + } + Err(MetadataCastError::Alignment) => { + assert_ne!(selected_endpoint & align_mask, 0); + } + Err(MetadataCastError::Size) => unreachable!(), + } + } + + #[kani::proof] + fn prove_aligned_suffix_end_and_size_imply_aligned_start() { + let align = any_layout_align(); + #[allow(clippy::arithmetic_side_effects)] + let align_mask = align.get() - 1; + let addr: usize = kani::any(); + let bytes_len: usize = kani::any(); + let object_size: usize = kani::any(); + let Some(end) = addr.checked_add(bytes_len) else { + kani::assume(false); + loop {} + }; + kani::assume(end & align_mask == 0); + kani::assume(object_size & align_mask == 0); + kani::assume(object_size <= bytes_len); + + #[allow(clippy::arithmetic_side_effects)] + let split_at = bytes_len - object_size; + let Some(object_start) = addr.checked_add(split_at) else { unreachable!() }; + assert_eq!(object_start & align_mask, 0); + } + + #[cfg(kani_slow)] + #[kani::proof] + #[kani::solver(kissat)] + fn prove_size_formula_bounds_size_offset() { + // Prove the bound for arbitrary byte counts, including counts that + // are not whole elements. The `size_for_elems` contract relates this + // formula to checked element products. This harness checks the + // formula's bound independently of that implementation. + let trailing: TrailingSliceLayout = kani::any(); + let trailing_size: usize = kani::any(); + let Some(object_size) = slice_dst_size_for_trailing_bytes(trailing, trailing_size) else { + kani::assume(false); + loop {} + }; + let size_offset = trailing.size_offset(); + let Some(without_padding) = size_offset.checked_add(trailing_size) else { unreachable!() }; + + assert!(without_padding <= object_size); + } + + // Older supported compilers resolve the unchecked operation syntax in + // `add_scaled_metadata` to `polyfills::NumExt`; newer compilers prefer the + // inherent methods. Use UFCS to verify the fallback implementation which + // Kani's modern rustc cannot select in the production helper. + #[kani::proof] + #[kani::solver(kissat)] + fn prove_num_ext_affine_fallback_matches_checked_arithmetic() { + let base: usize = kani::any(); + let metadata: usize = kani::any(); + let multiple: usize = kani::any(); + let Some(scaled) = metadata.checked_mul(multiple) else { + kani::assume(false); + loop {} + }; + let Some(expected) = base.checked_add(scaled) else { + kani::assume(false); + loop {} + }; + + // SAFETY: The checked multiplication returned `Some`, satisfying + // `NumExt::unchecked_mul`'s no-overflow contract. + let scaled = + unsafe { ::unchecked_mul(metadata, multiple) }; + // SAFETY: The checked addition returned `Some`, and `scaled` is the + // checked product, satisfying `NumExt::unchecked_add`'s no-overflow + // contract. + let actual = + unsafe { ::unchecked_add(base, scaled) }; + assert_eq!(actual, expected); + } + + #[cfg(kani_slow)] #[kani::proof] fn prove_requires_dynamic_padding() { - let layout: DstLayout = kani::any(); + let layout: DstLayout = any_valid_dst_layout(); let SizeInfo::SliceDst(size_info) = layout.size_info else { kani::assume(false); @@ -2029,31 +4698,35 @@ fn prove_requires_dynamic_padding() { loop {} }; - let Some(unpadded_size) = size_info.offset.checked_add(trailing_slice_size) else { - // The `unpadded_size` exceeds `usize::MAX`; `meta`` is invalid. + let Some(trailing_slice_end) = size_info.offset.checked_add(trailing_slice_size) else { + // The trailing slice end exceeds `usize::MAX`; `meta` is invalid. kani::assume(false); loop {} }; - if unpadded_size >= DstLayout::MAX_SIZE { - // The `unpadded_size` exceeds `isize::MAX`; `meta` is invalid. + let Some(size) = slice_dst_size_for_trailing_bytes(size_info, trailing_slice_size) else { + kani::assume(false); + loop {} + }; + + if size > DstLayout::MAX_SIZE || trailing_slice_end > size { + // This metadata cannot describe a valid instance. kani::assume(false); loop {} } - let trailing_padding = util::padding_needed_for(unpadded_size, layout.align); - if !layout.requires_dynamic_padding() { - assert!(trailing_padding == 0); + assert_eq!(trailing_slice_end, size); } } + #[cfg(kani_slow)] #[kani::proof] fn prove_dst_layout_extend() { use crate::util::{max, min, padding_needed_for}; - let base: DstLayout = kani::any(); - let field: DstLayout = kani::any(); + let base: DstLayout = any_valid_dst_layout(); + let field: DstLayout = any_valid_dst_layout(); let packed: Option = kani::any(); if let Some(max_align) = packed { @@ -2069,9 +4742,6 @@ fn prove_dst_layout_extend() { unreachable!(); }; - // Under the above conditions, `DstLayout::extend` will not panic. - let composite = base.extend(field, packed); - // The field's alignment is clamped by `max_align` (i.e., the // `packed` attribute, if any) [1]. // @@ -2082,10 +4752,6 @@ fn prove_dst_layout_extend() { // alignment of the field's type. let field_align = min(field.align, packed.unwrap_or(DstLayout::THEORETICAL_MAX_ALIGN)); - // The struct's alignment is the maximum of its previous alignment and - // `field_align`. - assert_eq!(composite.align, max(base.align, field_align)); - // Compute the minimum amount of inter-field padding needed to // satisfy the field's alignment, and offset of the trailing field. // [1] @@ -2095,7 +4761,38 @@ fn prove_dst_layout_extend() { // Inter-field padding is guaranteed to be the minimum required in // order to satisfy each field's (possibly altered) alignment. let padding = padding_needed_for(base_size, field_align); - let offset = base_size + padding; + let offset = match base_size.checked_add(padding) { + Some(offset) => offset, + None => { + kani::assume(false); + loop {} + } + }; + + // Restrict the proof to extensions whose zero-metadata composite size + // is representable. This semantic condition implies the + // representability of each implementation intermediate below; prove + // those implications rather than assuming the intermediates. + match field.size_info { + SizeInfo::Sized { size } => kani::assume(offset.checked_add(size).is_some()), + SizeInfo::SliceDst(trailing) => { + let Some(field_min) = trailing.size_for_elems(0) else { + kani::assume(false); + loop {} + }; + kani::assume(offset.checked_add(field_min).is_some()); + + assert!(offset.checked_add(trailing.offset).is_some()); + assert!(offset.checked_add(trailing.size_base).is_some()); + } + } + + // Under the above conditions, `DstLayout::extend` will not panic. + let composite = base.extend(field, packed); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + assert_eq!(composite.align, max(base.align, field_align)); // For testing purposes, we'll also construct `alloc::Layout` // stand-ins for `DstLayout`, and show that `extend` behaves @@ -2132,29 +4829,64 @@ fn prove_dst_layout_extend() { panic!("The composite of two sized layouts must be sized.") } } - SizeInfo::SliceDst(TrailingSliceLayout { - offset: field_offset, - elem_size: field_elem_size, - }) => { - if let SizeInfo::SliceDst(TrailingSliceLayout { - offset: composite_offset, - elem_size: composite_elem_size, - }) = composite.size_info - { + SizeInfo::SliceDst(field_trailing) => { + if let SizeInfo::SliceDst(composite_trailing) = composite.size_info { // The offset of the trailing slice component is the sum // of the offset of the trailing field and the trailing // slice offset within that field. - assert_eq!(composite_offset, offset + field_offset); + assert_eq!(composite_trailing.offset, offset + field_trailing.offset); // The elem size is unchanged. - assert_eq!(composite_elem_size, field_elem_size); + assert_eq!(composite_trailing.elem_size, field_trailing.elem_size); + // Extension preserves the invariant that the physical + // trailing-slice field fits within the unrounded minimum + // size. This is one of the premises consumed by the + // validator soundness proof. + let composite_phase = + composite_trailing.size_rounding_align_and_phase.components().1; + let Some(composite_unrounded_min) = + composite_trailing.size_base.checked_add(composite_phase) + else { + unreachable!() + }; + assert!(composite_unrounded_min >= composite_trailing.offset); + // The normalized formula must continue to describe the + // size of the field at every trailing byte contribution + // for which the checked size computation succeeds. This + // is stronger than considering only products of a slice + // length and `elem_size`, and avoids a nonlinear + // multiplication in the verifier. + let trailing_size: usize = kani::any(); + let field_size = + match slice_dst_size_for_trailing_bytes(field_trailing, trailing_size) { + Some(size) => size, + None => { + kani::assume(false); + loop {} + } + }; + let expected_size = match offset.checked_add(field_size) { + Some(size) => size, + None => { + kani::assume(false); + loop {} + } + }; + assert_eq!( + slice_dst_size_for_trailing_bytes(composite_trailing, trailing_size), + Some(expected_size) + ); + + // `alloc::Layout` permits sizes which are not a multiple + // of alignment, so this represents the unpadded prefix of + // the DST through the start of its trailing slice. let field_analog = - Layout::from_size_align(field_offset, field_align.get()).unwrap(); + Layout::from_size_align(field_trailing.offset, field_align.get()).unwrap(); if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) { assert_eq!(actual_offset, offset); - assert_eq!(actual_composite.size(), composite_offset); + assert_eq!(actual_composite.size(), composite_trailing.offset); assert_eq!(actual_composite.align(), composite.align.get()); } else { // An error here reflects that composite of `base` @@ -2175,8 +4907,8 @@ fn prove_dst_layout_extend() { #[kani::proof] #[kani::should_panic] fn prove_dst_layout_extend_dst_panics() { - let base: DstLayout = kani::any(); - let field: DstLayout = kani::any(); + let base: DstLayout = any_valid_dst_layout(); + let field: DstLayout = any_valid_dst_layout(); let packed: Option = kani::any(); if let Some(max_align) = packed { @@ -2189,39 +4921,266 @@ fn prove_dst_layout_extend_dst_panics() { let _ = base.extend(field, packed); } - #[kani::proof] - fn prove_dst_layout_pad_to_align() { - use crate::util::padding_needed_for; - - let layout: DstLayout = kani::any(); - - let padded = layout.pad_to_align(); - + #[cfg(kani_slow)] + fn prove_dst_layout_pad_to_align_dst_invariants( + layout: DstLayout, + unpadded_trailing: TrailingSliceLayout, + padded: DstLayout, + ) { // Calling `pad_to_align` does not alter the `DstLayout`'s alignment. assert_eq!(padded.align, layout.align); + assert_eq!(padded.statically_shallow_unpadded, layout.statically_shallow_unpadded); - if let SizeInfo::Sized { size: unpadded_size } = layout.size_info { - if let SizeInfo::Sized { size: padded_size } = padded.size_info { - // If the layout is sized, it will remain sized after padding is - // added. Its sum will be its unpadded size and the size of the - // trailing padding needed to satisfy its alignment - // requirements. - let padding = padding_needed_for(unpadded_size, layout.align); - assert_eq!(padded_size, unpadded_size + padding); + let SizeInfo::SliceDst(padded_trailing) = padded.size_info else { + panic!("The padding of a DST layout must result in a DST layout.") + }; + assert_eq!(padded_trailing.offset, unpadded_trailing.offset); + assert_eq!(padded_trailing.elem_size, unpadded_trailing.elem_size); - // Prove that calling `DstLayout::pad_to_align` behaves - // identically to `Layout::pad_to_align`. - let layout_analog = - Layout::from_size_align(unpadded_size, layout.align.get()).unwrap(); - let padded_analog = layout_analog.pad_to_align(); - assert_eq!(padded_analog.align(), layout.align.get()); - assert_eq!(padded_analog.size(), padded_size); - } else { - panic!("The padding of a sized layout must result in a sized layout.") - } - } else { - // If the layout is a DST, padding cannot be statically added. - assert_eq!(padded.size_info, layout.size_info); - } + // A finalized DST formula is visibly aligned: the rounding alignment + // is at least the outer alignment, and the fixed base is a multiple of + // the outer alignment. This is the invariant which lets suffix + // validation infer alignment of the object's start from alignment of + // its end. + assert!(padded_trailing.size_rounding_align_and_phase.align() >= padded.align); + #[allow(clippy::arithmetic_side_effects)] + let padded_align_mask = padded.align.get() - 1; + assert_eq!(padded_trailing.size_base & padded_align_mask, 0); + assert_eq!(padded.pad_to_align(), padded); + let padded_phase = padded_trailing.size_rounding_align_and_phase.components().1; + let Some(padded_unrounded_min) = padded_trailing.size_base.checked_add(padded_phase) else { + unreachable!() + }; + assert!(padded_unrounded_min >= padded_trailing.offset); + } + + #[kani::proof] + fn prove_dst_layout_pad_to_align_sized() { + use crate::util::padding_needed_for; + + let layout: DstLayout = any_valid_dst_layout(); + let SizeInfo::Sized { size: unpadded_size } = layout.size_info else { + kani::assume(false); + loop {} + }; + + let padded = layout.pad_to_align(); + assert_eq!(padded.align, layout.align); + let SizeInfo::Sized { size: padded_size } = padded.size_info else { + panic!("The padding of a sized layout must result in a sized layout.") + }; + + // If the layout is sized, it will remain sized after padding is added. + // Its sum will be its unpadded size and the size of the trailing + // padding needed to satisfy its alignment requirements. + let padding = padding_needed_for(unpadded_size, layout.align); + assert_eq!(padded_size, unpadded_size + padding); + assert_eq!( + padded.statically_shallow_unpadded, + layout.statically_shallow_unpadded && padding == 0 + ); + + // Prove that calling `DstLayout::pad_to_align` behaves identically to + // `Layout::pad_to_align`. + let layout_analog = Layout::from_size_align(unpadded_size, layout.align.get()).unwrap(); + let padded_analog = layout_analog.pad_to_align(); + assert_eq!(padded_analog.align(), layout.align.get()); + assert_eq!(padded_analog.size(), padded_size); + } + + #[cfg(kani_slow)] + #[kani::proof] + fn prove_dst_layout_pad_to_align_dst_inner_rounding_transition() { + use crate::util::padding_needed_for; + + let layout: DstLayout = any_valid_dst_layout(); + let SizeInfo::SliceDst(trailing) = layout.size_info else { + kani::assume(false); + loop {} + }; + kani::assume(trailing.size_rounding_align_and_phase.align() >= layout.align); + + // `any_valid_dst_layout` requires the zero-metadata layout to be valid. + // Prove that this semantic condition bounds both the padded minimum + // and the implementation intermediate; do not assume the intermediate + // merely because the implementation uses it. + let Some(min_size) = trailing.size_for_elems(0) else { unreachable!() }; + let min_padding = padding_needed_for(min_size, layout.align); + let Some(padded_min_size) = min_size.checked_add(min_padding) else { unreachable!() }; + assert!(padded_min_size <= DstLayout::MAX_SIZE); + + let padding = padding_needed_for(trailing.size_base, layout.align); + let Some(expected_base) = trailing.size_base.checked_add(padding) else { unreachable!() }; + + let padded = layout.pad_to_align(); + let SizeInfo::SliceDst(padded_trailing) = padded.size_info else { unreachable!() }; + assert_eq!(padded_trailing.size_base, expected_base); + assert_eq!( + padded_trailing.size_rounding_align_and_phase, + trailing.size_rounding_align_and_phase + ); + prove_dst_layout_pad_to_align_dst_invariants(layout, trailing, padded); + } + + #[kani::proof] + #[kani::solver(kissat)] + fn prove_dst_layout_pad_to_align_dst_inner_rounding_formula() { + use crate::util::padding_needed_for; + + let size_align = any_layout_align(); + let outer_align = any_layout_align(); + kani::assume(size_align >= outer_align); + let size_base: usize = kani::any(); + let raw_size_phase: usize = kani::any(); + #[allow(clippy::arithmetic_side_effects)] + let size_phase = raw_size_phase & (size_align.get() - 1); + let trailing_size: usize = kani::any(); + + // Evaluate `size_base + round_up(size_phase + trailing_size, + // size_align)`, then round that complete size to `outer_align`. + // Compare with the equivalent expression obtained by rounding only + // `size_base` to `outer_align`, since `size_align` is a multiple of + // `outer_align`. Valid Rust objects have representable sizes; discard + // only trailing byte counts whose input or padded object size is not + // representable. + let Some(unrounded_inner) = size_phase.checked_add(trailing_size) else { + kani::assume(false); + loop {} + }; + let Some(rounded_inner) = + unrounded_inner.checked_add(padding_needed_for(unrounded_inner, size_align)) + else { + kani::assume(false); + loop {} + }; + let Some(unpadded_size) = size_base.checked_add(rounded_inner) else { + kani::assume(false); + loop {} + }; + let Some(expected_size) = + unpadded_size.checked_add(padding_needed_for(unpadded_size, outer_align)) + else { + kani::assume(false); + loop {} + }; + + let Some(padded_base) = size_base.checked_add(padding_needed_for(size_base, outer_align)) + else { + unreachable!() + }; + let Some(actual_size) = padded_base.checked_add(rounded_inner) else { unreachable!() }; + assert_eq!(actual_size, expected_size); + } + + #[cfg(kani_slow)] + #[kani::proof] + fn prove_dst_layout_pad_to_align_dst_outer_alignment_transition() { + use crate::util::padding_needed_for; + + let layout: DstLayout = any_valid_dst_layout(); + let SizeInfo::SliceDst(trailing) = layout.size_info else { + kani::assume(false); + loop {} + }; + let size_align = trailing.size_rounding_align_and_phase.align(); + kani::assume(size_align < layout.align); + + // `any_valid_dst_layout` requires the zero-metadata layout to be valid. + // Prove that this semantic condition bounds both the padded minimum + // and every implementation intermediate; do not assume the + // intermediates merely because the implementation uses them. + let Some(min_size) = trailing.size_for_elems(0) else { unreachable!() }; + let min_padding = padding_needed_for(min_size, layout.align); + let Some(padded_min_size) = min_size.checked_add(min_padding) else { unreachable!() }; + assert!(padded_min_size <= DstLayout::MAX_SIZE); + + let base_padding = padding_needed_for(trailing.size_base, size_align); + let Some(rounded_base) = trailing.size_base.checked_add(base_padding) else { + unreachable!() + }; + let Some(t) = + rounded_base.checked_add(trailing.size_rounding_align_and_phase.components().1) + else { + unreachable!() + }; + #[allow(clippy::arithmetic_side_effects)] + let expected_phase = t & (layout.align.get() - 1); + #[allow(clippy::arithmetic_side_effects)] + let expected_base = t - expected_phase; + + let padded = layout.pad_to_align(); + let SizeInfo::SliceDst(padded_trailing) = padded.size_info else { unreachable!() }; + assert_eq!(padded_trailing.size_base, expected_base); + assert_eq!( + padded_trailing.size_rounding_align_and_phase, + RoundingAlignAndPhase::new(layout.align, expected_phase) + ); + prove_dst_layout_pad_to_align_dst_invariants(layout, trailing, padded); + } + + #[kani::proof] + #[kani::solver(kissat)] + fn prove_dst_layout_pad_to_align_dst_outer_alignment_formula() { + use crate::util::padding_needed_for; + + let size_align = any_layout_align(); + let outer_align = any_layout_align(); + kani::assume(size_align < outer_align); + let size_base: usize = kani::any(); + let raw_size_phase: usize = kani::any(); + #[allow(clippy::arithmetic_side_effects)] + let size_phase = raw_size_phase & (size_align.get() - 1); + let trailing_size: usize = kani::any(); + + // Evaluate `size_base + round_up(size_phase + trailing_size, + // size_align)`, then round that complete size to `outer_align`. + let Some(unrounded_inner) = size_phase.checked_add(trailing_size) else { + kani::assume(false); + loop {} + }; + let Some(rounded_inner) = + unrounded_inner.checked_add(padding_needed_for(unrounded_inner, size_align)) + else { + kani::assume(false); + loop {} + }; + let Some(unpadded_size) = size_base.checked_add(rounded_inner) else { + kani::assume(false); + loop {} + }; + let Some(expected_size) = + unpadded_size.checked_add(padding_needed_for(unpadded_size, outer_align)) + else { + kani::assume(false); + loop {} + }; + + // Since `outer_align` is a larger multiple of `size_align`, absorb + // the inner rounding into the outer rounding and split the fixed + // contribution into a base and phase: + // + // fixed_contribution = round_up(size_base, size_align) + size_phase + // new_phase = fixed_contribution % outer_align + // new_base = fixed_contribution - new_phase + // new_base + round_up(new_phase + trailing_size, outer_align) + let Some(rounded_base) = size_base.checked_add(padding_needed_for(size_base, size_align)) + else { + unreachable!() + }; + let Some(t) = rounded_base.checked_add(size_phase) else { unreachable!() }; + #[allow(clippy::arithmetic_side_effects)] + let new_phase = t & (outer_align.get() - 1); + #[allow(clippy::arithmetic_side_effects)] + let new_base = t - new_phase; + let Some(new_unrounded_inner) = new_phase.checked_add(trailing_size) else { + unreachable!() + }; + let Some(new_rounded_inner) = + new_unrounded_inner.checked_add(padding_needed_for(new_unrounded_inner, outer_align)) + else { + unreachable!() + }; + let Some(actual_size) = new_base.checked_add(new_rounded_inner) else { unreachable!() }; + assert_eq!(actual_size, expected_size); } } diff --git a/rust/zerocopy/src/lib.rs b/rust/zerocopy/src/lib.rs index 572f0563fe2f..6dbeeb7488be 100644 --- a/rust/zerocopy/src/lib.rs +++ b/rust/zerocopy/src/lib.rs @@ -253,7 +253,7 @@ )] #![cfg_attr( __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, - deny(fuzzy_provenance_casts, lossy_provenance_casts) + deny(implicit_provenance_casts) )] #![deny( clippy::all, @@ -315,10 +315,6 @@ clippy::indexing_slicing, ))] #![cfg_attr(not(any(test, kani, feature = "std")), no_std)] -#![cfg_attr( - all(feature = "simd-nightly", target_arch = "arm"), - feature(stdarch_arm_neon_intrinsics) -)] #![cfg_attr( all(feature = "simd-nightly", any(target_arch = "powerpc", target_arch = "powerpc64")), feature(stdarch_powerpc) @@ -326,10 +322,6 @@ #![cfg_attr(feature = "float-nightly", feature(f16, f128))] #![cfg_attr(doc_cfg, feature(doc_cfg))] #![cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, feature(coverage_attribute))] -#![cfg_attr( - any(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, miri), - feature(layout_for_ptr) -)] #![cfg_attr(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), feature(test))] // This is a hack to allow zerocopy-derive derives to work in this crate. They @@ -401,7 +393,7 @@ wrappers::*, }; -#[cfg(any(feature = "alloc", test, kani))] +#[cfg(any(feature = "alloc", test))] extern crate alloc; #[cfg(any(feature = "alloc", test))] use alloc::{boxed::Box, vec::Vec}; @@ -411,7 +403,7 @@ // Used by `KnownLayout`. #[doc(hidden)] pub use crate::layout::*; -// Used by `TryFromBytes::is_bit_valid`. +// Used by `TryFromBytes::is_safe`. #[doc(hidden)] pub use crate::pointer::{invariant::BecauseImmutable, Maybe, Ptr}; // For each trait polyfill, as soon as the corresponding feature is stable, the @@ -786,10 +778,16 @@ fn only_derive_is_allowed_to_implement_this_trait() /// where `size == size_of::()` /// - If `Self` is a slice DST, then `LAYOUT.size_info == /// SizeInfo::SliceDst(slice_layout)` where: + /// - `slice_layout.size_rounding_align_and_phase` decodes to an alignment + /// that is a power of two and a phase which is less than that alignment /// - The size, `size`, of an instance of `Self` with `elems` trailing - /// slice elements is equal to `slice_layout.offset + - /// slice_layout.elem_size * elems` rounded up to the nearest multiple - /// of `LAYOUT.align` + /// slice elements is equal to `slice_layout.size_base + + /// round_up(size_phase + slice_layout.elem_size * elems, size_align)`, + /// where `(size_align, size_phase)` is decoded from + /// `slice_layout.size_rounding_align_and_phase` + /// - The trailing slice begins at byte offset `slice_layout.offset` + /// - `slice_layout.offset + slice_layout.elem_size * elems` does not + /// overflow and is no greater than `size` /// - For such an instance, any bytes in the range `[slice_layout.offset + /// slice_layout.elem_size * elems, size)` are padding and must not be /// assumed to be initialized @@ -1004,11 +1002,7 @@ fn to_elem_count(self) -> usize { #[inline] fn size_for_metadata(self, layout: DstLayout) -> Option { match layout.size_info { - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { - let slice_len = elem_size.checked_mul(self)?; - let without_padding = offset.checked_add(slice_len)?; - without_padding.checked_add(util::padding_needed_for(without_padding, layout.align)) - } + SizeInfo::SliceDst(trailing) => trailing.size_for_elems(self), // NOTE: This branch is unreachable, but we return `None` rather // than `unreachable!()` to avoid generating panic paths. SizeInfo::Sized { .. } => None, @@ -1073,29 +1067,85 @@ fn raw_from_ptr_len(data: NonNull, elems: usize) -> NonNull { #[inline(always)] fn pointer_to_metadata(ptr: *mut [T]) -> usize { - #[allow(clippy::as_conversions)] - let slc = ptr as *const [()]; + #[cfg(not(no_zerocopy_slice_ptr_len_1_79_0))] + { + ptr.len() + } - // SAFETY: - // - `()` has alignment 1, so `slc` is trivially aligned. - // - `slc` was derived from a non-null pointer. - // - The size is 0 regardless of the length, so it is sound to - // materialize a reference regardless of location. - // - By invariant, `self.ptr` has valid provenance. - let slc = unsafe { &*slc }; + #[cfg(no_zerocopy_slice_ptr_len_1_79_0)] + { + // `*mut [T]::len` was not stable before Rust 1.79. In every Rust + // version from our 1.56 MSRV through 1.78, `Hash for *mut T` + // decomposes a raw pointer and passes its address and metadata to + // the hasher in that order [1]. `Hash for usize` passes its value + // to `Hasher::write_usize` [2]. Capture those two values to obtain + // a candidate for the slice length without dereferencing `ptr` or + // constructing a reference. + // + // This historical `Hash` implementation is only used to produce a + // candidate. Before returning it, we reconstruct a raw slice and + // authenticate the candidate with `ptr::eq`, which compares slice + // lengths as well as addresses [3]. Thus, an unexpected `Hash` + // implementation cannot cause us to return incorrect metadata; it + // can only fail to produce an authenticated candidate. + // + // [1] Per https://doc.rust-lang.org/1.56.0/src/core/hash/mod.rs.html#776-782: + // + // let (address, metadata) = self.to_raw_parts(); + // state.write_usize(address as usize); + // metadata.hash(state); + // + // [2] Per https://doc.rust-lang.org/1.56.0/src/core/hash/mod.rs.html#628-656: + // + // fn hash(&self, state: &mut H) { + // state.$meth(*self) + // } + // ... + // (usize, write_usize), + // + // [3] Per https://doc.rust-lang.org/1.56.0/std/ptr/fn.eq.html: + // + // Slices are also compared by their length (fat pointers). + struct MetadataHasher { + values: [usize; 2], + writes: usize, + valid: bool, + } - // This is correct because the preceding `as` cast preserves the number - // of slice elements. [1] - // - // [1] Per https://doc.rust-lang.org/reference/expressions/operator-expr.html#pointer-to-pointer-cast: - // - // For slice types like `[T]` and `[U]`, the raw pointer types `*const - // [T]`, `*mut [T]`, `*const [U]`, and `*mut [U]` encode the number of - // elements in this slice. Casts between these raw pointer types - // preserve the number of elements. ... The same holds for `str` and - // any compound type whose unsized tail is a slice type, such as - // struct `Foo(i32, [u8])` or `(u64, Foo)`. - slc.len() + impl Hasher for MetadataHasher { + #[inline(always)] + fn finish(&self) -> u64 { + 0 + } + + #[inline(always)] + fn write(&mut self, _bytes: &[u8]) { + self.valid = false; + } + + #[inline(always)] + fn write_usize(&mut self, value: usize) { + match self.values.get_mut(self.writes) { + Some(slot) => *slot = value, + None => self.valid = false, + } + self.writes = self.writes.saturating_add(1); + } + } + + let mut hasher = MetadataHasher { values: [0; 2], writes: 0, valid: true }; + core::hash::Hash::hash(&ptr, &mut hasher); + assert!( + hasher.valid && hasher.writes == 2, + "unexpected raw-pointer Hash implementation" + ); + + let elems = hasher.values[1]; + #[allow(clippy::as_conversions)] + let reconstructed = ptr::slice_from_raw_parts_mut(ptr as *mut T, elems); + assert!(ptr::eq(ptr, reconstructed), "captured value is not raw-slice metadata"); + elems + } } } @@ -1182,11 +1232,29 @@ fn pointer_to_metadata(ptr: *mut [T]) -> usize { #[doc(hidden)] pub const REPR_C_UNION_VARIANT_ID: i128 = -3; +/// Marker types used to disambiguate implementations of projection traits that +/// would otherwise conflict. +#[doc(hidden)] +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub mod project_clients { + pub enum TryFromBytesDerive {} + + pub enum ProjectDerive {} +} + +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +#[doc(hidden)] +pub use zerocopy_derive::Project; + /// # Safety /// -/// `Self::ProjectToTag` must satisfy its safety invariant. +/// `>::ProjectToTag` must satisfy its safety invariant. +/// +/// The `Client` parameter exists solely to disambiguate between implementations +/// of `HasTag` that would otherwise conflict. #[doc(hidden)] -pub unsafe trait HasTag { +pub unsafe trait HasTag { fn only_derive_is_allowed_to_implement_this_trait() where Self: Sized; @@ -1198,28 +1266,35 @@ fn only_derive_is_allowed_to_implement_this_trait() /// /// # Safety /// - /// It must be the case that, for all `slf: Ptr<'_, Self, I>`, it is sound - /// to project from `slf` to `Ptr<'_, Self::Tag, I>` using this projection. + /// It must be the case that, for all `slf: Ptr<'_, Self, I>` where + /// `I::Aliasing` is `Shared`, it is sound to project it using this + /// projection to a `Ptr<'_, Self::Tag, (Shared, I::Alignment, + /// I::Validity)>`. type ProjectToTag: pointer::cast::Project; } /// Projects a given field from `Self`. /// -/// All implementations of `HasField` for a particular field `f` in `Self` -/// should use the same `Field` type; this ensures that `Field` is inferable -/// given an explicit `VARIANT_ID` and `FIELD_ID`. +/// All implementations of `HasField` for a particular `Client` and field `f` +/// in `Self` should use the same `Field` type; this ensures that `Field` is +/// inferable given an explicit `Client`, `VARIANT_ID`, and `FIELD_ID`. +/// +/// The `Client` parameter exists solely to disambiguate between implementations +/// of `HasField` that would otherwise conflict. /// /// # Safety /// /// A field `f` is `HasField` for `Self` if and only if: /// -/// - If `Self` has the layout of a struct or union type, then `VARIANT_ID` is -/// `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID` respectively; otherwise, if -/// `Self` has the layout of an enum type, `VARIANT_ID` is the numerical index -/// of the enum variant in which `f` appears. Note that `Self` does not need -/// to actually *be* such a type – it just needs to have the same layout as -/// such a type. For example, a `#[repr(transparent)]` wrapper around an enum -/// has the same layout as that enum. +/// - If `Self` has the layout of a struct type, `VARIANT_ID` is +/// `STRUCT_VARIANT_ID`. If `Self` has the layout of a `repr(C)` union type, +/// `VARIANT_ID` is `REPR_C_UNION_VARIANT_ID`; for other union layouts, it is +/// `UNION_VARIANT_ID`. Otherwise, if `Self` has the layout of an enum type and +/// `f` appears in a variant named `v`, `VARIANT_ID` is +/// `zerocopy::ident_id!(v)`. Note that `Self` does not need to actually *be* +/// such a type – it just needs to have the same layout as such a type. For +/// example, a `#[repr(transparent)]` wrapper around an enum has the same +/// layout as that enum. /// - If `f` has name `n`, `FIELD_ID` is `zerocopy::ident_id!(n)`; otherwise, /// if `f` is at index `i`, `FIELD_ID` is `zerocopy::ident_id!(i)`. /// - `Field` is a type with the same visibility as `f`. @@ -1232,8 +1307,8 @@ fn only_derive_is_allowed_to_implement_this_trait() /// /// The implementation of `project` must satisfy its safety post-condition. #[doc(hidden)] -pub unsafe trait HasField: - HasTag +pub unsafe trait HasField: + HasTag { fn only_derive_is_allowed_to_implement_this_trait() where @@ -1263,15 +1338,20 @@ fn only_derive_is_allowed_to_implement_this_trait() /// other words, it is a type-level function over invariants; `I` goes in, /// `Self::Invariants` comes out. /// +/// The `Client` parameter exists solely to disambiguate between implementations +/// of `ProjectField` (and their corresponding `HasField` implementations) that +/// would otherwise conflict. +/// /// # Safety /// -/// `T: ProjectField` if, for a -/// `ptr: Ptr<'_, T, I>` such that `T::is_projectable(ptr).is_ok()`, -/// `>::project(ptr.as_inner())` +/// `T: ProjectField` if, for a +/// `ptr: Ptr<'_, T, I>` such that `T::is_projectable` returns `Ok(())` +/// when passed the tag pointer projected from `ptr`, +/// `>::project(ptr.as_inner())` /// conforms to `T::Invariants`. #[doc(hidden)] -pub unsafe trait ProjectField: - HasField +pub unsafe trait ProjectField: + HasField where I: invariant::Invariants, { @@ -1290,29 +1370,35 @@ fn only_derive_is_allowed_to_implement_this_trait() /// Is the given field projectable from `ptr`? /// - /// If a field with [`Self::Invariants`] is projectable from the referent, - /// this function produces an `Ok(ptr)` from which the projection can be - /// made; otherwise `Err`. + /// If a field with [`Self::Invariants`] is projectable from the containing + /// value whose projected tag is `ptr`, this function produces `Ok(())`; + /// otherwise it produces `Err`. /// /// This method must be overriden if the field's projectability depends on /// the value of the bytes in `ptr`. #[inline(always)] - fn is_projectable<'a>(_ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + fn is_projectable<'a>( + _ptr: Ptr< + 'a, + >::Tag, + (invariant::Shared, I::Alignment, I::Validity), + >, + ) -> Result<(), Self::Error> { trait IsInfallible { const IS_INFALLIBLE: bool; } - struct Projection( - PhantomData<(Field, I, T)>, + struct Projection( + PhantomData<(Client, Field, I, T)>, ) where - T: ?Sized + HasField, + T: ?Sized + HasField, I: invariant::Invariants; - impl IsInfallible - for Projection + impl IsInfallible + for Projection where - T: ?Sized + HasField, + T: ?Sized + HasField, I: invariant::Invariants, { const IS_INFALLIBLE: bool = { @@ -1323,7 +1409,9 @@ impl IsInfallible // referent. This default implementation of `is_projectable` // is non-destructive, as it does not overwrite any part of // the referent. - crate::STRUCT_VARIANT_ID | crate::UNION_VARIANT_ID => true, + crate::STRUCT_VARIANT_ID + | crate::UNION_VARIANT_ID + | crate::REPR_C_UNION_VARIANT_ID => true, _enum_variant => { use crate::invariant::{Validity, ValidityKind}; match I::Validity::KIND { @@ -1336,9 +1424,9 @@ impl IsInfallible // type – the type itself is irrelevant. ValidityKind::Uninit | ValidityKind::Initialized => true, // The projectability of an enum field from an - // `AsInitialized` or `Valid` state is a dynamic + // `AsInitialized` or `Safe` state is a dynamic // property of its tag. - ValidityKind::AsInitialized | ValidityKind::Valid => false, + ValidityKind::AsInitialized | ValidityKind::Safe => false, } } }; @@ -1348,7 +1436,7 @@ impl IsInfallible } const_assert!( - as IsInfallible>::IS_INFALLIBLE + as IsInfallible>::IS_INFALLIBLE ); Ok(()) @@ -1614,6 +1702,64 @@ fn only_derive_is_allowed_to_implement_this_trait() /// } /// ``` /// +#[cfg_attr( + zerocopy_unstable_ptr, + doc = r#" +# Field invariants + +This experimental feature requires `--cfg zerocopy_unstable_ptr`. + +Named fields of structs, enum variants, and unions can specify additional +runtime checks using `#[zerocopy(invariant(expression))]`: + +``` +# use zerocopy_derive::TryFromBytes; +#[derive(TryFromBytes)] +struct Foo { + a: u8, + #[zerocopy(invariant((*a.read() % 2) == (*b.read() as u8)))] + b: bool, + #[zerocopy(invariant(*c.read() > 0))] + c: i16, +} +``` + +Each expression must return a `bool`. It has access to validated, read-only +[`Ptr`]s to the current field and all preceding fields of the struct or +variant, using their field names. A union's invariants have access only to +the current field. The expression can use the existing [`Ptr`] APIs to +inspect those fields. In this example, `read()` copies each field without +requiring alignment, and dereferencing the resulting [`ReadOnly`] accesses +the copied value. + +Rust's usual restrictions on local bindings apply; for example, a field name +cannot shadow an in-scope constant. + +Fields are checked in declaration order. Each field's bit validity is +checked before its invariants run. Multiple invariants on a field run in +attribute order. For structs and enums, validation stops at the first invalid +field or invariant that returns `false`. For unions, a failed bit-validity +check or invariant causes validation to try the next field; validation +succeeds as soon as one field and all its invariants pass. Each expression +runs in its own closure; `return` returns from that expression, and panics +propagate to the caller. Only the selected enum variant's fields and +invariants are checked. Expressions may have arbitrary side effects, even +when the conversion fails or its result is discarded. + +These predicates are additional acceptance checks beyond Rust's bit validity; +bit-valid bytes may still be rejected. See [What is a "valid instance"?] for +the distinction. + +[What is a "valid instance"?]: trait@TryFromBytes#what-is-a-valid-instance + +Invariants are not supported on tuple fields. Types with invariants cannot +derive [`FromZeros`] or [`FromBytes`], whose conversions do not perform runtime +validation. These checks apply to conversions through [`TryFromBytes`]; they +do not restrict ordinary construction or mutation of Rust values. + +"# +)] +/// /// # Portability /// /// To ensure consistent endianness for enums with multi-byte representations, @@ -1708,6 +1854,14 @@ fn only_derive_is_allowed_to_implement_this_trait() /// If you are negatively affected by lack of support for a particular type, /// we encourage you to let us know by [filing an issue][github-repo]. /// +/// In this trait's conversion methods, a "valid instance" must also pass any +/// configured field invariants, including those on nested fields. Rust bit +/// validity alone does not guarantee that a conversion succeeds: an invariant +/// may reject otherwise bit-valid bytes. Such predicates check acceptance at +/// conversion time; they do not constrain subsequent mutation or ordinary Rust +/// construction. See the [derive's field invariants][derive] documentation for +/// the experimental attribute's syntax, evaluation order, and side effects. +/// /// # `TryFromBytes` is not symmetrical with [`IntoBytes`] /// /// There are some types which implement both `TryFromBytes` and [`IntoBytes`], @@ -1754,25 +1908,25 @@ fn only_derive_is_allowed_to_implement_this_trait() /// /// # Safety /// - /// Unsafe code may assume that, if `is_bit_valid(candidate)` returns true, + /// Unsafe code may assume that, if `is_safe(candidate)` returns true, /// `*candidate` contains a valid `Self`. /// /// # Panics /// - /// `is_bit_valid` may panic. Callers are responsible for ensuring that any - /// `unsafe` code remains sound even in the face of `is_bit_valid` - /// panicking. (We support user-defined validation routines; so long as - /// these routines are not required to be `unsafe`, there is no way to - /// ensure that these do not generate panics.) + /// `is_safe` may panic. Callers are responsible for ensuring that any + /// `unsafe` code remains sound even in the face of `is_safe` panicking. (We + /// support user-defined validation routines; so long as these routines are + /// not required to be `unsafe`, there is no way to ensure that these do not + /// generate panics.) /// - /// Besides user-defined validation routines panicking, `is_bit_valid` will - /// either panic or fail to compile if called on a pointer with [`Shared`] - /// aliasing when `Self: !Immutable`. + /// Besides user-defined validation routines panicking, `is_safe` will either + /// panic or fail to compile if called on a pointer with [`Shared`] aliasing + /// when `Self: !Immutable`. /// /// [`UnsafeCell`]: core::cell::UnsafeCell /// [`Shared`]: invariant::Shared #[doc(hidden)] - fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool + fn is_safe(candidate: Maybe<'_, Self, A>) -> bool where A: invariant::Alignment; @@ -1872,7 +2026,7 @@ fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool @variant "dynamic_padding" ] )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_ref_from_bytes(source: &[u8]) -> Result<&Self, TryCastError<&[u8], Self>> @@ -1885,7 +2039,7 @@ fn try_ref_from_bytes(source: &[u8]) -> Result<&Self, TryCastError<&[u8], Self>> // This call may panic. If that happens, it doesn't cause any soundness // issues, as we have not generated any invalid state which we need to // fix before returning. - match source.try_into_valid() { + match source.try_into_safe() { Ok(valid) => Ok(valid.as_ref()), Err(e) => { Err(e.map_src(|src| src.as_bytes::().as_ref()).into()) @@ -1995,7 +2149,7 @@ fn try_ref_from_bytes(source: &[u8]) -> Result<&Self, TryCastError<&[u8], Self>> @variant "dynamic_padding" ] )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> @@ -2105,7 +2259,7 @@ fn try_ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), TryCastError<&[u @variant "dynamic_padding" ] )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> @@ -2199,7 +2353,7 @@ fn try_ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), TryCastError<&[u #[doc = codegen_header!("h5", "try_mut_from_bytes")] /// /// See [`TryFromBytes::try_ref_from_bytes`](#method.try_ref_from_bytes.codegen). - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_mut_from_bytes(bytes: &mut [u8]) -> Result<&mut Self, TryCastError<&mut [u8], Self>> @@ -2212,7 +2366,7 @@ fn try_mut_from_bytes(bytes: &mut [u8]) -> Result<&mut Self, TryCastError<&mut [ // This call may panic. If that happens, it doesn't cause any soundness // issues, as we have not generated any invalid state which we need to // fix before returning. - match source.try_into_valid() { + match source.try_into_safe() { Ok(source) => Ok(source.as_mut()), Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), } @@ -2308,7 +2462,7 @@ fn try_mut_from_bytes(bytes: &mut [u8]) -> Result<&mut Self, TryCastError<&mut [ #[doc = codegen_header!("h5", "try_mut_from_prefix")] /// /// See [`TryFromBytes::try_ref_from_prefix`](#method.try_ref_from_prefix.codegen). - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_mut_from_prefix( @@ -2408,7 +2562,7 @@ fn try_mut_from_prefix( #[doc = codegen_header!("h5", "try_mut_from_suffix")] /// /// See [`TryFromBytes::try_ref_from_suffix`](#method.try_ref_from_suffix.codegen). - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_mut_from_suffix( @@ -2513,7 +2667,7 @@ fn try_mut_from_suffix( @variant "dynamic_padding" ] )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_ref_from_bytes_with_elems( @@ -2529,7 +2683,7 @@ fn try_ref_from_bytes_with_elems( // This call may panic. If that happens, it doesn't cause any soundness // issues, as we have not generated any invalid state which we need to // fix before returning. - match source.try_into_valid() { + match source.try_into_safe() { Ok(source) => Ok(source.as_ref()), Err(e) => { Err(e.map_src(|src| src.as_bytes::().as_ref()).into()) @@ -2634,7 +2788,7 @@ fn try_ref_from_bytes_with_elems( @variant "dynamic_padding" ] )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_ref_from_prefix_with_elems( @@ -2742,7 +2896,7 @@ fn try_ref_from_prefix_with_elems( @variant "dynamic_padding" ] )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_ref_from_suffix_with_elems( @@ -2838,7 +2992,7 @@ fn try_ref_from_suffix_with_elems( #[doc = codegen_header!("h5", "try_mut_from_bytes_with_elems")] /// /// See [`TryFromBytes::try_ref_from_bytes_with_elems`](#method.try_ref_from_bytes_with_elems.codegen). - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_mut_from_bytes_with_elems( @@ -2854,7 +3008,7 @@ fn try_mut_from_bytes_with_elems( // This call may panic. If that happens, it doesn't cause any soundness // issues, as we have not generated any invalid state which we need to // fix before returning. - match source.try_into_valid() { + match source.try_into_safe() { Ok(source) => Ok(source.as_mut()), Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), } @@ -2949,7 +3103,7 @@ fn try_mut_from_bytes_with_elems( #[doc = codegen_header!("h5", "try_mut_from_prefix_with_elems")] /// /// See [`TryFromBytes::try_ref_from_prefix_with_elems`](#method.try_ref_from_prefix_with_elems.codegen). - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_mut_from_prefix_with_elems( @@ -3049,7 +3203,7 @@ fn try_mut_from_prefix_with_elems( #[doc = codegen_header!("h5", "try_mut_from_suffix_with_elems")] /// /// See [`TryFromBytes::try_ref_from_suffix_with_elems`](#method.try_ref_from_suffix_with_elems.codegen). - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_mut_from_suffix_with_elems( @@ -3116,24 +3270,14 @@ fn try_mut_from_suffix_with_elems( bench = "try_read_from_bytes", format = "coco_static_size", )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_read_from_bytes(source: &[u8]) -> Result> where Self: Sized, { - // FIXME(#2981): If `align_of::() == 1`, validate `source` in-place. - - let candidate = match CoreMaybeUninit::::read_from_bytes(source) { - Ok(candidate) => candidate, - Err(e) => { - return Err(TryReadError::Size(e.with_dst())); - } - }; - // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of - // its bytes are initialized. - unsafe { try_read_from(source, candidate) } + try_read_from(source) } /// Attempts to read a `Self` from the prefix of the given `source`. @@ -3194,24 +3338,24 @@ fn try_read_from_bytes(source: &[u8]) -> Result> bench = "try_read_from_prefix", format = "coco_static_size", )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), TryReadError<&[u8], Self>> where Self: Sized, { - // FIXME(#2981): If `align_of::() == 1`, validate `source` in-place. - - let (candidate, suffix) = match CoreMaybeUninit::::read_from_prefix(source) { - Ok(candidate) => candidate, - Err(e) => { - return Err(TryReadError::Size(e.with_dst())); - } + let (prefix, suffix) = match SplitAt::split_at(source, mem::size_of::()) { + Some(split) => split.via_immutable(), + None => return Err(SizeError::new(source).into()), }; - // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of - // its bytes are initialized. - unsafe { try_read_from(source, candidate).map(|slf| (slf, suffix)) } + match try_read_from(prefix) { + Ok(slf) => Ok((slf, suffix)), + Err(e) => Err(e.map_src( + #[inline(always)] + |_| source, + )), + } } /// Attempts to read a `Self` from the suffix of the given `source`. @@ -3273,44 +3417,72 @@ fn try_read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), TryReadError<&[u bench = "try_read_from_suffix", format = "coco_static_size", )] - #[must_use = "has no side effects"] + #[must_use = "the conversion result must be checked"] #[cfg_attr(zerocopy_inline_always, inline(always))] #[cfg_attr(not(zerocopy_inline_always), inline)] fn try_read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), TryReadError<&[u8], Self>> where Self: Sized, { - // FIXME(#2981): If `align_of::() == 1`, validate `source` in-place. - - let (prefix, candidate) = match CoreMaybeUninit::::read_from_suffix(source) { - Ok(candidate) => candidate, - Err(e) => { - return Err(TryReadError::Size(e.with_dst())); - } + let split_at = match source.len().checked_sub(mem::size_of::()) { + Some(split_at) => split_at, + None => return Err(SizeError::new(source).into()), }; - // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of - // its bytes are initialized. - unsafe { try_read_from(source, candidate).map(|slf| (prefix, slf)) } + // SAFETY: `checked_sub` returned the difference without overflow [1], + // so `split_at = source.len() - size_of::() <= source.len()`. + // This satisfies `SplitAt::split_at_unchecked`'s precondition. + // + // [1] Per https://doc.rust-lang.org/1.56.0/std/primitive.usize.html#method.checked_sub: + // + // Checked integer subtraction. Computes `self - rhs`, returning + // `None` if overflow occurred. + let (prefix, suffix) = + unsafe { SplitAt::split_at_unchecked(source, split_at) }.via_immutable(); + match try_read_from(suffix) { + Ok(slf) => Ok((prefix, slf)), + Err(e) => Err(e.map_src( + #[inline(always)] + |_| source, + )), + } } } +/// Validates and interprets the given affix of `source`'s bytes as a `&T`. +/// +/// Returns the destination reference and excess bytes on success. All errors, +/// including validity errors, contain the original `&S`. #[inline(always)] -fn try_ref_from_prefix_suffix( - source: &[u8], +fn try_ref_from_prefix_suffix( + source: &S, cast_type: CastType, meta: Option, -) -> Result<(&T, &[u8]), TryCastError<&[u8], T>> { - match Ptr::from_ref(source).try_cast_into::(cast_type, meta) { - Ok((source, prefix_suffix)) => { +) -> Result<(&T, &[u8]), TryCastError<&S, T>> +where + S: IntoBytes + Immutable + ?Sized, + T: TryFromBytes + KnownLayout + Immutable + ?Sized, +{ + match Ptr::from_ref(source.as_bytes()).try_cast_into::(cast_type, meta) { + Ok((candidate, prefix_suffix)) => { // This call may panic. If that happens, it doesn't cause any soundness // issues, as we have not generated any invalid state which we need to // fix before returning. - match source.try_into_valid() { + match candidate.try_into_safe() { Ok(valid) => Ok((valid.as_ref(), prefix_suffix.as_ref())), - Err(e) => Err(e.map_src(|src| src.as_bytes::().as_ref()).into()), + Err(e) => Err(e + .map_src( + #[inline(always)] + |_| source, + ) + .into()), } } - Err(e) => Err(e.map_src(Ptr::as_ref).into()), + Err(e) => Err(e + .map_src( + #[inline(always)] + |_| source, + ) + .into()), } } @@ -3325,7 +3497,7 @@ fn try_mut_from_prefix_suffix Ok((valid.as_mut(), prefix_suffix.as_mut())), Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), } @@ -3339,49 +3511,112 @@ fn swap((t, u): (T, U)) -> (U, T) { (u, t) } -/// # Safety -/// -/// All bytes of `candidate` must be initialized. #[inline(always)] -unsafe fn try_read_from( - source: S, - mut candidate: CoreMaybeUninit, -) -> Result> { +fn try_read_from(source: &S) -> Result> +where + S: IntoBytes + Immutable + ?Sized, + T: TryFromBytes, +{ + let bytes = source.as_bytes(); + + if bytes.len() != mem::size_of::() { + return Err(SizeError::new(source).into()); + } + + // FIXME(#2981): Avoid the validation copy when validation can safely use + // a pointer derived from `source`'s shared borrow. + + // Initialize the candidate in its final location. A typed move of a + // `MaybeUninit` may discard initialized bytes at `T`'s padding offsets + // [1], but validation requires every byte to be initialized. Do not move + // `candidate` between this copy and validation. + // + // [1] Per https://doc.rust-lang.org/1.93.1/std/mem/union.MaybeUninit.html#validity: + // + // Moving or copying a value of type `MaybeUninit` (i.e., performing a + // "typed copy") will exactly preserve the contents, including the + // provenance, of all non-padding bytes of type `T` in the value's + // representation. + let mut candidate = CoreMaybeUninit::::uninit(); + + // SAFETY: The earlier `if bytes.len() != mem::size_of::()` returns on a + // size mismatch, so reaching this copy implies that `bytes.len()` equals + // `mem::size_of::()`. Copying that many `u8`s cannot overrun `bytes`. + // `candidate.as_mut_ptr()` is writable for the same number of bytes because + // `MaybeUninit` has `T`'s size. Both pointers are non-null and aligned + // for `u8`, including when `T` is zero-sized. The fresh local allocation + // cannot overlap `source`, which is an argument. Thus the read, write, + // alignment, and non-overlap requirements of [2] hold. Copying `u8`s + // initializes every destination byte, including those at `T`'s padding + // offsets. + // + // These writes cannot violate `candidate`'s bit validity because every bit + // pattern is valid for `MaybeUninit` [3], even if it is invalid for `T`. + // + // [2] Per https://doc.rust-lang.org/1.56.0/std/ptr/fn.copy_nonoverlapping.html: + // + // Copies `count * size_of::()` bytes from `src` to `dst`. The source + // and destination must *not* overlap. + // + // [3] Per https://doc.rust-lang.org/1.56.0/std/mem/union.MaybeUninit.html#layout: + // + // ... any bit value is valid for a `MaybeUninit` ... + unsafe { + ptr::copy_nonoverlapping( + bytes.as_ptr(), + candidate.as_mut_ptr().cast::(), + mem::size_of::(), + ); + } + // We use `from_mut` despite not mutating via `c_ptr` so that we don't need // to add a `T: Immutable` bound. let c_ptr = Ptr::from_mut(&mut candidate); - // SAFETY: `c_ptr` has no uninitialized sub-ranges because it derived from - // `candidate`, which the caller promises is entirely initialized. Since - // `candidate` is a `MaybeUninit`, it has no validity requirements, and so - // no values written to an `Initialized` `c_ptr` can violate its validity. - // Since `c_ptr` has `Exclusive` aliasing, no mutations may happen except - // via `c_ptr` so long as it is live, so we don't need to worry about the - // fact that `c_ptr` may have more restricted validity than `candidate`. - let c_ptr = unsafe { c_ptr.assume_validity::() }; - let mut c_ptr = c_ptr.cast::<_, crate::pointer::cast::CastSized, _>(); - // Since we don't have `T: KnownLayout`, we hack around that by using - // `Wrapping`, which implements `KnownLayout` even if `T` doesn't. + // SAFETY: `c_ptr` has no uninitialized sub-ranges because it derived from + // `candidate`, whose bytes were all initialized by the copy above and which + // has not been moved since. Since `candidate` is a `MaybeUninit`, it has no + // validity requirements, and so no values written to an `Initialized` + // `c_ptr` can violate its validity. Since `c_ptr` has `Exclusive` aliasing, + // no mutations may happen except via `c_ptr` so long as it is live, so we + // don't need to worry about the fact that `c_ptr` may have more restricted + // validity than `candidate`. + let c_ptr = unsafe { c_ptr.assume_validity::() }; + + let c_ptr = c_ptr.cast::<_, crate::pointer::cast::CastSized, _>(); + + // SAFETY: `c_ptr` originated from a reference to `candidate`, so its + // address is aligned for `MaybeUninit`, which has `T`'s alignment [1]. + // `CastSized` preserves that address. `ReadOnly` is `repr(transparent)` + // with a single `T` field, so it also has `T`'s alignment [2], including + // when `T` is zero-sized. // + // [1] Per https://doc.rust-lang.org/1.56.0/std/mem/union.MaybeUninit.html#layout: + // + // `MaybeUninit` is guaranteed to have the same size, alignment, and + // ABI as `T`: + // + // [2] Per https://doc.rust-lang.org/1.93.1/reference/type-layout.html#the-transparent-representation: + // + // ... same layout and ABI as the only non-size 0 non-alignment 1 field, + // if present, or unit otherwise. + let mut c_ptr = unsafe { c_ptr.assume_alignment::() }; + // This call may panic. If that happens, it doesn't cause any soundness // issues, as we have not generated any invalid state which we need to fix // before returning. - if !Wrapping::::is_bit_valid(c_ptr.reborrow_shared().forget_aligned()) { + if !T::is_safe(c_ptr.reborrow_shared()) { return Err(ValidityError::new(source).into()); } - fn _assert_same_size_and_validity() - where - Wrapping: pointer::TransmuteFrom, - T: pointer::TransmuteFrom, invariant::Valid, invariant::Valid>, - { - } - - _assert_same_size_and_validity::(); - - // SAFETY: We just validated that `candidate` contains a valid - // `Wrapping`, which has the same size and bit validity as `T`, as - // guaranteed by the preceding type assertion. + // SAFETY: `T::is_safe` returned true for `candidate`'s initialized bytes, + // so it contains a valid `T`, as required by [1]. Validation used a shared + // `ReadOnly` pointer, and the bytes have not been modified since. + // + // [1] Per https://doc.rust-lang.org/1.56.0/std/mem/union.MaybeUninit.html#method.assume_init: + // + // It is up to the caller to guarantee that the `MaybeUninit` really is + // in an initialized state. Ok(unsafe { candidate.assume_init() }) } @@ -5464,22 +5699,32 @@ fn read_from(source: &[u8]) -> Option } } -/// Interprets the given affix of the given bytes as a `&Self`. +/// Interprets the given affix of `source`'s bytes as a `&T`. /// -/// This method computes the largest possible size of `Self` that can fit in the -/// prefix or suffix bytes of `source`, then attempts to return both a reference -/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. -/// If there are insufficient bytes, or if that affix of `source` is not -/// appropriately aligned, this returns `Err`. +/// This method uses `meta` if provided; otherwise, it computes the largest +/// possible size of `T` that can fit in the prefix or suffix bytes of `source`. +/// It returns both a reference to those bytes interpreted as a `T`, and a +/// reference to the excess bytes. If there are insufficient bytes, or if that +/// affix of `source` is not appropriately aligned, this returns `Err` containing +/// the original `&S`. #[inline(always)] -fn ref_from_prefix_suffix( - source: &[u8], +fn ref_from_prefix_suffix( + source: &S, meta: Option, cast_type: CastType, -) -> Result<(&T, &[u8]), CastError<&[u8], T>> { - let (slf, prefix_suffix) = Ptr::from_ref(source) +) -> Result<(&T, &[u8]), CastError<&S, T>> +where + S: IntoBytes + Immutable + ?Sized, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + let (slf, prefix_suffix) = Ptr::from_ref(source.as_bytes()) .try_cast_into::<_, BecauseImmutable>(cast_type, meta) - .map_err(|err| err.map_src(|s| s.as_ref()))?; + .map_err(|err| { + err.map_src( + #[inline(always)] + |_| source, + ) + })?; Ok((slf.recall_validity().as_ref(), prefix_suffix.as_ref())) } @@ -5602,6 +5847,10 @@ fn mut_from_prefix_suffix( /// if its field is [`IntoBytes`]; else, /// - if the type has no generic parameters, it is [`IntoBytes`] if the type /// is sized and has no padding bytes; else, +/// - if the type is `repr(C)` without an `align(N)` modifier for `N > 1` +/// (it may have `align(1)` or `packed(N)`), and every field is `T`, +/// `[T; N]`, or a final `[T]` for the same type parameter `T`, it is +/// [`IntoBytes`]; else, /// - if the type is `repr(C)`, its fields must be [`Unaligned`]. /// - If the type is an enum: /// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, @@ -6493,6 +6742,39 @@ fn from_mut_slice(slc: &mut [u8]) -> &mut Unsized { } } + // These types exercise a nested slice DST whose inner trailing-padding + // operation cannot be flattened into the alignment of the packed outer + // type. For `NestedPackedDstOuter<[u8]>`, the trailing slice begins at + // offset 7, but the size for `elems` elements is + // `2 + round_up(5 + elems, 4)`. + // + // The Reference rejects directly placing an alignment-4 field in a + // `repr(packed(2))` type. Rust currently accepts this construction through + // generic indirection. So long as Rust accepts it, this test deliberately + // treats its layout as following the otherwise-applicable `repr(C)` and + // `repr(packed)` rules. + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C, align(4))] + struct NestedPackedDstWord([u8; 4]); + + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C)] + struct NestedPackedDstInner { + a: NestedPackedDstWord, + b: u8, + tail: T, + } + + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C, packed(2))] + struct NestedPackedDstOuter { + z: u8, + inner: ManuallyDrop>, + } + + #[repr(C, align(2))] + struct Align2(T); + #[test] fn test_known_layout() { // Test that `$ty` and `ManuallyDrop<$ty>` have the expected layout. @@ -6507,17 +6789,25 @@ macro_rules! test { }; } - let layout = - |offset, align, trailing_slice_elem_size, statically_shallow_unpadded| DstLayout { - align: NonZeroUsize::new(align).unwrap(), + let layout = |offset, align, trailing_slice_elem_size, statically_shallow_unpadded| { + let align = NonZeroUsize::new(align).unwrap(); + let size_phase = offset % align; + DstLayout { + align, size_info: match trailing_slice_elem_size { None => SizeInfo::Sized { size: offset }, - Some(elem_size) => { - SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) - } + Some(elem_size) => SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base: offset - size_phase, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + align, size_phase, + ), + }), }, statically_shallow_unpadded, - }; + } + }; test!((), layout(0, 1, None, false)); test!(u8, layout(1, 1, None, false)); @@ -6533,6 +6823,153 @@ macro_rules! test { test!(str, layout(0, 1, Some(1), true)); } + #[test] + fn test_known_layout_pointer_to_metadata() { + fn test(data: *mut T, elems: usize) { + let ptr = ptr::slice_from_raw_parts_mut(data, elems); + assert_eq!(<[T] as KnownLayout>::pointer_to_metadata(ptr), elems); + } + + for elems in [0, 1, usize::MAX] { + test(ptr::null_mut::(), elems); + } + + test(NonNull::::dangling().as_ptr(), 0); + + let mut bytes = [1, 2, 3]; + test(bytes.as_mut_ptr(), bytes.len()); + assert_eq!(bytes, [1, 2, 3]); + + test(NonNull::<()>::dangling().as_ptr(), usize::MAX); + + // Retaining and inspecting a raw pointer after freeing its allocation + // is safe so long as the pointer is not dereferenced. + let deallocated = { + let mut byte = Box::new(0u8); + let ptr: *mut u8 = &mut *byte; + drop(byte); + ptr + }; + test(deallocated, 3); + + // Metadata extraction requires neither element storage nor alignment + // for `T`; it does not access the pointer's referent. + test(NonNull::::dangling().as_ptr().cast::(), usize::MAX); + } + + #[cfg(feature = "derive")] + #[test] + fn test_known_layout_pointer_to_metadata_derive() { + #[derive(KnownLayout)] + #[repr(C)] + struct Dst { + prefix: u8, + trailing: [u8], + } + + for elems in [0, 1, usize::MAX] { + let ptr = ptr::slice_from_raw_parts_mut(ptr::null_mut::(), elems); + #[allow(clippy::as_conversions)] + let ptr = ptr as *mut Dst; + assert_eq!(Dst::pointer_to_metadata(ptr), elems); + } + } + + #[test] + fn test_nested_packed_dst_layout() { + fn assert_native_size(expected: usize) { + let value = NestedPackedDstOuter { + z: 0, + inner: ManuallyDrop::new(NestedPackedDstInner { + a: NestedPackedDstWord([0; 4]), + b: 0, + tail: [0; N], + }), + }; + let value: &NestedPackedDstOuter<[u8]> = &value; + assert_eq!(mem::size_of_val(value), expected); + assert_eq!(NestedPackedDstOuter::<[u8]>::size_for_metadata(N), Some(expected)); + } + + assert_native_size::<0>(10); + assert_native_size::<1>(10); + assert_native_size::<2>(10); + assert_native_size::<3>(10); + assert_native_size::<4>(14); + assert_native_size::<7>(14); + assert_native_size::<8>(18); + + let layout = NestedPackedDstOuter::<[u8]>::LAYOUT; + assert_eq!(layout.align.get(), 2); + assert!(layout.requires_dynamic_padding()); + assert_eq!( + layout.size_info, + SizeInfo::SliceDst(TrailingSliceLayout { + offset: 7, + elem_size: 1, + size_base: 6, + size_rounding_align_and_phase: RoundingAlignAndPhase::new( + NonZeroUsize::new(4).unwrap(), + 1 + ), + }) + ); + + // Eight bytes used to be incorrectly accepted for a zero-element + // value. The actual Rust layout requires ten bytes. + let short = Align2([0u8; 8]); + assert!(NestedPackedDstOuter::<[u8]>::ref_from_bytes_with_elems(&short.0, 0).is_err()); + assert!(pointer::Ptr::from_ref(&short.0[..]) + .try_cast_into::, pointer::BecauseImmutable>( + CastType::Prefix, + Some(0), + ) + .is_err()); + + let full = Align2([0, 0, 0, 0, 0, 0, 0, 0, b'S', b'!']); + let value = NestedPackedDstOuter::<[u8]>::ref_from_bytes_with_elems(&full.0, 0).unwrap(); + assert_eq!(mem::size_of_val(value), 10); + + // When metadata is inferred, lengths 0 through 3 all produce a + // ten-byte object. Parsing chooses the largest trailing-slice length + // which fits exactly. + let inferred = NestedPackedDstOuter::<[u8]>::ref_from_bytes(&full.0).unwrap(); + assert_eq!(pointer::Ptr::from_ref(inferred).len(), 3); + + // Trailing padding includes the inner type's padding, even when the + // outer type's alignment is reduced by packing. + let trailing_layout = trailing_slice_layout::>(); + for (elems, expected) in [(0, 3), (1, 2), (2, 1), (3, 0), (4, 3)] { + assert_eq!(trailing_layout.padding_for_elems(elems), expected); + } + } + + // Before Rust 1.76, rust-lang/rust#118537 caused `addr_of_mut!` to + // miscompute the offset of an unsized field in a packed struct. The fix is + // rust-lang/rust#118540. Keep the layout and parsing regression above on + // all supported compilers, and gate only the projection regression which + // necessarily exercises that independent compiler bug. + #[rustversion::since(1.76.0)] + #[test] + fn test_nested_packed_dst_projection() { + let full = Align2([0, 0, 0, 0, 0, 0, 0, 0, b'S', b'!']); + + // The projected inner value occupies the final eight bytes of the full + // ten-byte input, including the two sentinel bytes. + let (value, rest) = pointer::Ptr::from_ref(&full.0[..]) + .try_cast_into::, pointer::BecauseImmutable>( + CastType::Prefix, + Some(0), + ) + .unwrap(); + assert!(rest.as_ref().is_empty()); + let inner = value.project::().unwrap(); + assert_eq!( + inner.as_bytes::().as_ref(), + &[0, 0, 0, 0, 0, 0, b'S', b'!'] + ); + } + #[cfg(feature = "derive")] #[test] fn test_known_layout_derive() { @@ -6588,10 +7025,19 @@ fn _assert_kl(_: &T) {} statically_shallow_unpadded: false, }; - let unsized_layout = |align, elem_size, offset, statically_shallow_unpadded| DstLayout { - align: NonZeroUsize::new(align).unwrap(), - size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), - statically_shallow_unpadded, + let unsized_layout = |align, elem_size, offset, statically_shallow_unpadded| { + let align = NonZeroUsize::new(align).unwrap(); + let size_phase = offset % align; + DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset, + elem_size, + size_base: offset - size_phase, + size_rounding_align_and_phase: RoundingAlignAndPhase::new(align, size_phase), + }), + statically_shallow_unpadded, + } }; // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | @@ -7152,6 +7598,112 @@ fn test_try_from_bytes_try_read_from() { ); } + #[test] + fn test_try_read_from_typed_source() { + #[derive(IntoBytes, Immutable)] + #[repr(transparent)] + struct Source(T); + + fn check(source: &S, valid: bool) { + match try_read_from::<_, [bool; 4]>(source) { + Ok(value) => { + assert!(valid); + assert_eq!(value, [false, true, false, true]); + } + Err(error) => { + assert!(!valid); + assert!(matches!(error, TryReadError::Validity(_))); + assert!(ptr::eq(error.into_src(), source)); + } + } + + let error = try_read_from::<_, [bool; 5]>(source).unwrap_err(); + assert!(matches!(error, TryReadError::Size(_))); + assert!(ptr::eq(error.into_src(), source)); + } + + for (value, valid) in [(u16::from_ne_bytes([0, 1]), true), (0x0202, false)] { + let source = Source([value; 2]); + check(&source, valid); + let source: &Source<[u16]> = &source; + check(source, valid); + } + + let source = Source([(); 2]); + assert!(try_read_from::<_, ()>(&source).is_ok()); + let source: &Source<[()]> = &source; + assert!(try_read_from::<_, ()>(source).is_ok()); + } + + #[test] + fn test_try_read_initialized_padding() { + #[derive(KnownLayout, Immutable, Debug, PartialEq, Eq)] + #[repr(C, align(8))] + struct Padded(u8); + + // SAFETY: `Padded` has only a `u8` field and padding, so all + // initialized byte sequences are valid. Per + // https://doc.rust-lang.org/1.93.1/reference/behavior-considered-undefined.html#invalid-values: + // + // An integer (`i*`/`u*`), floating point value (`f*`), or raw pointer + // must be initialized, i.e., must not be obtained from uninitialized + // memory. + // + // A `struct`, tuple, and array requires all fields/elements to be + // valid at their respective type. + const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for Padded; |candidate| { + // Observe every byte while validation is running, including + // padding that a typed move of `MaybeUninit` could + // discard. The returned `Padded` need not retain its padding. + assert_eq!(candidate.as_bytes::().as_ref(), &[0xA5; 8]); + true + }) + }; + + let source = [0xA5; 8]; + assert_eq!(Padded::try_read_from_bytes(&source), Ok(Padded(0xA5))); + + let prefix_source = [0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0]; + let (value, suffix) = Padded::try_read_from_prefix(&prefix_source).unwrap(); + assert_eq!(value, Padded(0xA5)); + assert!(ptr::eq(suffix, &prefix_source[8..])); + + let suffix_source = [0, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5, 0xA5]; + let (prefix, value) = Padded::try_read_from_suffix(&suffix_source).unwrap(); + assert_eq!(value, Padded(0xA5)); + assert!(ptr::eq(prefix, &suffix_source[..1])); + } + + #[test] + fn test_try_read_error_sources_and_zero_sized() { + let invalid = [2, 3]; + for (error, source) in [ + (bool::try_read_from_bytes(&invalid[..1]).unwrap_err(), &invalid[..1]), + (bool::try_read_from_prefix(&invalid).unwrap_err(), &invalid[..]), + (bool::try_read_from_suffix(&invalid).unwrap_err(), &invalid[..]), + ] { + assert!(matches!(error, TryReadError::Validity(_))); + assert!(ptr::eq(error.into_src(), source)); + } + for source in [&[][..], &invalid[..]] { + let error = bool::try_read_from_bytes(source).unwrap_err(); + assert!(matches!(error, TryReadError::Size(_))); + assert!(ptr::eq(error.into_src(), source)); + } + for error in [ + bool::try_read_from_prefix(&invalid[..0]).unwrap_err(), + bool::try_read_from_suffix(&invalid[..0]).unwrap_err(), + ] { + assert!(matches!(error, TryReadError::Size(_))); + assert!(ptr::eq(error.into_src(), &invalid[..0])); + } + + assert_eq!(<()>::try_read_from_bytes(&[]), Ok(())); + assert_eq!(<()>::try_read_from_prefix(&invalid), Ok(((), &invalid[..]))); + assert_eq!(<()>::try_read_from_suffix(&invalid), Ok((&invalid[..], ()))); + } + #[test] fn test_ref_from_mut_from_bytes() { // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` @@ -7502,6 +8054,16 @@ fn test_new_box_zeroed_with_elems() { assert_eq!(&*s, &[0, 3, 0]); } + #[test] + fn test_new_box_zeroed_with_elems_nested_packed_dst() { + let mut value = NestedPackedDstOuter::<[u8]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(mem::size_of_val(&*value), 10); + + // Exercise the whole-object write that exposed the former + // under-allocation under Miri and AddressSanitizer. + value.zero(); + } + #[test] fn test_new_box_zeroed_with_elems_empty() { let s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(0).unwrap(); @@ -7571,6 +8133,133 @@ fn test_deprecated_from_bytes() { assert_eq!(rest.len(), 4); } + #[test] + fn test_ref_from_prefix_suffix_typed_source() { + #[derive(IntoBytes, Immutable)] + #[repr(transparent)] + struct Source(T); + + fn check(source: &S) { + let bytes = source.as_bytes(); + assert_eq!(bytes.len(), 4); + + for cast_type in [CastType::Prefix, CastType::Suffix] { + let (expected, expected_rest) = match cast_type { + CastType::Prefix => (&bytes[..3], &bytes[3..]), + CastType::Suffix => (&bytes[1..], &bytes[..1]), + }; + for meta in [None, Some(1)] { + let (value, rest) = + ref_from_prefix_suffix::<_, [[u8; 3]]>(source, meta, cast_type).unwrap(); + assert_eq!(value, &[expected]); + assert!(ptr::eq(value.as_bytes(), expected)); + assert!(ptr::eq(rest, expected_rest)); + } + + let err = + ref_from_prefix_suffix::<_, [u8; 5]>(source, None, cast_type).unwrap_err(); + assert!(matches!(err, CastError::Size(_))); + assert!(ptr::eq(err.into_src(), source)); + + let err = + ref_from_prefix_suffix::<_, [u8]>(source, Some(5), cast_type).unwrap_err(); + assert!(matches!(err, CastError::Size(_))); + assert!(ptr::eq(err.into_src(), source)); + } + } + + check(&Source([0x0102u16, 0x0304])); + let source = Source([false, true, false, true]); + let source: &Source<[bool]> = &source; + check(source); + } + + #[test] + fn test_ref_from_prefix_suffix_typed_source_alignment_error() { + let storage = Align::<[bool; 9], AU64>::new([false; 9]); + for cast_type in [CastType::Prefix, CastType::Suffix] { + let source = match cast_type { + CastType::Prefix => &storage.t[1..], + CastType::Suffix => &storage.t[..], + }; + let err = ref_from_prefix_suffix::<_, AU64>(source, None, cast_type).unwrap_err(); + assert!(matches!(err, CastError::Alignment(_))); + let recovered: &[bool] = err.into_src(); + assert!(ptr::eq(recovered, source)); + + let err = try_ref_from_prefix_suffix::<_, AU64>(source, cast_type, None).unwrap_err(); + assert!(matches!(err, TryCastError::Alignment(_))); + let recovered: &[bool] = err.into_src(); + assert!(ptr::eq(recovered, source)); + } + } + + #[test] + fn test_try_ref_from_prefix_suffix_typed_source() { + #[derive(IntoBytes, Immutable)] + #[repr(transparent)] + struct Source(T); + + fn check(source: &S) { + let bytes = source.as_bytes(); + assert_eq!(bytes, [0, 1, 0, 1]); + + for cast_type in [CastType::Prefix, CastType::Suffix] { + let (expected, expected_rest) = match cast_type { + CastType::Prefix => (&bytes[..3], &bytes[3..]), + CastType::Suffix => (&bytes[1..], &bytes[..1]), + }; + for meta in [None, Some(1)] { + let (value, rest) = + try_ref_from_prefix_suffix::<_, [[bool; 3]]>(source, cast_type, meta) + .unwrap(); + assert_eq!(value.as_bytes(), expected); + assert!(ptr::eq(value.as_bytes(), expected)); + assert!(ptr::eq(rest, expected_rest)); + } + + let err = try_ref_from_prefix_suffix::<_, [bool; 5]>(source, cast_type, None) + .unwrap_err(); + assert!(matches!(err, TryCastError::Size(_))); + assert!(ptr::eq(err.into_src(), source)); + + let err = try_ref_from_prefix_suffix::<_, [bool]>(source, cast_type, Some(5)) + .unwrap_err(); + assert!(matches!(err, TryCastError::Size(_))); + assert!(ptr::eq(err.into_src(), source)); + } + } + + check(&Source([u16::from_ne_bytes([0, 1]); 2])); + let source = Source([false, true, false, true]); + let source: &Source<[bool]> = &source; + check(source); + } + + #[test] + fn test_try_ref_from_prefix_suffix_typed_source_validity_error() { + fn check(source: &S) { + for cast_type in [CastType::Prefix, CastType::Suffix] { + let err = + try_ref_from_prefix_suffix::<_, bool>(source, cast_type, None).unwrap_err(); + assert!(matches!(err, TryCastError::Validity(_))); + assert!(ptr::eq(err.into_src(), source)); + + for meta in [None, Some(1)] { + let err = try_ref_from_prefix_suffix::<_, [bool]>(source, cast_type, meta) + .unwrap_err(); + assert!(matches!(err, TryCastError::Validity(_))); + assert!(ptr::eq(err.into_src(), source)); + } + } + } + + let source = [u16::from_ne_bytes([2, 2]); 2]; + check(&source); + check(&source[..]); + check(source.as_bytes()); + } + #[test] fn test_try_ref_from_prefix_suffix() { use crate::util::testutil::Align; diff --git a/rust/zerocopy/src/macros.rs b/rust/zerocopy/src/macros.rs index ec67c03a44fc..73f6dd0ca0ae 100644 --- a/rust/zerocopy/src/macros.rs +++ b/rust/zerocopy/src/macros.rs @@ -1033,7 +1033,7 @@ unsafe impl $(<$($tyvar),*>)? $crate::TryFromBytes for $name$(<$($tyvar),*>)? )? { #[inline(always)] - fn is_bit_valid(_: $crate::Maybe<'_, Self, A>) -> bool + fn is_safe(_: $crate::Maybe<'_, Self, A>) -> bool where A: $crate::invariant::Alignment, { @@ -1179,7 +1179,7 @@ unsafe impl $crate::TryFromBytes for $name )* { #[inline(always)] - fn is_bit_valid(_: $crate::Maybe<'_, Self, A>) -> bool + fn is_safe(_: $crate::Maybe<'_, Self, A>) -> bool where A: $crate::invariant::Alignment, { @@ -1411,6 +1411,38 @@ const fn transmute_ref(t: &T) -> &U let x: &SliceDst = transmute_ref!(slice_dst_big); assert_eq!(x, slice_dst_small); + // A packed outer DST can preserve the trailing field's stronger + // rounding alignment. These two types consequently have different + // normalized layout formulas even though both have size `2 + 4 * n` + // for every trailing slice length. Ensure the semantic size-sequence + // comparison accepts this representation-preserving transmutation. + #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + #[repr(C, align(2))] + struct TransmuteAlign2([u8; 4]); + + #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + #[repr(C, packed(2))] + struct TransmutePacked4 { + head: u16, + tail: [u32], + } + + #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + #[repr(C, packed(2))] + struct TransmutePacked2 { + head: u16, + tail: [TransmuteAlign2], + } + + #[repr(C, align(2))] + struct TransmuteBytes([u8; 10]); + + let bytes = TransmuteBytes([0; 10]); + let src = TransmutePacked4::ref_from_bytes(&bytes.0).unwrap(); + let dst: &TransmutePacked2 = transmute_ref!(src); + assert_eq!(mem::size_of_val(src), 10); + assert_eq!(mem::size_of_val(dst), 10); + // Test that it's legal to transmute a reference while shrinking the // lifetime (note that `X` has the lifetime `'static`). let x: &[u8; 8] = transmute_ref!(X); @@ -1471,6 +1503,19 @@ fn drop(&mut self) { // its `PanicOnDrop` temporary, which would cause a panic. assert_eq!(y.as_ref().map_err(|p| &p.src.0), Err::<&bool, _>(&2u8)); mem::forget(y); + + #[derive(Debug, TryFromBytes)] + #[repr(C, align(8))] + struct PaddedBool(bool); + + // A failed transmutation must return every source byte, including + // bytes which would be padding in the destination type. + let src = [2u8, 1, 2, 3, 4, 5, 6, 7]; + let x: Result = try_transmute!(src); + assert_eq!(x.unwrap_err().into_src(), src); + + let x: Result = try_transmute!([0u8; 8]); + assert!(!x.unwrap().0); } #[test] @@ -1687,9 +1732,9 @@ fn test_macros_evaluate_args_once() { #[test] fn test_include_value() { const AS_U32: u32 = include_value!("../testdata/include_value/data"); - assert_eq!(AS_U32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + assert_eq!(AS_U32, u32::from_ne_bytes(*b"abcd")); const AS_I32: i32 = include_value!("../testdata/include_value/data"); - assert_eq!(AS_I32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + assert_eq!(AS_I32, i32::from_ne_bytes(*b"abcd")); } #[test] diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs index 949b60a3f83e..8beb6267b217 100644 --- a/rust/zerocopy/src/pointer/inner.rs +++ b/rust/zerocopy/src/pointer/inner.rs @@ -289,10 +289,10 @@ impl<'a, T> PtrInner<'a, T> /// /// Given `let (left, right) = ptr.split_at(l_len)`, it is guaranteed that /// `left` and `right` are contiguous and non-overlapping if - /// `l_len.padding_needed_for() == 0`. This is true for all `[T]`. + /// the left part has no trailing padding. This is true for all `[T]`. /// - /// If `l_len.padding_needed_for() != 0`, then the left pointer will overlap - /// the right pointer to satisfy `T`'s padding requirements. + /// Any trailing padding in the left part begins at the right part's + /// starting address and may overlap its referent. #[inline] #[must_use] pub unsafe fn split_at_unchecked( @@ -313,23 +313,19 @@ pub unsafe fn split_at_unchecked( // Trivially, `slf.meta() <= slf.meta()`. let right = unsafe { right.slice_unchecked(l_len..self.meta().get()) }; - // SAFETY: If `l_len.padding_needed_for() == 0`, then `left` and `right` - // are non-overlapping. Proof: `left` is constructed `slf` with `l_len` - // as its (exclusive) upper bound. If `l_len.padding_needed_for() == 0`, - // then `left` requires no trailing padding following its final element. - // Since `right` is constructed from `slf`'s trailing slice with `l_len` - // as its (inclusive) lower bound, no byte is referred to by both - // pointers. + // SAFETY: If the left part has no trailing padding, then `left` and + // `right` are non-overlapping. Proof: `left` has `l_len` trailing + // elements, and `right` starts at element `l_len` of the original + // trailing slice. Without padding after `left`'s final element, no + // byte is referred to by both pointers. // - // Conversely, `l_len.padding_needed_for() == N`, where `N - // > 0`, `left` requires `N` bytes of trailing padding following its - // final element. Since `right` is constructed from the trailing slice - // of `slf` with `l_len` as its (inclusive) lower bound, the first `N` - // bytes of `right` are aliased by `left`. + // Any trailing padding in `left` begins at `right`'s starting + // address. If `right` is nonzero sized, the referents overlap. (left, right) } - /// Produces the trailing slice of `self`. + /// Produces the trailing slice of `self` using `T::Elem` as its element + /// representation. #[inline] #[must_use] pub fn trailing_slice(self) -> PtrInner<'a, [T::Elem]> @@ -368,9 +364,10 @@ pub fn trailing_slice(self) -> PtrInner<'a, [T::Elem]> // 1. If `ptr`'s referent is not zero sized, then `ptr` has valid // provenance for `A` because `raw` is derived from the same // allocated object as `self` via provenance-preserving operations. - // 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a byte - // range which is entirely contained in `A`, by previous safety proof - // on `bytes`. + // 2. `T: SplitAt` guarantees that `T::Elem` has the size of the actual + // trailing element. The new slice has the same length and start as + // the original trailing slice, so it covers the same byte range. + // If nonzero sized, that range is entirely contained in `A`. // 3. `ptr` addresses a byte range whose length fits in an `isize`, by // consequence of #2. // 4. `ptr` addresses a byte range which does not wrap around the diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs index 7ff0d43dad5e..8dfaf82e134f 100644 --- a/rust/zerocopy/src/pointer/invariant.rs +++ b/rust/zerocopy/src/pointer/invariant.rs @@ -47,7 +47,7 @@ pub trait Alignment: Sealed { fn read(ptr: crate::Ptr<'_, T, I>) -> T where T: Copy + Read, - I: Invariants, + I: Invariants, I::Aliasing: Reference; } @@ -64,35 +64,39 @@ fn read(ptr: crate::Ptr<'_, T, I>) -> T /// all `T: ?Sized`. Any `V: Validity` must guarantee that this set is only a /// function of the *bit validity* of the referent type, `T`, and not of any /// other property of `T`. As a consequence, given `V: Validity`, `T`, and `U` -/// where `T` and `U` have the same bit validity, `S(V, T) = S(V, U)`. +/// where `T` and `U` have the same bit validity, `S(T, V) = S(U, V)`. /// /// It is guaranteed that the referent of any `ptr: Ptr` is a member of /// `S(T, V)`. Unsafe code must ensure that this guarantee will be upheld for /// any existing `Ptr`s or any `Ptr`s that that code creates. /// -/// An important implication of this guarantee is that it restricts what -/// transmutes are sound, where "transmute" is used in this context to refer to -/// changing the referent type or validity invariant of a `Ptr`, as either -/// change may change the set of bit values permitted to appear in the referent. -/// In particular, the following are necessary (but not sufficient) conditions -/// in order for a transmute from `src: Ptr` to `dst: Ptr` to be -/// sound: -/// - If `S(T, V) = S(U, W)`, then no restrictions apply; otherwise, +/// An important implication of this guarantee is that it restricts which exact +/// reinterpretations are sound. Here, an exact reinterpretation changes the +/// referent type or validity invariant of a `Ptr` while preserving the exact +/// byte range. In particular, given `src: Ptr` and `dst: Ptr` which +/// refer to the same byte range, the following are necessary (but not +/// sufficient) conditions: +/// - If `S(T, V) = S(U, W)`, then no additional validity-preservation +/// restrictions apply; otherwise, /// - If `dst` permits mutation of its referent (e.g. via `Exclusive` aliasing /// or interior mutation under `Shared` aliasing), then it must hold that -/// `S(T, V) ⊇ S(U, W)` - in other words, the transmute must not expand the -/// set of allowed referent bit patterns. A violation of this requirement +/// `S(T, V) ⊇ S(U, W)` - in other words, the reinterpretation must not expand +/// the set of allowed referent bit patterns. A violation of this requirement /// would permit using `dst` to write `x` where `x ∈ S(U, W)` but `x ∉ S(T, /// V)`, which would violate the guarantee that `src`'s referent may only /// contain values in `S(T, V)`. /// - If the referent may be mutated without going through `dst` while `dst` is /// live (e.g. via interior mutation on a `Shared`-aliased `Ptr` or `&` /// reference), then it must hold that `S(T, V) ⊆ S(U, W)` - in other words, -/// the transmute must not shrink the set of allowed referent bit patterns. A -/// violation of this requirement would permit using `src` or another -/// mechanism (e.g. a `&` reference used to derive `src`) to write `x` where -/// `x ∈ S(T, V)` but `x ∉ S(U, W)`, which would violate the guarantee that -/// `dst`'s referent may only contain values in `S(U, W)`. +/// the reinterpretation must not shrink the set of allowed referent bit +/// patterns. A violation of this requirement would permit using `src` or +/// another mechanism (e.g. a `&` reference used to derive `src`) to write `x` +/// where `x ∈ S(T, V)` but `x ∉ S(U, W)`, which would violate the guarantee +/// that `dst`'s referent may only contain values in `S(U, W)`. +/// +/// These conditions describe only exact reinterpretations. Shrinking +/// projections may have validity that depends on the relationship between the +/// projected region and bytes outside it, and require additional reasoning. pub unsafe trait Validity: Sealed { const KIND: ValidityKind; } @@ -101,7 +105,7 @@ pub enum ValidityKind { Uninit, AsInitialized, Initialized, - Valid, + Safe, } /// An [`Aliasing`] invariant which is either [`Shared`] or [`Exclusive`]. @@ -147,7 +151,7 @@ impl Alignment for Unaligned { fn read(ptr: crate::Ptr<'_, T, I>) -> T where T: Copy + Read, - I: Invariants, + I: Invariants, I::Aliasing: Reference, { (*ptr.into_unalign().as_ref()).into_inner() @@ -162,7 +166,7 @@ impl Alignment for Aligned { fn read(ptr: crate::Ptr<'_, T, I>) -> T where T: Copy + Read, - I: Invariants, + I: Invariants, I::Aliasing: Reference, { *ptr.as_ref() @@ -227,13 +231,20 @@ unsafe impl Validity for Initialized { /// The referent of a `Ptr` is valid for `T`, upholding bit validity and any /// library safety invariants. -pub enum Valid {} -// SAFETY: `Valid`'s validity is well-defined for all `T: ?Sized`, and is not a +pub enum Safe {} +// SAFETY: `Safe`'s validity is well-defined for all `T: ?Sized`, and is not a // function of any property of `T` other than its bit validity. -unsafe impl Validity for Valid { - const KIND: ValidityKind = ValidityKind::Valid; +unsafe impl Validity for Safe { + const KIND: ValidityKind = ValidityKind::Safe; } +/// Proof helper for casts whose validity requirement does not depend on the +/// referent type. +/// +/// Currently this covers `Uninit`, which permits any byte state, and +/// `Initialized`, which requires every referent byte to be initialized +/// regardless of referent type. +/// /// # Safety /// /// `DT: CastableFrom` is sound if `SV = DV = Uninit` or `SV = DV = @@ -289,7 +300,7 @@ impl Sealed for Aligned {} impl Sealed for Uninit {} impl Sealed for AsInitialized {} impl Sealed for Initialized {} - impl Sealed for Valid {} + impl Sealed for Safe {} impl Sealed for (A, AA, V) {} diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs index d6eacc52febe..0ce543391530 100644 --- a/rust/zerocopy/src/pointer/mod.rs +++ b/rust/zerocopy/src/pointer/mod.rs @@ -25,9 +25,9 @@ use crate::wrappers::ReadOnly; /// A shorthand for a maybe-valid, maybe-aligned reference. Used as the argument -/// to [`TryFromBytes::is_bit_valid`]. +/// to [`TryFromBytes::is_safe`]. /// -/// [`TryFromBytes::is_bit_valid`]: crate::TryFromBytes::is_bit_valid +/// [`TryFromBytes::is_safe`]: crate::TryFromBytes::is_safe pub type Maybe<'a, T, Alignment = invariant::Unaligned> = Ptr<'a, ReadOnly, (invariant::Shared, Alignment, invariant::Initialized)>; @@ -47,10 +47,7 @@ pub(crate) fn is_zeroed(ptr: Ptr<'_, T, I>) -> bool pub mod cast { use core::{marker::PhantomData, mem}; - use crate::{ - layout::{SizeInfo, TrailingSliceLayout}, - HasField, KnownLayout, PtrInner, - }; + use crate::{layout::SizeInfo, HasField, KnownLayout, PtrInner}; /// A pointer cast or projection. /// @@ -182,11 +179,9 @@ pub enum CastUnsized {} // SAFETY: By the `static_assert!`, `Src` and `Dst` are either: // - Both sized and equal in size - // - Both slice DSTs with the same trailing slice offset and element size - // and with align_of::() == align_of::(). These ensure that any - // given pointer metadata encodes the same size for both `Src` and `Dst` - // (note that the alignment is required as it affects the amount of - // trailing padding). Thus, `project` preserves the set of referent bytes. + // - Both slice DSTs with the same trailing-slice offset, the same object + // size for every metadata value, and with align_of::() == + // align_of::(). Thus, `project` preserves the set of referent bytes. unsafe impl Project for CastUnsized where Src: ?Sized + KnownLayout, @@ -201,10 +196,11 @@ fn project(src: PtrInner<'_, Src>) -> *mut Dst { let dst = ::LAYOUT; match (src.size_info, dst.size_info) { (SizeInfo::Sized { size: src_size }, SizeInfo::Sized { size: dst_size }) => src_size == dst_size, - ( - SizeInfo::SliceDst(TrailingSliceLayout { offset: src_offset, elem_size: src_elem_size }), - SizeInfo::SliceDst(TrailingSliceLayout { offset: dst_offset, elem_size: dst_elem_size }) - ) => src.align.get() == dst.align.get() && src_offset == dst_offset && src_elem_size == dst_elem_size, + (SizeInfo::SliceDst(src_trailing), SizeInfo::SliceDst(dst_trailing)) => { + src.align.get() == dst.align.get() + && src_trailing.offset == dst_trailing.offset + && src_trailing.has_same_size_sequence(dst_trailing) + }, _ => false, } }); @@ -225,10 +221,8 @@ unsafe impl Cast for CastUnsized // SAFETY: By the `static_assert!` in `Project::project`, `Src` and `Dst` // are either: // - Both sized and equal in size - // - Both slice DSTs with the same alignment, trailing slice offset, and - // element size. These ensure that any given pointer metadata encodes the - // same size for both `Src` and `Dst` (note that the alignment is required - // as it affects the amount of trailing padding). + // - Both slice DSTs with the same alignment and trailing-slice offset, and + // with equal sizes for every metadata value. unsafe impl CastExact for CastUnsized where Src: ?Sized + KnownLayout, @@ -240,18 +234,22 @@ unsafe impl CastExact for CastUnsized /// /// A `Projection` is a [`Project`] which implements projection by /// delegating to an implementation of [`HasField::project`]. + /// + /// The `Client` parameter exists to select between implementations of + /// [`HasField`] that would otherwise conflict. #[allow(missing_debug_implementations, missing_copy_implementations)] - pub struct Projection { + pub struct Projection { _never: core::convert::Infallible, + _client: PhantomData, _phantom: PhantomData, } // SAFETY: `HasField::project` has the same safety post-conditions as // `Project::project`. - unsafe impl Project - for Projection + unsafe impl + Project for Projection where - T: HasField, + T: HasField, { #[inline(always)] fn project(src: PtrInner<'_, T>) -> *mut T::Type { @@ -292,10 +290,10 @@ fn project(src: PtrInner<'_, T>) -> *mut T::Type { // // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/595): // Cite the documentation once it's updated. - unsafe impl Cast - for Projection + unsafe impl Cast + for Projection where - T: HasField, + T: HasField, { } diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs index 7213f6f4a04e..0fcd2ed11d37 100644 --- a/rust/zerocopy/src/pointer/ptr.rs +++ b/rust/zerocopy/src/pointer/ptr.rs @@ -169,7 +169,7 @@ mod _conversions { use crate::pointer::cast::{CastExact, CastSized, IdCast}; /// `&'a T` → `Ptr<'a, T>` - impl<'a, T> Ptr<'a, T, (Shared, Aligned, Valid)> + impl<'a, T> Ptr<'a, T, (Shared, Aligned, Safe)> where T: 'a + ?Sized, { @@ -183,20 +183,19 @@ pub fn from_ref(ptr: &'a T) -> Self { // 1. `ptr`, by invariant on `&'a T`, conforms to the alignment // invariant of `Aligned`. // 2. `ptr`'s referent, by invariant on `&'a T`, is a bit-valid `T`. - // This satisfies the requirement that a `Ptr` + // This satisfies the requirement that a `Ptr` // point to a bit-valid `T`. Even if `T` permits interior - // mutation, this invariant guarantees that the returned `Ptr` - // can only ever be used to modify the referent to store - // bit-valid `T`s, which ensures that the returned `Ptr` cannot - // be used to violate the soundness of the original `ptr: &'a T` - // or of any other references that may exist to the same - // referent. + // mutation, this invariant guarantees that the returned `Ptr` can + // only ever be used to modify the referent to store bit-valid + // `T`s, which ensures that the returned `Ptr` cannot be used to + // violate the soundness of the original `ptr: &'a T` or of any + // other references that may exist to the same referent. unsafe { Self::from_inner(inner) } } } /// `&'a mut T` → `Ptr<'a, T>` - impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Safe)> where T: 'a + ?Sized, { @@ -211,7 +210,7 @@ pub fn from_mut(ptr: &'a mut T) -> Self { // invariant of `Aligned`. // 2. `ptr`'s referent, by invariant on `&'a mut T`, is a bit-valid // `T`. This satisfies the requirement that a `Ptr` point to a bit-valid `T`. This invariant guarantees + // Safe)>` point to a bit-valid `T`. This invariant guarantees // that the returned `Ptr` can only ever be used to modify the // referent to store bit-valid `T`s, which ensures that the // returned `Ptr` cannot be used to violate the soundness of the @@ -224,7 +223,7 @@ pub fn from_mut(ptr: &'a mut T) -> Self { impl<'a, T, I> Ptr<'a, T, I> where T: 'a + ?Sized, - I: Invariants, + I: Invariants, I::Aliasing: Reference, { /// Converts `self` to a shared reference. @@ -239,7 +238,7 @@ pub fn as_ref(self) -> &'a T { let raw = self.as_inner().as_non_null(); // SAFETY: `self` satisfies the `Aligned` invariant, so we know that // `raw` is validly-aligned for `T`. - #[cfg(miri)] + #[cfg(all(miri, __ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE))] unsafe { crate::util::miri_promise_symbolic_alignment( raw.as_ptr().cast(), @@ -264,7 +263,7 @@ pub fn as_ref(self) -> &'a T { // // 3. The pointer must point to a validly-initialized instance of // `T`. This is ensured by-contract on `Ptr`, because the - // `I::Validity` is `Valid`. + // `I::Validity` is `Safe`. // // 4. You must enforce Rust’s aliasing rules. This is ensured by // contract on `Ptr`, because `I::Aliasing: Reference`. Either it @@ -377,7 +376,7 @@ pub fn reborrow_shared<'b>(&'b mut self) -> Ptr<'b, T, (Shared, I::Alignment, I: } /// `Ptr<'a, T>` → `&'a mut T` - impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Safe)> where T: 'a + ?Sized, { @@ -389,7 +388,7 @@ pub fn as_mut(self) -> &'a mut T { let mut raw = self.as_inner().as_non_null(); // SAFETY: `self` satisfies the `Aligned` invariant, so we know that // `raw` is validly-aligned for `T`. - #[cfg(miri)] + #[cfg(all(miri, __ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE))] unsafe { crate::util::miri_promise_symbolic_alignment( raw.as_ptr().cast(), @@ -413,8 +412,8 @@ pub fn as_mut(self) -> &'a mut T { // This is ensured by contract on all `PtrInner`s. // // 3. The pointer must point to a validly-initialized instance of - // `T`. This is ensured by-contract on `Ptr`, because the - // validity invariant is `Valid`. + // `T`. This is ensured by-contract on `Ptr`, because the validity + // invariant is `Safe`. // // 4. You must enforce Rust’s aliasing rules. This is ensured by // contract on `Ptr`, because the `ALIASING_INVARIANT` is @@ -431,6 +430,10 @@ impl<'a, T: ?Sized, I> Ptr<'a, T, I> where I: Invariants, { + /// Reinterprets the same byte region as `U` with validity `V`. + /// + /// This preserves the aliasing invariant, uses [`SizeEq`] to select a + /// [`CastExact`] implementation, and conservatively forgets alignment. #[must_use] #[inline(always)] pub fn transmute(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> @@ -443,6 +446,12 @@ pub fn transmute(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> self.transmute_with::>::CastFrom, R>() } + /// Reinterprets the same byte region as `U` with validity `V` using + /// `C`. + /// + /// `C: CastExact` preserves the byte region. The aliasing invariant is + /// preserved, while alignment is conservatively forgotten because the + /// destination type may have a different alignment requirement. #[inline] #[must_use] pub fn transmute_with(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> @@ -462,11 +471,15 @@ pub fn transmute_with(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V // at the same time, as neither can perform interior mutation // - It is directly guaranteed that it is sound for shared code to // operate on these references simultaneously - // - By `U: TransmuteFromPtr`, it + // - By `U: TransmuteFromPtr`, it // is sound to perform this transmute using `C`. unsafe { self.project_transmute_unchecked::<_, _, C>() } } + /// Changes only the validity invariant to `V`. + /// + /// The referent type and byte region are unchanged, so this preserves + /// the existing aliasing and alignment invariants. #[inline] #[must_use] pub fn recall_validity(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> @@ -562,7 +575,7 @@ pub fn into_unalign( // FIXME(#1359): This should be a `transmute_with` call. // Unfortunately, to avoid blanket impl conflicts, we only implement // `TransmuteFrom` for `Unalign` (and vice versa) specifically - // for `Valid` validity, not for all validity types. + // for `Safe` validity, not for all validity types. // SAFETY: // - By `CastSized: Cast`, `CastSized` preserves referent address, @@ -589,7 +602,7 @@ pub fn into_unalign( impl<'a, T, I> Ptr<'a, T, I> where T: ?Sized, - I: Invariants, + I: Invariants, I::Aliasing: Reference, { /// Reads the referent. @@ -656,6 +669,22 @@ pub fn unify_invariants< unsafe { self.assume_invariants::() } } + /// Downgrades `self` to shared aliasing. + #[inline] + #[must_use] + pub(super) fn into_shared(self) -> Ptr<'a, T, (Shared, I::Alignment, I::Validity)> + where + I::Aliasing: Reference, + { + // SAFETY: `I::Aliasing: Reference` guarantees that the source + // aliasing is either `Shared` or `Exclusive`. `Shared` requires no + // transition. If it is `Exclusive`, consuming `self` ends the only + // permitted access to the referent, so the returned pointer can + // soundly carry `Shared` aliasing. The referent type, alignment, + // and validity invariants are unchanged. + unsafe { self.assume_invariants() } + } + /// Assumes that `self`'s referent is validly-aligned for `T` if /// required by `A`. /// @@ -738,40 +767,40 @@ pub unsafe fn assume_initialized( unsafe { self.assume_validity::() } } - /// A shorthand for `self.assume_validity()`. + /// A shorthand for `self.assume_validity()`. /// /// # Safety /// /// The caller promises to uphold the safety preconditions of - /// `self.assume_validity()`. + /// `self.assume_validity()`. #[must_use] #[inline] - pub unsafe fn assume_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)> { + pub unsafe fn assume_safe(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Safe)> { // SAFETY: The caller has promised to uphold the safety // preconditions. - unsafe { self.assume_validity::() } + unsafe { self.assume_validity::() } } /// Checks that `self`'s referent is validly initialized for `T`, - /// returning a `Ptr` with `Valid` on success. + /// returning a `Ptr` with `Safe` on success. /// /// # Panics /// /// This method will panic if - /// [`T::is_bit_valid`][TryFromBytes::is_bit_valid] panics. + /// [`T::is_safe`][TryFromBytes::is_safe] panics. /// /// # Safety /// /// On error, unsafe code may rely on this method's returned /// `ValidityError` containing `self`. #[inline] - pub fn try_into_valid( + pub fn try_into_safe( mut self, - ) -> Result, ValidityError> + ) -> Result, ValidityError> where T: TryFromBytes + Read - + TryTransmuteFromPtr, + + TryTransmuteFromPtr, ReadOnly: Read, I::Aliasing: Reference, I: Invariants, @@ -779,14 +808,13 @@ pub fn try_into_valid( // This call may panic. If that happens, it doesn't cause any // soundness issues, as we have not generated any invalid state // which we need to fix before returning. - if T::is_bit_valid(self.reborrow().transmute::<_, _, _>().reborrow_shared()) { - // SAFETY: If `T::is_bit_valid`, code may assume that `self` - // contains a bit-valid instance of `T`. By `T: - // TryTransmuteFromPtr`, so - // long as `self`'s referent conforms to the `Valid` validity - // for `T` (which we just confirmed), then this transmute is - // sound. - Ok(unsafe { self.assume_valid() }) + if T::is_safe(self.reborrow().transmute::<_, _, _>().reborrow_shared()) { + // SAFETY: If `T::is_safe` returns true, code may assume that + // `self` contains a valid `T`, so its referent conforms to + // `Safe` for `T`. By `T: TryTransmuteFromPtr`, given that condition, changing `self`'s + // validity to `Safe` is sound. + Ok(unsafe { self.assume_safe() }) } else { Err(ValidityError::new(self)) } @@ -803,7 +831,7 @@ pub fn forget_aligned(self) -> Ptr<'a, T, (I::Aliasing, Unaligned, I::Validity)> } /// Casts of the referent type. -#[cfg_attr(not(zerocopy_unstable_ptr), allow(unreachable_pub))] +#[allow(unreachable_pub)] // False positive on MSRV pub use _casts::TryWithError; mod _casts { use core::cell::UnsafeCell; @@ -875,19 +903,22 @@ pub fn cast(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> } #[inline(always)] - pub fn project( + pub fn project( mut self, ) -> Result, T::Error> where - T: ProjectField, + T: ProjectField, I::Aliasing: Reference, { use crate::pointer::cast::Projection; - match T::is_projectable(self.reborrow().project_tag()) { + match >::is_projectable( + self.reborrow().project_tag::(), + ) { Ok(()) => { let inner = self.as_inner(); - let projected = inner.project::<_, Projection>(); - // SAFETY: By `T: ProjectField`, + let projected = + inner.project::<_, Projection>(); + // SAFETY: By `T: ProjectField`, // for `self: Ptr<'_, T, I>` such that `T::is_projectable` // (which we've verified in this match arm), // `T::project(self.as_inner())` conforms to @@ -908,17 +939,22 @@ pub fn project( #[must_use] #[inline(always)] - pub(crate) fn project_tag(self) -> Ptr<'a, T::Tag, I> + pub fn project_tag( + self, + ) -> Ptr<'a, >::Tag, (Shared, I::Alignment, I::Validity)> where - T: HasTag, + T: HasTag, + I::Aliasing: Reference, { - // SAFETY: By invariant on `Self::ProjectToTag`, this is a sound - // projection. - let tag = unsafe { self.project_transmute_unchecked::<_, _, T::ProjectToTag>() }; - // SAFETY: By invariant on `Self::ProjectToTag`, the projected - // pointer has the same alignment as `ptr`. - let tag = unsafe { tag.assume_alignment() }; - tag.unify_invariants() + let ptr = self.into_shared(); + // SAFETY: By invariant on `ProjectToTag`, it is sound to project + // `ptr` to a shared tag pointer with the same validity invariant. + let ptr = unsafe { + ptr.project_transmute_unchecked::<_, _, >::ProjectToTag>() + }; + // SAFETY: By invariant on `ProjectToTag`, the projected pointer has + // the same alignment as `self`. + unsafe { ptr.assume_alignment() } } /// Attempts to transform the pointer, restoring the original on @@ -1015,11 +1051,11 @@ impl<'a, T, I> Ptr<'a, T, I> #[allow(clippy::wrong_self_convention)] #[must_use] #[inline] - pub fn as_bytes(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Valid)> + pub fn as_bytes(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Safe)> where - [u8]: TransmuteFromPtr, + [u8]: TransmuteFromPtr, { - self.transmute_with::<[u8], Valid, AsBytesCast, _>().bikeshed_recall_aligned() + self.transmute_with::<[u8], Safe, AsBytesCast, _>().bikeshed_recall_aligned() } } @@ -1071,7 +1107,7 @@ pub fn as_slice(self) -> Ptr<'a, [T], I> { /// alignment of `[u8]` is 1. impl<'a, I> Ptr<'a, [u8], I> where - I: Invariants, + I: Invariants, { /// Attempts to cast `self` to a `U` using the given cast type. /// @@ -1135,7 +1171,7 @@ pub fn try_cast_into( // it is derived from `try_cast_into`, which promises that the // object described by `target` is validly aligned for `U`. // 2. By trait bound, `self` - and thus `target` - is a bit-valid - // `[u8]`. `Ptr<[u8], (_, _, Valid)>` and `Ptr<_, (_, _, + // `[u8]`. `Ptr<[u8], (_, _, Safe)>` and `Ptr<_, (_, _, // Initialized)>` have the same bit validity, and so neither // `self` nor `res` can be used to write a value to the referent // which violates the other's validity invariant. @@ -1143,14 +1179,14 @@ pub fn try_cast_into( // SAFETY: // 0. `self` and `remainder` both have the type `[u8]`. Thus, they - // have `UnsafeCell`s at the same locations. Type casting does - // not affect aliasing. + // have `UnsafeCell`s at the same locations. Type casting does not + // affect aliasing. // 1. `[u8]` has no alignment requirement. - // 2. `self` has validity `Valid` and has type `[u8]`. Since + // 2. `self` has validity `Safe` and has type `[u8]`. Since // `remainder` references a subset of `self`'s referent, it is // also a bit-valid `[u8]`. Thus, neither `self` nor `remainder` - // can be used to write a value to the referent which violates - // the other's validity invariant. + // can be used to write a value to the referent which violates the + // other's validity invariant. let remainder = unsafe { Ptr::from_inner(remainder) }; Ok((res, remainder)) @@ -1333,6 +1369,23 @@ mod tests { use crate::util::AsAddress; use crate::{pointer::BecauseImmutable, util::testutil::AU64, FromBytes, Immutable}; + #[test] + fn test_project_tag_downgrades_aliasing() { + #[allow(dead_code)] + #[derive(zerocopy_derive::Project)] + #[repr(u8)] + enum Enum { + Variant(u8), + } + + let mut value = Enum::Variant(0); + let _: Ptr< + '_, + >::Tag, + (Shared, Aligned, Safe), + > = Ptr::from_mut(&mut value).project_tag::(); + } + mod test_ptr_try_cast_into_soundness { use super::*; diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs index ef9836698203..75f9312f622e 100644 --- a/rust/zerocopy/src/pointer/transmute.rs +++ b/rust/zerocopy/src/pointer/transmute.rs @@ -24,14 +24,14 @@ FromBytes, Immutable, IntoBytes, Unalign, }; -/// Transmutations which are sound to attempt, conditional on validating the bit -/// validity of the destination type. +/// Exact pointer reinterpretations which are sound to attempt, conditional on +/// establishing the destination validity invariant. /// /// If a `Ptr` transmutation is `TryTransmuteFromPtr`, then it is sound to /// perform that transmutation so long as some additional mechanism is used to -/// validate that the referent is bit-valid for the destination type. That -/// validation mechanism could be a type bound (such as `TransmuteFrom`) or a -/// runtime validity check. +/// establish that the referent is `DV`-valid for the destination type. That +/// mechanism could be a type bound (such as `TransmuteFrom`) or a runtime +/// validity check. /// /// # Safety /// @@ -49,14 +49,14 @@ /// Given `src: Ptr` and `dst: Ptr`, /// `Dst: TryTransmuteFromPtr` is sound if all of the /// following hold: -/// - Forwards transmutation: Either of the following hold: +/// - Preserve destination validity: Either of the following hold: /// - So long as `dst` is active, no mutation of `dst`'s referent is allowed /// except via `dst` itself /// - The set of `DV`-valid referents of `dst` is a superset of the set of /// `SV`-valid referents of `src` (NOTE: this condition effectively bans /// shrinking or overwriting transmutes, which cannot satisfy this /// condition) -/// - Reverse transmutation: Either of the following hold: +/// - Preserve source validity: Either of the following hold: /// - `dst` does not permit mutation of its referent /// - The set of `DV`-valid referents of `dst` is a subset of the set of /// `SV`-valid referents of `src` (NOTE: this condition effectively bans @@ -126,7 +126,7 @@ pub unsafe trait TryTransmuteFromPtr< pub enum BecauseMutationCompatible {} // SAFETY: -// - Forwards transmutation: By `Dst: MutationCompatible`, we +// - Preserve destination validity: By `Dst: MutationCompatible`, we // know that at least one of the following holds: // - So long as `dst: Ptr` is active, no mutation of its referent is // allowed except via `dst` itself if either of the following hold: @@ -138,17 +138,18 @@ pub enum BecauseMutationCompatible {} // referent as `src`. By `Dst: TransmuteFrom`, the set of // `DV`-valid referents of `dst` is a superset of the set of `SV`-valid // referents of `src`. -// - Reverse transmutation: Since the underlying cast is size-preserving, `dst` -// addresses the same referent as `src`. By `Src: TransmuteFrom`, -// the set of `DV`-valid referents of `src` is a subset of the set of -// `SV`-valid referents of `dst`. +// - Preserve source validity: Since the underlying cast is size-preserving, +// `dst` addresses the same referent as `src`. By +// `Src: TransmuteFrom`, the set of `DV`-valid referents of `dst` +// is a subset of the set of `SV`-valid referents of `src`. // - No safe code, given access to `src` and `dst`, can cause undefined // behavior: By `Dst: MutationCompatible`, at least one of // the following holds: // - `A` is `Exclusive` // - `Src: Immutable` and `Dst: Immutable` -// - `Dst: InvariantsEq`, which guarantees that `Src` and `Dst` have the -// same invariants, and permit interior mutation on the same byte ranges +// - `Dst: SharedCompatible`, which guarantees that it is sound for safe +// code to operate on `&Src` and `&Dst` pointing to the same byte range at +// the same time unsafe impl TryTransmuteFromPtr for Dst where @@ -162,9 +163,9 @@ unsafe impl } // SAFETY: -// - Forwards transmutation: Since aliasing is `Shared` and `Src: Immutable`, +// - Preserve destination validity: Since aliasing is `Shared` and `Src: Immutable`, // `src` does not permit mutation of its referent. -// - Reverse transmutation: Since aliasing is `Shared` and `Dst: Immutable`, +// - Preserve source validity: Since aliasing is `Shared` and `Dst: Immutable`, // `dst` does not permit mutation of its referent. // - No safe code, given access to `src` and `dst`, can cause undefined // behavior: `Src: Immutable` and `Dst: Immutable` @@ -187,9 +188,9 @@ unsafe impl TryTransmuteFromPtr` and `Self: Read` -/// - `Self: InvariantsEq`, and, for some `V`: -/// - `Dst: TransmuteFrom` -/// - `Src: TransmuteFrom` +/// - `Self: SharedCompatible`, and, for some `V`: +/// - `Self: TransmuteFrom` +/// - `Src: TransmuteFrom` pub unsafe trait MutationCompatible {} #[allow(missing_copy_implementations, missing_debug_implementations)] @@ -204,38 +205,39 @@ unsafe impl {} +pub unsafe trait SharedCompatible {} // SAFETY: Trivially sound to have multiple `&T` pointing to the same referent. -unsafe impl InvariantsEq for T {} +unsafe impl SharedCompatible for T {} -// SAFETY: `Dst: InvariantsEq + TransmuteFrom`, and `Src: +// SAFETY: `Dst: SharedCompatible + TransmuteFrom`, and `Src: // TransmuteFrom`. unsafe impl - MutationCompatible for Dst + MutationCompatible for Dst where Src: TransmuteFrom, - Dst: TransmuteFrom + InvariantsEq, + Dst: TransmuteFrom + SharedCompatible, { } #[allow(missing_debug_implementations, missing_copy_implementations)] -pub enum BecauseInvariantsEq {} +pub enum BecauseSharedCompatible {} -macro_rules! unsafe_impl_invariants_eq { +macro_rules! unsafe_impl_shared_compatible { ($tyvar:ident => $t:ty, $u:ty) => {{ crate::util::macros::__unsafe(); // SAFETY: The caller promises that this is sound. - unsafe impl<$tyvar> InvariantsEq<$t> for $u {} + unsafe impl<$tyvar> SharedCompatible<$t> for $u {} // SAFETY: The caller promises that this is sound. - unsafe impl<$tyvar> InvariantsEq<$u> for $t {} + unsafe impl<$tyvar> SharedCompatible<$u> for $t {} }}; } @@ -253,11 +255,11 @@ unsafe impl<$tyvar> InvariantsEq<$u> for $t {} // validity as `T` // // [2] https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E -unsafe impl InvariantsEq for ManuallyDrop {} +unsafe impl SharedCompatible for ManuallyDrop {} // SAFETY: See previous safety comment. -unsafe impl InvariantsEq> for T {} +unsafe impl SharedCompatible> for T {} -/// Transmutations which are always sound. +/// Exact pointer reinterpretations which are always sound. /// /// `TransmuteFromPtr` is a shorthand for [`TryTransmuteFromPtr`] and /// [`TransmuteFrom`]. @@ -293,8 +295,12 @@ unsafe impl< { } -/// Denotes that any `SV`-valid `Src` may soundly be transmuted into a -/// `DV`-valid `Self`. +/// Denotes that, for equally-sized referents, every bit pattern allowed for an +/// `SV`-valid `Src` is also allowed for a `DV`-valid `Self`. +/// +/// `TransmuteFrom` does not by itself authorize a pointer transmutation. It +/// provides a directional validity implication used by the pointer-transmutation +/// machinery. /// /// # Safety /// @@ -333,7 +339,7 @@ impl SizeEq for T { // SAFETY: Since `Src: IntoBytes`, the set of valid `Src`'s is the set of // initialized bit patterns, which is exactly the set allowed in the referent of // any `Initialized` `Ptr`. -unsafe impl TransmuteFrom for Dst +unsafe impl TransmuteFrom for Dst where Src: IntoBytes + ?Sized, Dst: ?Sized, @@ -341,18 +347,19 @@ unsafe impl TransmuteFrom for Dst } // SAFETY: Since `Dst: FromBytes`, any initialized bit pattern may appear in the -// referent of a `Ptr`. This is exactly equal to the set of +// referent of a `Ptr`. This is exactly equal to the set of // bit patterns which may appear in the referent of any `Initialized` `Ptr`. -unsafe impl TransmuteFrom for Dst +unsafe impl TransmuteFrom for Dst where Src: ?Sized, Dst: FromBytes + ?Sized, { } -// FIXME(#2354): This seems like a smell - the soundness of this bound has -// nothing to do with `Src` or `Dst` - we're basically just saying `[u8; N]` is -// transmutable into `[u8; N]`. +// FIXME(#2354): This relation does not depend on `Src` or `Dst`. That is +// expected semantically: `Initialized` permits the same states regardless of +// referent type. The awkward part is encoding that validity-state relation as a +// trait relation between referent types. // SAFETY: The set of allowed bit patterns in the referent of any `Initialized` // `Ptr` is the same regardless of referent type. @@ -363,9 +370,10 @@ unsafe impl TransmuteFrom for Dst { } -// FIXME(#2354): This seems like a smell - the soundness of this bound has -// nothing to do with `Dst` - we're basically just saying that any type is -// transmutable into `MaybeUninit<[u8; N]>`. +// FIXME(#2354): This relation does not depend on `Src` or `Dst`. That is +// expected semantically: every validity state implies `Uninit`, which permits +// any byte sequence. The awkward part is encoding that validity-state relation +// as a trait relation between referent types. // SAFETY: A `Dst` with validity `Uninit` permits any byte sequence, and // therefore can be transmuted from any value. @@ -400,7 +408,7 @@ unsafe impl TransmuteFrom for Dst // sound for these two references to exist at the same time since it's already // possible for safe code to get into this state. #[allow(clippy::multiple_unsafe_ops_per_block)] -const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Unalign) }; +const _: () = unsafe { unsafe_impl_shared_compatible!(T => T, Unalign) }; // SAFETY: // - `Wrapping` has the same size as `T` [1]. @@ -433,7 +441,7 @@ unsafe impl TransmuteFrom for Dst // already possible for safe code to obtain a `&Wrapping` and a `&T` pointing // to the same referent at the same time. Thus, this must be sound. #[allow(clippy::multiple_unsafe_ops_per_block)] -const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Wrapping) }; +const _: () = unsafe { unsafe_impl_shared_compatible!(T => T, Wrapping) }; // SAFETY: // - `UnsafeCell` has the same size as `T` [1]. @@ -478,7 +486,7 @@ unsafe impl TransmuteFrom for Dst // explicitly guaranteed, but it's obvious from `MaybeUninit`'s documentation // that this is the intention: // https://doc.rust-lang.org/1.85.0/core/mem/union.MaybeUninit.html -unsafe impl TransmuteFrom for MaybeUninit {} +unsafe impl TransmuteFrom for MaybeUninit {} impl SizeEq for MaybeUninit { type CastFrom = CastSizedExact; diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs index e49f2a887ffa..7a8876a0587b 100644 --- a/rust/zerocopy/src/ref.rs +++ b/rust/zerocopy/src/ref.rs @@ -10,7 +10,7 @@ // those terms. use super::*; use crate::pointer::{ - BecauseInvariantsEq, BecauseMutationCompatible, MutationCompatible, TransmuteFromPtr, + BecauseMutationCompatible, BecauseSharedCompatible, MutationCompatible, TransmuteFromPtr, }; mod def { @@ -18,7 +18,7 @@ mod def { use crate::{ ByteSlice, ByteSliceMut, CloneableByteSlice, CopyableByteSlice, IntoByteSlice, - IntoByteSliceMut, + IntoByteSliceMut, KnownLayout, PointerMetadata, }; /// A typed reference derived from a byte slice. @@ -59,30 +59,83 @@ mod def { /// } /// } /// ``` - pub struct Ref( - // INVARIANTS: The referent (via `.deref`, `.deref_mut`, `.into`) byte - // slice is aligned to `T`'s alignment and its size corresponds to a - // valid size for `T`. - B, - PhantomData, - ); + pub struct Ref { + // INVARIANTS: + // - The referent (via `.deref`, `.deref_mut`, `.into`) byte slice is + // aligned to `T`'s alignment, and its size is valid for `T`. + // - If `T: KnownLayout`, let `metadata` be + // `T::PointerMetadata::from_elem_count` applied to `elem_count`. + // Then `T::size_for_metadata(metadata)` is `Some(referent.len())`. + bytes: B, + // Ignored when `T` does not implement `KnownLayout`. + elem_count: usize, + _marker: PhantomData, + } - impl Ref { + impl Ref { + /// Constructs a new `Ref` to a sized `T`. + /// + /// # Safety + /// + /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to + /// a byte slice which is aligned to `T`'s alignment and whose size is + /// `size_of::()`. + /// + /// [`deref`]: core::ops::Deref::deref + /// [`deref_mut`]: core::ops::DerefMut::deref_mut + /// [`into`]: core::convert::Into::into + pub(super) unsafe fn new_sized_unchecked(bytes: B) -> Ref { + // INVARIANTS: The caller has promised that `bytes`'s referent is + // validly aligned and has size `size_of::()`. If `T: + // KnownLayout`, then, since `T` is sized, `T::PointerMetadata` is + // `()`. `from_elem_count` returns `()` for any element count, and + // `T::size_for_metadata(()) == Some(size_of::())`. `size_of` + // reports this size in bytes [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + Ref { bytes, elem_count: 0, _marker: PhantomData } + } + } + + impl Ref { /// Constructs a new `Ref`. /// /// # Safety /// /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to - /// a byte slice which is aligned to `T`'s alignment and whose size is a - /// valid size for `T`. + /// a byte slice which is aligned to `T`'s alignment and satisfies + /// `T::size_for_metadata(metadata) == Some(referent.len())`. /// /// [`deref`]: core::ops::Deref::deref /// [`deref_mut`]: core::ops::DerefMut::deref_mut /// [`into`]: core::convert::Into::into - pub(crate) unsafe fn new_unchecked(bytes: B) -> Ref { + pub(super) unsafe fn new_unchecked(bytes: B, metadata: T::PointerMetadata) -> Ref { // INVARIANTS: The caller has promised that `bytes`'s referent is - // validly-aligned and has a valid size. - Ref(bytes, PhantomData) + // validly aligned and has exactly the size described by `metadata`. + // `KnownLayout::PointerMetadata` is `()` for sized types and + // `usize` for slice DSTs. For both permitted metadata types, + // `from_elem_count(metadata.to_elem_count()) == metadata`: `()` + // round-trips through zero, while `usize` round-trips through + // itself. Thus, storing the element count preserves the metadata + // whose size the caller validated. This agrees with Rust's + // language-level definition of slice pointer metadata [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/reference/dynamically-sized-types.html: + // + // Pointers to slices also store the number of elements of the + // slice. + Ref { bytes, elem_count: metadata.to_elem_count(), _marker: PhantomData } + } + + /// Gets the pointer metadata represented by this `Ref`. + #[inline(always)] + pub(super) fn pointer_metadata(&self) -> T::PointerMetadata { + // INVARIANTS: `KnownLayout::PointerMetadata` is either `()` or + // `usize`. For `()`, `from_elem_count` ignores the element count; + // for `usize`, it returns the element count unchanged. + T::PointerMetadata::from_elem_count(self.elem_count) } } @@ -96,16 +149,18 @@ impl Ref { /// `Any::downcast_ref`). /// /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// validly aligned for `T` and has a valid size for `T`. If `T: + /// KnownLayout`, then `T::size_for_metadata(self.pointer_metadata())` + /// returns `Some(referent.len())`. pub(crate) unsafe fn as_byte_slice(&self) -> &impl ByteSlice { // INVARIANTS: The caller promises not to call methods other than // those on `ByteSlice`. Since `B: ByteSlice`, dereference stability // guarantees that calling `ByteSlice` methods will not change the - // address or length of `self.0`'s referent. + // address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - &self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + &self.bytes } } @@ -118,17 +173,20 @@ impl Ref { /// [`ByteSliceMut`] other than `ByteSliceMut` methods (for example, via /// `Any::downcast_mut`). /// - /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// `as_byte_slice_mut` promises to return a `ByteSliceMut` whose + /// referent is validly aligned for `T` and has a valid size for `T`. If + /// `T: KnownLayout`, then + /// `T::size_for_metadata(self.pointer_metadata())` returns + /// `Some(referent.len())`. pub(crate) unsafe fn as_byte_slice_mut(&mut self) -> &mut impl ByteSliceMut { // INVARIANTS: The caller promises not to call methods other than // those on `ByteSliceMut`. Since `B: ByteSlice`, dereference // stability guarantees that calling `ByteSlice` methods will not - // change the address or length of `self.0`'s referent. + // change the address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - &mut self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + &mut self.bytes } } @@ -141,17 +199,20 @@ impl<'a, B: IntoByteSlice<'a>, T: ?Sized> Ref { /// [`IntoByteSlice`] other than `IntoByteSlice` methods (for example, /// via `Any::downcast_ref`). /// - /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// `into_byte_slice` promises to return an `IntoByteSlice` whose + /// referent is validly aligned for `T` and has a valid size for `T`. If + /// `T: KnownLayout`, then + /// `T::size_for_metadata(self.pointer_metadata())` returns + /// `Some(referent.len())`. pub(crate) unsafe fn into_byte_slice(self) -> impl IntoByteSlice<'a> { // INVARIANTS: The caller promises not to call methods other than // those on `IntoByteSlice`. Since `B: ByteSlice`, dereference // stability guarantees that calling `ByteSlice` methods will not - // change the address or length of `self.0`'s referent. + // change the address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + self.bytes } } @@ -164,33 +225,38 @@ impl<'a, B: IntoByteSliceMut<'a>, T: ?Sized> Ref { /// [`IntoByteSliceMut`] other than `IntoByteSliceMut` methods (for /// example, via `Any::downcast_mut`). /// - /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// `into_byte_slice_mut` promises to return an `IntoByteSliceMut` whose + /// referent is validly aligned for `T` and has a valid size for `T`. If + /// `T: KnownLayout`, then + /// `T::size_for_metadata(self.pointer_metadata())` returns + /// `Some(referent.len())`. pub(crate) unsafe fn into_byte_slice_mut(self) -> impl IntoByteSliceMut<'a> { // INVARIANTS: The caller promises not to call methods other than // those on `IntoByteSliceMut`. Since `B: ByteSlice`, dereference // stability guarantees that calling `ByteSlice` methods will not - // change the address or length of `self.0`'s referent. + // change the address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + self.bytes } } impl Clone for Ref { #[inline] fn clone(&self) -> Ref { - // INVARIANTS: Since `B: CloneableByteSlice`, `self.0.clone()` has - // the same address and length as `self.0`. Since `self.0` upholds - // the field invariants, so does `self.0.clone()`. - Ref(self.0.clone(), PhantomData) + // INVARIANTS: Since `B: CloneableByteSlice`, `self.bytes.clone()` has + // the same address and length as `self.bytes`. We copy + // `self.elem_count`, so the clone also has the same metadata. Since + // `self` upholds the field invariants, so does the clone. + Ref { bytes: self.bytes.clone(), elem_count: self.elem_count, _marker: PhantomData } } } - // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `.0` has the - // same address and length as the original `Ref`'s `.0`. Since the original - // upholds the field invariants, so does the copy. + // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `bytes` has the + // same address and length as the original `Ref`'s `bytes`. The copied + // `elem_count` has the same metadata. Since the original upholds the field + // invariants, so does the copy. impl Copy for Ref {} } @@ -198,7 +264,7 @@ impl Copy for Ref {} pub use def::Ref; use crate::pointer::{ - invariant::{Aligned, BecauseExclusive, Initialized, Unaligned, Valid}, + invariant::{Aligned, BecauseExclusive, Initialized, Safe, Unaligned}, BecauseRead, PtrInner, }; @@ -215,8 +281,14 @@ pub(crate) fn sized_from(bytes: B) -> Result, CastError> { return Err(err.with_src(bytes).into()); } - // SAFETY: We just validated size and alignment. - Ok(unsafe { Ref::new_unchecked(bytes) }) + // SAFETY: `size_of` returns `T`'s size in bytes [1]. The equality check + // above therefore establishes that `bytes` has exactly that byte + // length, and `validate_aligned_to` establishes alignment. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + Ok(unsafe { Ref::new_sized_unchecked(bytes) }) } } @@ -238,10 +310,15 @@ pub(crate) fn sized_from_prefix(bytes: B) -> Result<(Ref, B), CastError())?` ensures that the - // new `bytes` is exactly the size of `T`. By safety postcondition on + // new `bytes` is exactly the size of `T`, measured in bytes as specified + // for `size_of` [1]. By safety postcondition on // `SplitByteSlice::split_at` we can rely on `split_at` to produce the // correct `bytes` and `suffix`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::new_sized_unchecked(bytes) }; Ok((r, suffix)) } @@ -259,11 +336,16 @@ pub(crate) fn sized_from_suffix(bytes: B) -> Result<(B, Ref), CastError()`, // the `bytes` which results from `let (prefix, bytes) = - // bytes.split_at(split_at)?` has length `size_of::()`. After - // constructing `bytes`, we validate that it has the proper alignment. - // By safety postcondition on `SplitByteSlice::split_at` we can rely on - // `split_at` to produce the correct `prefix` and `bytes`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // bytes.split_at(split_at)?` has length `size_of::()`, measured in + // bytes as specified for `size_of` [1]. After constructing `bytes`, we + // validate that it has the proper alignment. By safety postcondition on + // `SplitByteSlice::split_at` we can rely on `split_at` to produce the + // correct `prefix` and `bytes`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::new_sized_unchecked(bytes) }; Ok((prefix, r)) } } @@ -306,17 +388,64 @@ impl Ref /// /// let _ = Ref::<_, ZSTy>::from_bytes(&b"UU"[..]); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline] pub fn from_bytes(source: B) -> Result, CastError> { static_assert_dst_is_not_zst!(T); - if let Err(e) = - Ptr::from_ref(source.deref()).try_cast_into_no_leftover::(None) + let metadata = match Ptr::from_ref(source.deref()) + .try_cast_into_no_leftover::(None) { - return Err(e.with_src(()).with_src(source)); - } - // SAFETY: `try_cast_into_no_leftover` validates size and alignment. - Ok(unsafe { Ref::new_unchecked(source) }) + Ok(ptr) => T::pointer_to_metadata(ptr.as_inner().as_ptr()), + Err(e) => return Err(e.with_src(()).with_src(source)), + }; + // SAFETY: By its safety postcondition, `try_cast_into_no_leftover` + // returned a `Ptr` aligned as required for references [2], over the + // same byte range as `source.deref()`. `pointer_to_metadata` returns + // the metadata stored in that pointer, and `size_for_metadata` computes + // the corresponding dynamically-known byte size [1]. Thus, + // `T::size_for_metadata(metadata)` is `Some(source.len())`. + // `ByteSlice` and its applicable subtraits guarantee that the stored + // `source` continues to dereference or convert to that address and + // length. These facts establish both preconditions of `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + Ok(unsafe { Ref::new_unchecked(source, metadata) }) } } @@ -360,14 +489,37 @@ impl Ref /// /// let _ = Ref::<_, ZSTy>::from_prefix(&b"UU"[..]); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline] pub fn from_prefix(source: B) -> Result<(Ref, B), CastError> { static_assert_dst_is_not_zst!(T); - let remainder = match Ptr::from_ref(source.deref()) + let (metadata, remainder) = match Ptr::from_ref(source.deref()) .try_cast_into::(CastType::Prefix, None) { - Ok((_, remainder)) => remainder, + Ok((ptr, remainder)) => (T::pointer_to_metadata(ptr.as_inner().as_ptr()), remainder), Err(e) => { return Err(e.with_src(()).with_src(source)); } @@ -381,11 +533,35 @@ pub fn from_prefix(source: B) -> Result<(Ref, B), CastError> { #[allow(unstable_name_collisions)] let split_at = unsafe { source.len().unchecked_sub(remainder.len()) }; let (bytes, suffix) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; - // SAFETY: `try_cast_into` validates size and alignment, and returns a - // `split_at` that indicates how many bytes of `source` correspond to a - // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we - // can rely on `split_at` to produce the correct `source` and `suffix`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // SAFETY: By its safety postcondition, `try_cast_into` returned an + // `Ptr` aligned as required for references [2], and a remainder + // which are non-overlapping, cover `source.deref()`, and place the + // `Ptr` at `source.deref()`'s address. + // Therefore, `split_at` is the typed pointer's byte length. By the + // safety postcondition of `SplitByteSlice::split_at`, `bytes` has that + // length and address. `pointer_to_metadata` returns the metadata stored + // in the typed pointer, and `size_for_metadata` computes the + // corresponding dynamically-known byte size [1]. Thus, + // `T::size_for_metadata(metadata)` is `Some(bytes.len())`. The + // `SplitByteSlice` and `ByteSlice` contracts preserve the established + // address and length, satisfying both preconditions of + // `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + let r = unsafe { Ref::new_unchecked(bytes, metadata) }; Ok((r, suffix)) } @@ -425,14 +601,37 @@ pub fn from_prefix(source: B) -> Result<(Ref, B), CastError> { /// /// let _ = Ref::<_, ZSTy>::from_suffix(&b"UU"[..]); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline] pub fn from_suffix(source: B) -> Result<(B, Ref), CastError> { static_assert_dst_is_not_zst!(T); - let remainder = match Ptr::from_ref(source.deref()) + let (metadata, remainder) = match Ptr::from_ref(source.deref()) .try_cast_into::(CastType::Suffix, None) { - Ok((_, remainder)) => remainder, + Ok((ptr, remainder)) => (T::pointer_to_metadata(ptr.as_inner().as_ptr()), remainder), Err(e) => { let e = e.with_src(()); return Err(e.with_src(source)); @@ -441,11 +640,39 @@ pub fn from_suffix(source: B) -> Result<(B, Ref), CastError> { let split_at = remainder.len(); let (prefix, bytes) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; - // SAFETY: `try_cast_into` validates size and alignment, and returns a - // `split_at` that indicates how many bytes of `source` correspond to a - // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we - // can rely on `split_at` to produce the correct `prefix` and `bytes`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // SAFETY: By its safety postcondition, `try_cast_into` returned an + // `Ptr` aligned as required for references [2], and a remainder + // which are non-overlapping, cover `source.deref()`, and place the + // remainder at `source.deref()`'s address. More precisely, + // `PtrInner::try_cast_into` constructs the typed suffix from the + // `r_slice` produced by `split_at_unchecked(split_at)` and returns its + // `l_slice` as `remainder`. Thus, even when `T` is zero-sized, the typed + // pointer and the `bytes` produced by splitting `source` at + // `remainder.len()` have the same address and length. By the safety + // postcondition of `SplitByteSlice::split_at`, `bytes` retains that + // address and length. `pointer_to_metadata` returns the metadata stored + // in the typed pointer, and `size_for_metadata` computes the + // corresponding dynamically-known byte size [1]. Thus, + // `T::size_for_metadata(metadata)` is `Some(bytes.len())`. The + // `SplitByteSlice` and `ByteSlice` contracts preserve the established + // address and length, satisfying both preconditions of + // `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + let r = unsafe { Ref::new_unchecked(bytes, metadata) }; Ok((prefix, r)) } } @@ -487,6 +714,24 @@ impl Ref /// /// let _ = Ref::<_, ZSTy>::from_bytes_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn from_bytes_with_elems(source: B, count: usize) -> Result, CastError> { static_assert_dst_is_not_zst!(T); @@ -497,7 +742,37 @@ pub fn from_bytes_with_elems(source: B, count: usize) -> Result, CastE if source.len() != expected_len { return Err(SizeError::new(source).into()); } - Self::from_bytes(source) + if let Err(e) = Ptr::from_ref(source.deref()) + .try_cast_into_no_leftover::(Some(count)) + { + return Err(e.with_src(()).with_src(source)); + } + // SAFETY: `size_for_metadata(count)` returned `expected_len`, and the + // equality check established that this is `source.len()`. This is the + // dynamically-known byte size described by `size_of_val` [1]. By its + // safety postcondition, `try_cast_into_no_leftover(Some(count))` only + // succeeded after producing a `Ptr` aligned as required for + // references [2], with that metadata over the same byte range as + // `source.deref()`. `ByteSlice` and its applicable subtraits guarantee + // that the stored `source` continues to dereference or convert to that + // address and length. These facts establish both preconditions of + // `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + Ok(unsafe { Ref::new_unchecked(source, count) }) } } @@ -538,6 +813,24 @@ impl Ref /// /// let _ = Ref::<_, ZSTy>::from_prefix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn from_prefix_with_elems( source: B, @@ -549,7 +842,7 @@ pub fn from_prefix_with_elems( None => return Err(SizeError::new(source).into()), }; let (prefix, bytes) = source.split_at(expected_len).map_err(SizeError::new)?; - Self::from_bytes(prefix).map(move |l| (l, bytes)) + Self::from_bytes_with_elems(prefix, count).map(move |l| (l, bytes)) } /// Constructs a `Ref` from the suffix of the given bytes with DST length @@ -584,6 +877,24 @@ pub fn from_prefix_with_elems( /// /// let _ = Ref::<_, ZSTy>::from_suffix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn from_suffix_with_elems( source: B, @@ -602,7 +913,7 @@ pub fn from_suffix_with_elems( // SAFETY: The preceding `source.len().checked_sub(expected_len)` // guarantees that `split_at` is in-bounds. let (bytes, suffix) = unsafe { source.split_at_unchecked(split_at) }; - Self::from_bytes(suffix).map(move |l| (bytes, l)) + Self::from_bytes_with_elems(suffix, count).map(move |l| (bytes, l)) } } @@ -618,12 +929,37 @@ impl<'a, B, T> Ref /// Note: this is an associated function, which means that you have to call /// it as `Ref::into_ref(r)` instead of `r.into_ref()`. This is so that /// there is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_into_ref", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline(always)] pub fn into_ref(r: Self) -> &'a T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = r.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `IntoByteSlice`. let b = unsafe { r.into_byte_slice() }; @@ -642,12 +978,12 @@ pub fn into_ref(r: Self) -> &'a T { return ptr.as_ref(); } - // PANICS: By post-condition on `into_byte_slice`, `b`'s size and - // alignment are valid for `T`. By post-condition, `b.into_byte_slice()` - // produces a byte slice with identical address and length to that - // produced by `b.deref()`. + // PANICS: By post-condition on `into_byte_slice`, `b` is aligned for + // `T`, and its size is exactly the size encoded by `metadata`. By + // post-condition, `b.into_byte_slice()` produces a byte slice with + // identical address and length to that produced by `b.deref()`. let ptr = Ptr::from_ref(b.into_byte_slice()) - .try_cast_into_no_leftover::(None) + .try_cast_into_no_leftover::(Some(metadata)) .expect("zerocopy internal error: into_ref should be infallible"); let ptr = ptr.recall_validity(); ptr.as_ref() @@ -666,12 +1002,32 @@ impl<'a, B, T> Ref /// Note: this is an associated function, which means that you have to call /// it as `Ref::into_mut(r)` instead of `r.into_mut()`. This is so that /// there is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_into_mut", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] #[must_use = "has no side effects"] #[inline(always)] pub fn into_mut(r: Self) -> &'a mut T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = r.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `IntoByteSliceMut`. let b = unsafe { r.into_byte_slice_mut() }; @@ -686,7 +1042,7 @@ pub fn into_mut(r: Self) -> &'a mut T { T, _, (BecauseRead, BecauseExclusive), - (BecauseMutationCompatible, BecauseInvariantsEq), + (BecauseMutationCompatible, BecauseSharedCompatible), >(ptr) }; @@ -697,13 +1053,13 @@ pub fn into_mut(r: Self) -> &'a mut T { return ptr.as_mut(); } - // PANICS: By post-condition on `into_byte_slice_mut`, `b`'s size and - // alignment are valid for `T`. By post-condition, - // `b.into_byte_slice_mut()` produces a byte slice with identical - // address and length to that produced by `b.deref_mut()`. + // PANICS: By post-condition on `into_byte_slice_mut`, `b` is aligned + // for `T`, and its size is exactly the size encoded by `metadata`. By + // post-condition, `b.into_byte_slice_mut()` produces a byte slice with + // identical address and length to that produced by `b.deref_mut()`. let ptr = Ptr::from_mut(b.into_byte_slice_mut()) - .try_cast_into_no_leftover::(None) - .expect("zerocopy internal error: into_ref should be infallible"); + .try_cast_into_no_leftover::(Some(metadata)) + .expect("zerocopy internal error: into_mut should be infallible"); let ptr = ptr.recall_validity::<_, (_, (_, _))>(); ptr.as_mut() } @@ -719,6 +1075,29 @@ impl Ref /// Note: this is an associated function, which means that you have to call /// it as `Ref::bytes(r)` instead of `r.bytes()`. This is so that there is /// no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn bytes(r: &Self) -> &[u8] { // SAFETY: We don't call any methods on `b` other than those provided by @@ -737,6 +1116,29 @@ impl Ref /// Note: this is an associated function, which means that you have to call /// it as `Ref::bytes_mut(r)` instead of `r.bytes_mut()`. This is so that /// there is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_bytes_mut", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn bytes_mut(r: &mut Self) -> &mut [u8] { // SAFETY: We don't call any methods on `b` other than those provided by @@ -755,6 +1157,12 @@ impl Ref /// Note: this is an associated function, which means that you have to call /// it as `Ref::read(r)` instead of `r.read()`. This is so that there is no /// conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_read_static_size", + format = "coco_static_size", + )] #[must_use = "has no side effects"] #[inline] pub fn read(r: &Self) -> T { @@ -780,14 +1188,20 @@ impl Ref /// Note: this is an associated function, which means that you have to call /// it as `Ref::write(r, t)` instead of `r.write(t)`. This is so that there /// is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_write_static_size", + format = "coco_static_size", + )] #[inline] pub fn write(r: &mut Self, t: T) { // SAFETY: We don't call any methods on `b` other than those provided by // `ByteSliceMut`. let b = unsafe { r.as_byte_slice_mut() }; - // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is - // a valid size and alignment for `T`. By safety invariant on + // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is a + // valid size and alignment for `T`. By safety invariant on // `ByteSlice`, we know that this is preserved via `.deref()`. Writing // `t` to the buffer will allow all of the bytes of `t` to be accessed // as a `[u8]`, but because `T: IntoBytes`, we know that this is sound. @@ -801,11 +1215,38 @@ impl Deref for Ref T: FromBytes + KnownLayout + Immutable + ?Sized, { type Target = T; + + /// Dereferences the value. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_deref", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] fn deref(&self) -> &T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = self.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `ByteSlice`. let b = unsafe { self.as_byte_slice() }; @@ -824,11 +1265,12 @@ fn deref(&self) -> &T { return ptr.as_ref(); } - // PANICS: By postcondition on `as_byte_slice`, `b`'s size and alignment - // are valid for `T`, and by invariant on `ByteSlice`, these are - // preserved through `.deref()`, so this `unwrap` will not panic. + // PANICS: By postcondition on `as_byte_slice`, `b` is aligned for `T`, + // and its size is exactly the size encoded by `metadata`. By invariant + // on `ByteSlice`, these are preserved through `.deref()`, so this + // `unwrap` will not panic. let ptr = Ptr::from_ref(b) - .try_cast_into_no_leftover::(None) + .try_cast_into_no_leftover::(Some(metadata)) .expect("zerocopy internal error: Deref::deref should be infallible"); let ptr = ptr.recall_validity(); ptr.as_ref() @@ -843,11 +1285,32 @@ impl DerefMut for Ref // add a separate inherent method for this? T: FromBytes + IntoBytes + KnownLayout + Immutable + ?Sized, { + /// Mutably dereferences the value. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_deref_mut", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] #[inline] fn deref_mut(&mut self) -> &mut T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = self.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `ByteSliceMut`. let b = unsafe { self.as_byte_slice_mut() }; @@ -862,7 +1325,7 @@ fn deref_mut(&mut self) -> &mut T { T, _, (BecauseRead, BecauseExclusive), - (BecauseMutationCompatible, BecauseInvariantsEq), + (BecauseMutationCompatible, BecauseSharedCompatible), >(ptr) }; @@ -873,12 +1336,12 @@ fn deref_mut(&mut self) -> &mut T { return ptr.as_mut(); } - // PANICS: By postcondition on `as_byte_slice_mut`, `b`'s size and - // alignment are valid for `T`, and by invariant on `ByteSlice`, these - // are preserved through `.deref_mut()`, so this `unwrap` will not - // panic. + // PANICS: By postcondition on `as_byte_slice_mut`, `b` is aligned for + // `T`, and its size is exactly the size encoded by `metadata`. By + // invariant on `ByteSlice`, these are preserved through `.deref_mut()`, + // so this `unwrap` will not panic. let ptr = Ptr::from_mut(b) - .try_cast_into_no_leftover::(None) + .try_cast_into_no_leftover::(Some(metadata)) .expect("zerocopy internal error: DerefMut::deref_mut should be infallible"); let ptr = ptr.recall_validity::<_, (_, (_, BecauseExclusive))>(); ptr.as_mut() @@ -958,13 +1421,13 @@ fn partial_cmp(&self, other: &Self) -> Option { /// `T: Sized` and `ptr`'s referent must have size `size_of::()`. #[inline(always)] unsafe fn cast_for_sized<'a, T, A, R, S>( - ptr: Ptr<'a, [u8], (A, Aligned, Valid)>, -) -> Ptr<'a, T, (A, Unaligned, Valid)> + ptr: Ptr<'a, [u8], (A, Aligned, Safe)>, +) -> Ptr<'a, T, (A, Unaligned, Safe)> where T: FromBytes + KnownLayout + ?Sized, A: crate::invariant::Aliasing, [u8]: MutationCompatible, - T: TransmuteFromPtr, + T: TransmuteFromPtr, { use crate::pointer::cast::{Cast, Project}; @@ -990,7 +1453,7 @@ unsafe impl Cast<[u8], T> for CastForSized {} ptr.recall_validity::() .cast::<_, CastForSized, _>() - .recall_validity::() + .recall_validity::() } #[cfg(test)] @@ -1028,6 +1491,66 @@ fn test_address() { assert_eq!(buf_ptr, deref_ptr); } + #[test] + #[allow(clippy::clone_on_copy)] // Exercise `Ref`'s manual `Clone` impl. + fn test_with_elems_preserves_ambiguous_metadata() { + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C, align(8))] + struct Frame { + header: [u8; 8], + body: [u8], + } + + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C, align(8))] + struct OddHeader { + header: [u8; 7], + body: [u8], + } + + // Dynamic trailing padding makes these pairs of distinct element + // counts produce the same total byte size. Thus, the byte length alone + // cannot recover the count supplied to a `*_with_elems` constructor. + assert_eq!(Frame::size_for_metadata(1), Some(16)); + assert_eq!(Frame::size_for_metadata(8), Some(16)); + assert_eq!(OddHeader::size_for_metadata(0), Some(8)); + assert_eq!(OddHeader::size_for_metadata(1), Some(8)); + + let bytes = Align::<[u8; 32], AU64>::default(); + + let whole = Ref::<_, Frame>::from_bytes_with_elems(&bytes.t[..16], 1).unwrap(); + assert_eq!(whole.body.len(), 1); + assert_eq!(Ref::bytes(&whole).len(), 16); + assert_eq!(whole.clone().body.len(), 1); + let copied = whole; + assert_eq!(copied.body.len(), 1); + assert_eq!(Ref::into_ref(whole).body.len(), 1); + + let (prefix, rest) = Ref::<_, Frame>::from_prefix_with_elems(&bytes.t[..], 1).unwrap(); + assert_eq!(prefix.body.len(), 1); + assert_eq!(rest.len(), 16); + + let (before, suffix) = Ref::<_, Frame>::from_suffix_with_elems(&bytes.t[..24], 1).unwrap(); + assert_eq!(before.len(), 8); + assert_eq!(suffix.body.len(), 1); + + // Zero is real metadata, not a sentinel for "infer from the bytes". + let empty = Ref::<_, OddHeader>::from_bytes_with_elems(&bytes.t[..8], 0).unwrap(); + assert!(empty.body.is_empty()); + assert!(Ref::into_ref(empty).body.is_empty()); + + // The implicit constructors still choose the largest fitting count, + // and retain the metadata selected during their validating cast. + let inferred = Ref::<_, Frame>::from_bytes(&bytes.t[..16]).unwrap(); + assert_eq!(inferred.body.len(), 8); + let (inferred, rest) = Ref::<_, Frame>::from_prefix(&bytes.t[..16]).unwrap(); + assert_eq!(inferred.body.len(), 8); + assert!(rest.is_empty()); + let (before, inferred) = Ref::<_, Frame>::from_suffix(&bytes.t[..16]).unwrap(); + assert!(before.is_empty()); + assert_eq!(inferred.body.len(), 8); + } + // Verify that values written to a `Ref` are properly shared between the // typed and untyped representations, that reads via `deref` and `read` // behave the same, and that writes via `deref_mut` and `write` behave the @@ -1251,6 +1774,13 @@ fn test_into_ref_mut() { *rf = u64::MAX; assert_eq!(buf.t, [0xFF; 8]); + + // Exercise the unsized `into_mut` route, which must use the metadata + // retained by the constructor rather than infer fresh metadata. + let mut buf = Align::<[u8; 8], u16>::default(); + let r = Ref::<_, [u16]>::from_bytes_with_elems(&mut buf.t[..], 4).unwrap(); + let rf = Ref::into_mut(r); + assert_eq!(rf.len(), 4); } #[test] @@ -1325,8 +1855,13 @@ fn bench_into_mut_sized(b: &mut Bencher) { let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); b.iter(move || { // SAFETY: The preceding `from_bytes` succeeded, and so we know that - // `buf` is validly-aligned and has the correct length. - let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + // `buf` is validly aligned and has the `size_of::()` byte + // length required by `new_sized_unchecked` [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::<&mut [u8], AU64>::new_sized_unchecked(&mut *buf) }; test::black_box(Ref::into_mut(test::black_box(r))); }); } @@ -1349,8 +1884,13 @@ fn bench_deref_mut_sized(b: &mut Bencher) { let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); b.iter(|| { // SAFETY: The preceding `from_bytes` succeeded, and so we know that - // `buf` is validly-aligned and has the correct length. - let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + // `buf` is validly aligned and has the `size_of::()` byte + // length required by `new_sized_unchecked` [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::<&mut [u8], AU64>::new_sized_unchecked(&mut *buf) }; let mut temp = test::black_box(r); test::black_box(temp.deref_mut()); }); diff --git a/rust/zerocopy/src/split_at.rs b/rust/zerocopy/src/split_at.rs index d7778425a31d..4aeb602f69e0 100644 --- a/rust/zerocopy/src/split_at.rs +++ b/rust/zerocopy/src/split_at.rs @@ -10,13 +10,19 @@ // those terms. use super::*; -use crate::pointer::invariant::{Aligned, Exclusive, Invariants, Shared, Valid}; +use crate::pointer::invariant::{Aligned, Exclusive, Invariants, Safe, Shared}; /// Types that can be split in two. /// /// This trait generalizes Rust's existing support for splitting slices to /// support slices and slice-based dynamically-sized types ("slice DSTs"). /// +/// For a `repr(C, packed)` or `repr(C, packed(N))` struct, the derive wraps +/// the trailing field's [`Elem`](Self::Elem) in [`Unalign`]. The returned +/// slice can therefore be accessed even when packing misaligns the original +/// elements. Each packed struct in the trailing-field chain adds one wrapper; +/// unpacked structs and [`ManuallyDrop`] forward their trailing type's `Elem`. +/// /// # Implementation /// /// **Do not implement this trait yourself!** Instead, use @@ -38,6 +44,28 @@ /// This derive performs a sophisticated, compile-time safety analysis to /// determine whether a type is `SplitAt`. /// +/// A packed struct exposes its trailing elements through `Unalign`: +/// +/// ``` +/// use zerocopy::{FromBytes, SplitAt, Unalign}; +/// # use zerocopy_derive::*; +/// +/// #[derive(FromBytes, IntoBytes, KnownLayout, SplitAt)] +/// #[repr(C, packed)] +/// struct Packet { +/// tag: u8, +/// words: [u16], +/// } +/// +/// let mut bytes = [0u8; 5]; +/// let packet = Packet::mut_from_bytes(&mut bytes[..]).unwrap(); +/// let (left, right): (&mut Packet, &mut [Unalign]) = +/// packet.split_at_mut(1).unwrap().via_into_bytes(); +/// left.tag = 1; +/// right[0] = Unalign::new(0x0202); +/// assert_eq!(bytes, [1, 0, 0, 2, 2]); +/// ``` +/// /// # Safety /// /// This trait does not convey any safety guarantees to code outside this crate. @@ -58,10 +86,20 @@ )] // # Safety // -// The trailing slice is well-aligned for its element type. `Self` is `[T]`, or -// a `repr(C)` or `repr(transparent)` slice DST. +// `Self` is a slice, a `repr(C)` (possibly packed) or `repr(transparent)` slice +// DST, or `ManuallyDrop` for some `T: SplitAt`. +// +// `Self::Elem` has the same size, bit validity, and `UnsafeCell` coverage as +// the actual trailing element type. Access through either representation +// preserves the other's validity, including writes through mutable references +// and interior mutation through shared references. For every aligned `Self`, +// its trailing slice's address is aligned for `Self::Elem`. pub unsafe trait SplitAt: KnownLayout { - /// The element type of the trailing slice. + /// The element type exposed by the split's returned slice. + /// + /// For `[T]`, this is `T`. For a struct, this is the trailing field's `Elem`, + /// wrapped in [`Unalign`] if the struct is packed. Nested packed structs + /// produce nested `Unalign` wrappers. type Elem; #[doc(hidden)] @@ -188,10 +226,7 @@ unsafe fn split_at_mut_unchecked(&mut self, l_len: usize) -> Split<&mut Self> { /// Attempts to split `self` in two. /// /// Returns `None` if `l_len` is greater than the length of `self`'s - /// trailing slice, or if the given `l_len` would result in [the trailing - /// padding](KnownLayout#slice-dst-layout) of the left portion overlapping - /// the right portion. - /// + /// trailing slice. /// /// # Examples /// @@ -249,7 +284,8 @@ fn split_at_mut(&mut self, l_len: usize) -> Option> { } } -// SAFETY: `[T]`'s trailing slice is `[T]`, which is trivially aligned. +// SAFETY: `[T]` exposes its own element type, preserving size, validity, and +// `UnsafeCell` coverage. An aligned slice is aligned for its elements. unsafe impl SplitAt for [T] { type Elem = T; @@ -262,6 +298,29 @@ fn only_derive_is_allowed_to_implement_this_trait() } } +// SAFETY: `ManuallyDrop` has the same layout and bit validity as `T` [1]. +// Its `KnownLayout` implementation preserves `T`'s metadata and trailing-slice +// layout. Forwarding `T::Elem` therefore preserves its size, validity, and +// alignment guarantees. Its `UnsafeCell` coverage also matches `T`'s, as +// established for `ManuallyDrop` in `impls.rs`; accesses through the forwarded +// element representation preserve validity in both directions. +// +// [1] Per https://doc.rust-lang.org/1.93.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop` is guaranteed to have the same layout and bit validity as +// `T` +unsafe impl SplitAt for ManuallyDrop { + type Elem = T::Elem; + + #[inline] + #[allow(dead_code)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } +} + /// A `T` that has been split into two possibly-overlapping parts. /// /// For some dynamically sized types, the padding that appears after the @@ -270,10 +329,11 @@ fn only_derive_is_allowed_to_implement_this_trait() /// requires trailing padding, the trailing padding of the left part of the /// split `T` will overlap the right part. If `T` is a mutable reference or /// permits interior mutation, you must ensure that the left and right parts do -/// not overlap. You can do this at zero-cost using using -/// [`Self::via_immutable`], [`Self::via_into_bytes`], or -/// [`Self::via_unaligned`], or with a dynamic check by using -/// [`Self::via_runtime_check`]. +/// not overlap. Use [`Self::via_into_bytes`] or +/// [`Self::via_no_dynamic_padding`] to establish this without a runtime check, +/// or [`Self::via_runtime_check`] to check a particular split. Shared references +/// to an [`Immutable`] type may overlap; use [`Self::via_immutable`] in that +/// case. #[derive(Debug)] pub struct Split { /// A pointer to the source slice DST. @@ -304,7 +364,7 @@ impl<'a, T> Split<&'a T> T: ?Sized + SplitAt, { #[inline(always)] - fn into_ptr(self) -> Split> { + fn into_ptr(self) -> Split> { let source = Ptr::from_ref(self.source); // SAFETY: `Ptr::from_ref(self.source)` points to exactly `self.source` // and thus maintains the invariants of `self` with respect to `l_len`. @@ -423,59 +483,100 @@ pub fn via_into_bytes(self) -> (&'a T, &'a [T::Elem]) (l.as_ref(), r.as_ref()) } - /// Produces the split parts of `self`, using [`Unaligned`] to ensure that - /// it is sound to have concurrent references to both parts. - /// - /// # Examples - /// - /// ``` - /// use zerocopy::{SplitAt, FromBytes}; - /// # use zerocopy_derive::*; - /// - /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Unaligned)] - /// #[repr(C)] - /// struct Packet { - /// length: u8, - /// body: [u8], - /// } - /// - /// // These bytes encode a `Packet`. - /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; - /// - /// let packet = Packet::ref_from_bytes(bytes).unwrap(); - /// - /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); - /// - /// // Attempt to split `packet` at `length`. - /// let split = packet.split_at(packet.length as usize).unwrap(); - /// - /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to - /// // return concurrent references to `packet` and `rest`. - /// let (packet, rest) = split.via_unaligned(); - /// - /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [1, 2, 3, 4]); - /// assert_eq!(rest, [5, 6, 7, 8, 9]); - /// ``` - /// - #[doc = codegen_header!("h5", "split_via_unaligned")] - /// - /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + /// Produces the split parts of `self` using [`Self::via_no_dynamic_padding`]. + #[deprecated(note = "use `Split::via_no_dynamic_padding` instead")] + #[doc(hidden)] #[must_use = "has no side effects"] #[inline(always)] pub fn via_unaligned(self) -> (&'a T, &'a [T::Elem]) where T: Unaligned, { - let (l, r) = self.into_ptr().via_unaligned(); + self.via_no_dynamic_padding() + } + + /// Produces the split parts of `self` using a compile-time assertion to + /// ensure that it is unconditionally sound to have concurrent references to + /// both parts. + /// + /// When possible, you should prefer using [`Self::via_into_bytes`], which + /// ensures this safety property through regular trait bounds. + /// + /// # Examples + /// + /// In the below example, a single byte of padding exists (on most + /// platforms) between `Packet<[u16]>`'s `length` and `body` fields, + /// precluding the use of [`Self::via_into_bytes`]. + /// + /// ``` + /// use zerocopy::SplitAt; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, KnownLayout)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: B, + /// } + /// + /// let packet: &Packet<[u16]> = &Packet { length: 4, body: [1u16, 2, 3, 4, 5, 6, 7, 8, 9] }; + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// if let Some(split) = packet.split_at(packet.length as usize) { + /// // Every body length requires no trailing padding. + /// let (packet, rest) = split.via_no_dynamic_padding(); + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// } else { + /// unreachable!("The packet's length field is within its body"); + /// } + /// ``` + /// + /// # Compile-Time Assertions + /// + /// This method rejects at compile-time any type that admits dynamic + /// trailing padding. In the below example, some `split_at` indices would + /// produce left parts whose trailing padding overlaps with the right part: + /// + /// ```compile_fail,E0080 + /// use zerocopy::SplitAt; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, KnownLayout)] + /// #[repr(C, align(2))] + /// struct Packet { + /// length: u16, + /// body: B, + /// } + /// + /// let packet: &Packet<[u8]> = &Packet { length: 4, body: [1u8, 2, 3, 4, 5, 6, 7, 8, 9] }; + /// + /// if let Some(split) = packet.split_at(packet.length as usize) { + /// let _ = split.via_no_dynamic_padding(); // ⚠ Compile Error! + /// } else { + /// unreachable!("The packet's length field is within its body"); + /// } + /// ``` + /// + /// If you need to split such types, use [`Self::via_runtime_check`]. + /// + #[doc = codegen_header!("h5", "split_via_no_dynamic_padding")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_no_dynamic_padding(self) -> (&'a T, &'a [T::Elem]) { + let (l, r) = self.into_ptr().via_no_dynamic_padding(); (l.as_ref(), r.as_ref()) } /// Produces the split parts of `self`, using a dynamic check to ensure that /// it is sound to have concurrent references to both parts. You should /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or - /// [`Self::via_unaligned`], which have no runtime cost. + /// [`Self::via_no_dynamic_padding`], which perform no runtime check. /// /// Note that this check is overly conservative if `T` is [`Immutable`]; for /// some types, this check will reject some splits which @@ -484,52 +585,40 @@ pub fn via_unaligned(self) -> (&'a T, &'a [T::Elem]) /// # Examples /// /// ``` - /// use zerocopy::{SplitAt, FromBytes, IntoBytes, network_endian::U16}; + /// use zerocopy::SplitAt; /// # use zerocopy_derive::*; /// - /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Debug)] + /// #[derive(SplitAt, KnownLayout)] /// #[repr(C, align(2))] - /// struct Packet { - /// length: U16, - /// body: [u8], + /// struct Packet { + /// length: u16, + /// body: B, /// } /// - /// // These bytes encode a `Packet`. - /// let bytes = [ - /// 4u16.to_be(), - /// 1u16.to_be(), - /// 2u16.to_be(), - /// 3u16.to_be(), - /// 4u16.to_be() - /// ]; - /// - /// let packet = Packet::ref_from_bytes(bytes.as_bytes()).unwrap(); + /// let packet: &Packet<[u8]> = &Packet { length: 4, body: [1u8, 2, 3, 4, 5, 6, 7, 8, 9] }; /// /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [0, 1, 0, 2, 0, 3, 0, 4]); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); /// - /// // Attempt to split `packet` at `length`. - /// let split = packet.split_at(packet.length.into()).unwrap(); + /// // The two-byte prefix and four-byte body need no trailing padding. + /// if let Some(split) = packet.split_at(packet.length as usize) { + /// if let Ok((packet, rest)) = split.via_runtime_check() { + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// } else { + /// unreachable!("A four-byte body requires no trailing padding"); + /// } + /// } else { + /// unreachable!("The packet's length field is within its body"); + /// } /// - /// // Use a dynamic check to prove that it's okay to return concurrent - /// // references to `packet` and `rest`. - /// let (packet, rest) = split.via_runtime_check().unwrap(); - /// - /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [0, 1, 0, 2]); - /// assert_eq!(rest, [0, 3, 0, 4]); - /// - /// // Attempt to split `packet` at `length - 1`. - /// let idx = packet.length.get() - 1; - /// let split = packet.split_at(idx as usize).unwrap(); - /// - /// // Attempt (and fail) to use a dynamic check to prove that it's okay - /// // to return concurrent references to `packet` and `rest`. Note that - /// // this is a case of `via_runtime_check` being overly conservative. - /// // Although the left and right parts indeed overlap, the `Immutable` - /// // bound ensures that concurrently referencing these overlapping - /// // parts is sound. - /// assert!(split.via_runtime_check().is_err()); + /// // A three-byte body needs a padding byte that would overlap `rest`. + /// if let Some(split) = packet.split_at((packet.length - 1) as usize) { + /// assert!(split.via_runtime_check().is_err()); + /// } else { + /// unreachable!("One less than the packet's length is within its body"); + /// } /// ``` /// #[doc = codegen_section!( @@ -602,7 +691,7 @@ impl<'a, T> Split<&'a mut T> T: ?Sized + SplitAt, { #[inline(always)] - fn into_ptr(self) -> Split> { + fn into_ptr(self) -> Split> { let source = Ptr::from_mut(self.source); // SAFETY: `Ptr::from_mut(self.source)` points to exactly `self.source`, // and thus maintains the invariants of `self` with respect to `l_len`. @@ -665,101 +754,135 @@ pub fn via_into_bytes(self) -> (&'a mut T, &'a mut [T::Elem]) (l.as_mut(), r.as_mut()) } - /// Produces the split parts of `self`, using [`Unaligned`] to ensure that - /// it is sound to have concurrent references to both parts. - /// - /// # Examples - /// - /// ``` - /// use zerocopy::{SplitAt, FromBytes}; - /// # use zerocopy_derive::*; - /// - /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Unaligned)] - /// #[repr(C)] - /// struct Packet { - /// length: u8, - /// body: B, - /// } - /// - /// // These bytes encode a `Packet`. - /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; - /// - /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); - /// - /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); - /// - /// { - /// // Attempt to split `packet` at `length`. - /// let split = packet.split_at_mut(packet.length as usize).unwrap(); - /// - /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to - /// // return concurrent references to `packet` and `rest`. - /// let (packet, rest) = split.via_unaligned(); - /// - /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [1, 2, 3, 4]); - /// assert_eq!(rest, [5, 6, 7, 8, 9]); - /// - /// rest.fill(0); - /// } - /// - /// assert_eq!(packet.length, 4); - /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); - /// ``` - /// - /// # Code Generation - /// - /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + /// Produces the split parts of `self` using [`Self::via_no_dynamic_padding`]. + #[deprecated(note = "use `Split::via_no_dynamic_padding` instead")] + #[doc(hidden)] #[must_use = "has no side effects"] #[inline(always)] pub fn via_unaligned(self) -> (&'a mut T, &'a mut [T::Elem]) where T: Unaligned, { - let (l, r) = self.into_ptr().via_unaligned(); - (l.as_mut(), r.as_mut()) + self.via_no_dynamic_padding() } - /// Produces the split parts of `self`, using a dynamic check to ensure that - /// it is sound to have concurrent references to both parts. You should - /// prefer using [`Self::via_into_bytes`] or [`Self::via_unaligned`], which - /// have no runtime cost. + /// Produces the split parts of `self` using a compile-time assertion to + /// ensure that it is unconditionally sound to have concurrent references to + /// both parts. + /// + /// When possible, you should prefer using [`Self::via_into_bytes`], which + /// ensures this safety property through regular trait bounds. /// /// # Examples /// + /// In the below example, a single byte of padding exists (on most + /// platforms) between `Packet<[u16]>`'s `length` and `body` fields, + /// precluding the use of [`Self::via_into_bytes`]. + /// /// ``` - /// use zerocopy::{SplitAt, FromBytes}; + /// use zerocopy::SplitAt; /// # use zerocopy_derive::*; /// - /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Debug)] + /// #[derive(SplitAt, KnownLayout)] /// #[repr(C)] /// struct Packet { /// length: u8, /// body: B, /// } /// - /// // These bytes encode a `Packet`. - /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; - /// - /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// let packet: &mut Packet<[u16]> = &mut Packet { length: 4, body: [1u16, 2, 3, 4, 5, 6, 7, 8, 9] }; /// /// assert_eq!(packet.length, 4); /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); /// - /// { - /// // Attempt to split `packet` at `length`. - /// let split = packet.split_at_mut(packet.length as usize).unwrap(); - /// - /// // Use a dynamic check to prove that it's okay to return concurrent - /// // references to `packet` and `rest`. - /// let (packet, rest) = split.via_runtime_check().unwrap(); - /// + /// // Every body length requires no trailing padding. + /// if let Some(split) = packet.split_at_mut(packet.length as usize) { + /// let (packet, rest) = split.via_no_dynamic_padding(); /// assert_eq!(packet.length, 4); /// assert_eq!(packet.body, [1, 2, 3, 4]); /// assert_eq!(rest, [5, 6, 7, 8, 9]); - /// /// rest.fill(0); + /// } else { + /// unreachable!("The packet's length field is within its body"); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Compile-Time Assertions + /// + /// This method rejects at compile-time any type that admits dynamic + /// trailing padding. In the below example, some `split_at` indices would + /// produce left parts whose trailing padding overlaps with the right part: + /// + /// ```compile_fail,E0080 + /// use zerocopy::SplitAt; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, KnownLayout)] + /// #[repr(C, align(2))] + /// struct Packet { + /// length: u16, + /// body: B, + /// } + /// + /// let packet: &mut Packet<[u8]> = &mut Packet { length: 4, body: [1u8, 2, 3, 4, 5, 6, 7, 8, 9] }; + /// + /// if let Some(split) = packet.split_at_mut(packet.length as usize) { + /// let _ = split.via_no_dynamic_padding(); // ⚠ Compile Error! + /// } else { + /// unreachable!("The packet's length field is within its body"); + /// } + /// ``` + /// + /// If you need to split such types, use [`Self::via_runtime_check`]. + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_no_dynamic_padding(self) -> (&'a mut T, &'a mut [T::Elem]) { + let (l, r) = self.into_ptr().via_no_dynamic_padding(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_into_bytes`] or + /// [`Self::via_no_dynamic_padding`], which perform no runtime check. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::SplitAt; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, KnownLayout)] + /// #[repr(C, align(2))] + /// struct Packet { + /// length: u16, + /// body: B, + /// } + /// + /// let packet: &mut Packet<[u8]> = &mut Packet { length: 4, body: [1u8, 2, 3, 4, 5, 6, 7, 8, 9] }; + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // The two-byte prefix and four-byte body need no trailing padding. + /// if let Some(split) = packet.split_at_mut(packet.length as usize) { + /// if let Ok((packet, rest)) = split.via_runtime_check() { + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// rest.fill(0); + /// } else { + /// unreachable!("A four-byte body requires no trailing padding"); + /// } + /// } else { + /// unreachable!("The packet's length field is within its body"); /// } /// /// assert_eq!(packet.length, 4); @@ -805,7 +928,7 @@ pub unsafe fn via_unchecked(self) -> (&'a mut T, &'a mut [T::Elem]) { impl<'a, T, I> Split> where T: ?Sized + SplitAt, - I: Invariants, + I: Invariants, { fn into_ref(self) -> Split<&'a T> where @@ -860,37 +983,57 @@ fn via_into_bytes(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) unsafe { self.via_unchecked() } } - /// Produces the split parts of `self`, using [`Unaligned`] to ensure that - /// it is sound to have concurrent references to both parts. + /// Produces the split parts of `self` after the conservative layout + /// predicate establishes that `T` never requires dynamic trailing padding. #[inline(always)] - fn via_unaligned(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) - where - T: Unaligned, - { - // SAFETY: By `T: SplitAt + Unaligned`, `T` is either a slice or a - // `repr(C)` or `repr(transparent)` slice DST that is well-aligned at - // any address and length. If `T` is a slice DST with alignment 1, - // `repr(C)` or `repr(transparent)` ensures that no padding is placed - // after the final element of the trailing slice. Consequently, `T` can - // be split into strictly non-overlapping parts any any index. + fn via_no_dynamic_padding(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { + static_assert!( + T: ?Sized + KnownLayout => !T::LAYOUT.requires_dynamic_padding(), + "`Split::via_no_dynamic_padding` cannot be used with a type whose layout may require dynamic trailing padding; use `Split::via_runtime_check` instead" + ); + + // SAFETY: The assertion establishes that `requires_dynamic_padding()` + // is false. By that method's guarantee, every valid metadata—including + // `self.l_len()`, which is valid by `Split`'s invariant—describes an + // object with no trailing padding. By the postcondition of + // `PtrInner::split_at_unchecked`, the two returned referents are then + // contiguous and non-overlapping. Since the source already conforms to + // `I::Aliasing`, splitting it into disjoint byte ranges preserves that + // invariant: under `Exclusive`, neither result aliases or accesses the + // other's bytes; under `Shared`, interior mutation through one result + // cannot mutate bytes pointed to by the other. This satisfies Rust's + // reference aliasing rules [1]. + // + // [1] Per https://doc.rust-lang.org/1.93.1/std/ptr/index.html#pointer-to-reference-conversion: + // + // When creating a mutable reference, then while this reference + // exists, the memory it points to must not get accessed (read or + // written) through any other pointer or reference not derived from + // this reference. + // + // When creating a shared reference, then while this reference + // exists, the memory it points to must not get mutated (except inside + // `UnsafeCell`). unsafe { self.via_unchecked() } } /// Produces the split parts of `self`, using a dynamic check to ensure that /// it is sound to have concurrent references to both parts. You should /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or - /// [`Self::via_unaligned`], which have no runtime cost. + /// [`Self::via_no_dynamic_padding`], which perform no runtime check. #[inline(always)] fn via_runtime_check(self) -> Result<(Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>), Self> { let l_len = self.l_len(); + // `l_len` witnesses an object size at most `isize::MAX`, and + // `KnownLayout::LAYOUT` guarantees that it contains the trailing + // slice. Thus `padding_for_elems` returns the exact trailing padding. + let trailing_padding = crate::trailing_slice_layout::().padding_for_elems(l_len.get()); // FIXME(#1290): Once we require `KnownLayout` on all fields, add an // `IS_IMMUTABLE` associated const, and add `T::IS_IMMUTABLE ||` to the // below check. - if l_len.padding_needed_for() == 0 { - // SAFETY: By `T: SplitAt`, `T` is either `[T]`, or a `repr(C)` or - // `repr(transparent)` slice DST, for which the trailing padding - // needed to accommodate `l_len` trailing elements is - // `l_len.padding_needed_for()`. If no trailing padding is required, + if trailing_padding == 0 { + // SAFETY: As established above, `trailing_padding` is the exact + // padding after the left part's trailing slice. If it is zero, // the left and right parts are strictly non-overlapping. Ok(unsafe { self.via_unchecked() }) } else { @@ -903,7 +1046,7 @@ fn via_runtime_check(self) -> Result<(Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>), Sel /// # Safety /// /// The caller promises that if `I::Aliasing` is [`Exclusive`] or `T` - /// permits interior mutation, then `l_len.padding_needed_for() == 0`. + /// permits interior mutation, then the left part has no trailing padding. #[inline(always)] unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { let l_len = self.l_len(); @@ -915,11 +1058,11 @@ unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { // Lemma 0: `left` and `right` conform to the aliasing invariant // `I::Aliasing`. Proof: If `I::Aliasing` is `Exclusive` or `T` permits - // interior mutation, the caller promises that `l_len.padding_needed_for() - // == 0`. Consequently, by post-condition on `PtrInner::split_at_unchecked`, - // there is no trailing padding after `left`'s final element that would - // overlap into `right`. If `I::Aliasing` is shared and `T` forbids interior - // mutation, then overlap between their referents is permissible. + // interior mutation, the caller promises that the left part has no + // trailing padding. Consequently, by the postcondition of + // `PtrInner::split_at_unchecked`, `left` and `right` do not overlap. + // If `I::Aliasing` is shared and `T` forbids interior mutation, then + // overlap between their referents is permissible. // SAFETY: // 0. `left` conforms to the aliasing invariant of `I::Aliasing`, by Lemma 0. @@ -933,17 +1076,19 @@ unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { // SAFETY: // 0. `right` conforms to the aliasing invariant of `I::Aliasing`, by Lemma // 0. - // 1. `right` conforms to the alignment invariant of `I::Alignment, because - // if `ptr` with `I::Alignment = Aligned`, then by invariant on `T: - // SplitAt`, the trailing slice of `ptr` (from which `right` is derived) - // will also be well-aligned. + // 1. `right` conforms to the alignment invariant of `I::Alignment`: + // if the source is aligned, `T: SplitAt` guarantees that its + // trailing slice's address is aligned for `T::Elem`. Advancing by + // whole elements preserves that alignment. // 2. `right` conforms to the validity invariant of `I::Validity`, - // because `right: [T::Elem]` is derived from the trailing slice of - // `ptr`, which, by contract on `T: SplitAt::Elem`, has type - // `[T::Elem]`. The `left` part cannot be used to invalidate `right`, - // because the caller promises that if `I::Aliasing` is `Exclusive` - // or `T` permits interior mutation, then `l_len.padding_needed_for() - // == 0` and thus the parts will be non-overlapping. + // because `T: SplitAt` guarantees that `T::Elem` has the size and + // bit validity of the original trailing element. It also guarantees + // compatible shared and mutable access, so writes through `right` + // preserve the original elements' validity and interior mutation + // respects the original `UnsafeCell` coverage. The left part + // cannot invalidate `right`: when the references are exclusive or + // permit interior mutation, the caller guarantees no trailing + // padding and hence no overlap. let right = unsafe { Ptr::from_inner(right) }; (left, right) @@ -952,6 +1097,185 @@ unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { #[cfg(test)] mod tests { + use core::{cell::Cell, mem::ManuallyDrop}; + + use crate::{Immutable, KnownLayout, SplitAt, Unalign, Unaligned}; + + #[derive(KnownLayout, SplitAt, Immutable, Unaligned)] + #[repr(C, packed)] + struct Packed { + prefix: u8, + tail: ManuallyDrop, + } + + #[derive(KnownLayout, SplitAt, Immutable)] + #[repr(C, packed(2))] + struct Packed2 { + prefix: u8, + tail: ManuallyDrop, + } + + #[derive(KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct Inner { + prefix: u32, + tail: T, + } + + #[test] + fn test_split_at_packed() { + use crate::{FromBytes, IntoBytes}; + + #[derive(FromBytes, IntoBytes, KnownLayout, SplitAt, Immutable, Unaligned)] + #[repr(C, packed)] + struct Packet { + prefix: u8, + tail: [u16], + } + + // The tail starts at an odd address, exercising unaligned reads and + // writes on targets where `u16` has alignment 2. + #[repr(align(8))] + struct Bytes([u8; 9]); + + for i in 0..=4 { + let mut bytes = Bytes([1; 9]); + let packet = Packet::ref_from_bytes(&bytes.0).unwrap(); + let (left, right): (&Packet, &[Unalign]) = + packet.split_at(i).unwrap().via_immutable(); + assert_eq!(core::mem::size_of_val(left), 1 + 2 * i); + assert_eq!(right.len(), 4 - i); + assert!(right.iter().all(|elem| elem.get() == 0x0101)); + assert!(left.split_at(i).is_some()); + assert!(left.split_at(i + 1).is_none()); + assert!(packet.split_at(5).is_none()); + + let (_, right) = packet.split_at(i).unwrap().via_into_bytes(); + assert_eq!(right.len(), 4 - i); + let (_, right) = packet.split_at(i).unwrap().via_no_dynamic_padding(); + assert_eq!(right.len(), 4 - i); + + let packet = Packet::mut_from_bytes(&mut bytes.0).unwrap(); + let (left, right): (&mut Packet, &mut [Unalign]) = + packet.split_at_mut(i).unwrap().via_runtime_check().ok().unwrap(); + left.prefix = 2; + for elem in right { + *elem = Unalign::new(0x0202); + } + assert_eq!(bytes.0[0], 2); + assert!(bytes.0[1..1 + 2 * i].iter().all(|&byte| byte == 1)); + assert!(bytes.0[1 + 2 * i..].iter().all(|&byte| byte == 2)); + } + } + + #[test] + fn test_split_at_nested_packed() { + // A packing factor greater than 1 still exposes `Unalign` elements, + // including when the factor is below the original element alignment. + let mut words = Packed2 { prefix: 0, tail: ManuallyDrop::new([1u32, 2, 3]) }; + let dst: &mut Packed2<[u32]> = &mut words; + let (left, right): (&mut _, &mut [Unalign]) = + dst.split_at_mut(1).unwrap().via_no_dynamic_padding(); + left.prefix = 4; + right[0] = Unalign::new(5); + let dst: &Packed2<[u32]> = &words; + let (left, right) = dst.split_at(0).unwrap().via_immutable(); + assert_eq!(left.prefix, 4); + assert_eq!(right[0].get(), 1); + assert_eq!(right[1].get(), 5); + assert_eq!(right[2].get(), 3); + + let mut packet = Inner { + prefix: 0, + tail: Packed2 { + prefix: 1, + tail: ManuallyDrop::new(Packed { + prefix: 2, + tail: ManuallyDrop::new([3u16, 4, 5]), + }), + }, + }; + let dst: &Inner>> = &packet; + let (left, right): (&_, &[Unalign>]) = + dst.split_at(1).unwrap().via_immutable(); + assert_eq!(left.prefix, 0); + assert_eq!(right.len(), 2); + assert_eq!(right[0].get().get(), 4); + assert_eq!(right[1].get().get(), 5); + + // Split the packed field separately: the unpacked outer struct's + // padding does not constrain this split. + let dst: &mut Packed2> = &mut packet.tail; + let (left, right): (&mut _, &mut [Unalign>]) = + dst.split_at_mut(1).unwrap().via_no_dynamic_padding(); + left.prefix = 6; + right[0] = Unalign::new(Unalign::new(7)); + let dst: &Inner>> = &packet; + let (_, right) = dst.split_at(0).unwrap().via_immutable(); + assert_eq!(right[0].get().get(), 3); + assert_eq!(right[1].get().get(), 7); + assert_eq!(right[2].get().get(), 5); + } + + #[test] + fn test_split_at_packed_preserves_inner_padding() { + for i in 0..=4 { + let mut packet = Packed { + prefix: 1, + tail: ManuallyDrop::new(Inner { prefix: 0, tail: [2u8, 3, 4, 5] }), + }; + let dst: &Packed> = &packet; + let (left, right): (&_, &[Unalign]) = dst.split_at(i).unwrap().via_immutable(); + assert_eq!(right.len(), 4 - i); + assert_eq!(right.first().map(Unalign::get), [2, 3, 4, 5].get(i).copied()); + + // The inner `u32` prefix makes the tail start at byte 5 of the + // packed wrapper. Its size may still include inner padding. + let has_padding = core::mem::size_of_val(left) != 5 + i; + assert_eq!(dst.split_at(i).unwrap().via_runtime_check().is_err(), has_padding); + + let dst: &mut Packed> = &mut packet; + let split = dst.split_at_mut(i).unwrap().via_runtime_check(); + assert_eq!(split.is_err(), has_padding); + if let Ok((left, right)) = split { + left.prefix = 6; + for elem in right { + *elem = Unalign::new(7); + } + let dst: &Packed> = &packet; + let (left, right) = dst.split_at(i).unwrap().via_immutable(); + assert_eq!(left.prefix, 6); + assert!(right.iter().all(|elem| elem.get() == 7)); + } + } + } + + #[test] + fn test_split_at_packed_interior_mutation_and_zsts() { + let packet = Packed { + prefix: 0, + tail: ManuallyDrop::new([Cell::new(1u8), Cell::new(2), Cell::new(3)]), + }; + let dst: &Packed<[Cell]> = &packet; + let (left, right): (&_, &[Unalign>]) = + dst.split_at(1).unwrap().via_no_dynamic_padding(); + left.tail[0].set(4); + right[0].try_deref().unwrap().set(5); + assert_eq!(dst.tail[0].get(), 4); + assert_eq!(dst.tail[1].get(), 5); + assert_eq!(dst.tail[2].get(), 3); + + let mut packet = Packed { prefix: 0, tail: ManuallyDrop::new([(); 3]) }; + for i in 0..=3 { + let dst: &mut Packed<[()]> = &mut packet; + let (left, right): (&mut _, &mut [Unalign<()>]) = + dst.split_at_mut(i).unwrap().via_no_dynamic_padding(); + assert_eq!(left.tail.len(), i); + assert_eq!(right.len(), 3 - i); + assert_eq!(core::mem::size_of_val(left), 1); + } + } + #[cfg(feature = "derive")] #[test] fn test_split_at() { @@ -1011,6 +1335,8 @@ struct SliceDst { trailing: [u8], } + assert!(SliceDst::LAYOUT.requires_dynamic_padding()); + const N: usize = 16; let arr = [1u16; N]; @@ -1070,21 +1396,79 @@ struct Packet { assert_eq!(r, &[4]); } #[test] + #[allow(deprecated)] fn test_split_at_via_unaligned() { - use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt, Unaligned}; - #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Unaligned)] - #[repr(C)] - struct Packet { - length: u8, - body: [u8], + use crate::{Immutable, KnownLayout, Split, SplitAt, Unaligned}; + + fn via_unaligned<'a, T>(split: Split<&'a T>) -> (&'a T, &'a [T::Elem]) + where + T: ?Sized + SplitAt + Unaligned, + { + split.via_unaligned() } - let arr = [1, 2, 3, 4]; - let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + fn via_unaligned_mut<'a, T>(split: Split<&'a mut T>) -> (&'a mut T, &'a mut [T::Elem]) + where + T: ?Sized + SplitAt + Unaligned, + { + split.via_unaligned() + } + + #[derive(KnownLayout, SplitAt, Immutable, Unaligned)] + #[repr(C)] + struct Packet { + prefix: [u8; 2], + body: B, + } + + // Exercise generic callers using the original `T: Unaligned` bound. + assert!(!Packet::<[[u8; 2]]>::LAYOUT.requires_dynamic_padding()); + let packet = Packet { prefix: [0, 1], body: [[2, 3], [4, 5], [6, 7]] }; + let packet: &Packet<[[u8; 2]]> = &packet; let split = packet.split_at(2).unwrap(); - let (l, r) = split.via_unaligned(); - assert_eq!(l.length, 1); - assert_eq!(r, &[4]); + let (l, r) = via_unaligned(split); + assert_eq!(l.body, [[2, 3], [4, 5]]); + assert_eq!(r, &[[6, 7]]); + + let mut packet = Packet { prefix: [0, 1], body: [[2, 3], [4, 5], [6, 7]] }; + { + let packet: &mut Packet<[[u8; 2]]> = &mut packet; + let split = packet.split_at_mut(2).unwrap(); + let (l, r) = via_unaligned_mut(split); + l.body[0] = [8, 9]; + r[0] = [10, 11]; + } + assert_eq!(packet.body, [[8, 9], [4, 5], [10, 11]]); + } + + #[test] + fn test_split_at_via_no_dynamic_padding() { + use core::cell::Cell; + + use crate::SplitAt; + + // `u16` does not implement `Unaligned`, but `[u16]` never requires + // dynamic trailing padding. + let words = [1u16, 2, 3, 4]; + let split = SplitAt::split_at(&words[..], 2).unwrap(); + let (left, right) = split.via_no_dynamic_padding(); + assert_eq!(left, [1, 2]); + assert_eq!(right, [3, 4]); + + let mut words = [1u16, 2, 3, 4]; + let split = SplitAt::split_at_mut(&mut words[..], 2).unwrap(); + let (left, right) = split.via_no_dynamic_padding(); + left[0] = 5; + right[0] = 6; + assert_eq!(words, [5, 2, 6, 4]); + + // Exercise the safety-sensitive shared route with interior mutation. + let cells = [Cell::new(1u16), Cell::new(2), Cell::new(3)]; + let split = SplitAt::split_at(&cells[..], 2).unwrap(); + let (left, right) = split.via_no_dynamic_padding(); + left[0].set(4); + right[0].set(5); + assert_eq!([cells[0].get(), cells[1].get(), cells[2].get()], [4, 2, 5]); } } diff --git a/rust/zerocopy/src/util/macro_util.rs b/rust/zerocopy/src/util/macro_util.rs index ceeb80432b0b..fdcaa58d418c 100644 --- a/rust/zerocopy/src/util/macro_util.rs +++ b/rust/zerocopy/src/util/macro_util.rs @@ -28,8 +28,8 @@ use crate::{ pointer::{ - cast::CastSized, - invariant::{Aligned, Initialized, Valid}, + cast::CastSizedExact, + invariant::{Aligned, Initialized, Safe}, BecauseImmutable, }, FromBytes, Immutable, IntoBytes, KnownLayout, Ptr, ReadOnly, TryFromBytes, ValidityError, @@ -518,9 +518,9 @@ pub const fn hash_name(name: &str) -> i128 { /// /// `try_transmute` may either produce a post-monomorphization error or a panic /// if `Dst` is bigger than `Src`. Otherwise, `try_transmute` panics under the -/// same circumstances as [`is_bit_valid`]. +/// same circumstances as [`is_safe`]. /// -/// [`is_bit_valid`]: TryFromBytes::is_bit_valid +/// [`is_safe`]: TryFromBytes::is_safe #[inline(always)] pub fn try_transmute(src: Src) -> Result> where @@ -529,30 +529,64 @@ pub fn try_transmute(src: Src) -> Result> { static_assert!(Src, Dst => mem::size_of::() == mem::size_of::()); - let mu_src = mem::MaybeUninit::new(src); - // SAFETY: `MaybeUninit` has no validity requirements. - let mu_dst: mem::MaybeUninit> = - unsafe { crate::util::transmute_unchecked(mu_src) }; - - let ptr = Ptr::from_ref(&mu_dst); - - // SAFETY: Since `Src: IntoBytes`, and since `size_of::() == - // size_of::()` by the preceding assertion, all of `mu_dst`'s bytes are - // initialized. `MaybeUninit` has no validity requirements, so even if - // `ptr` is used to mutate its referent (which it actually can't be - it's - // a shared `ReadOnly` pointer), that won't violate its referent's validity. - let ptr = unsafe { ptr.assume_validity::() }; - if Dst::is_bit_valid(ptr.cast::<_, CastSized, _>()) { - // SAFETY: Since `Dst::is_bit_valid`, we know that `ptr`'s referent is - // bit-valid for `Dst`. `ptr` points to `mu_dst`, and no intervening - // operations have mutated it, so it is a bit-valid `Dst`. - Ok(ReadOnly::into_inner(unsafe { mu_dst.assume_init() })) + let src = mem::ManuallyDrop::new(ReadOnly::new(src)); + let mut ptr = + Ptr::from_ref(&*src).transmute_with::, Initialized, CastSizedExact, _>(); + if Dst::is_safe(ptr.reborrow_shared()) { + // SAFETY: `read_unaligned` requires that `ptr` be valid for reads and + // point to a properly initialized `ReadOnly` [1]. `ptr` was + // derived from a live reference to `src`; `CastSizedExact` preserves + // that reference's address, provenance, and exact byte extent. Thus, + // the entire range remains live and valid for reads. The reference is + // non-null even when `ReadOnly` is zero-sized, and the cast + // preserves that address, satisfying [1]'s explicit ZST requirement. + // + // `Src: IntoBytes` and the `ReadOnly` bridge let `transmute_with` + // establish that every byte in that exact range is initialized. + // `Dst::is_safe` then established that the same bytes are bit-valid for + // `Dst`, and thus for `ReadOnly`. The candidate was a shared + // `ReadOnly` pointer, so it could not mutate the bytes; no operation + // occurs between validation and this read which could do so. Alignment + // is not an additional precondition because [1] expressly permits + // unaligned pointers. + // + // `read_unaligned` leaves the source memory unchanged and makes a + // bitwise copy [1]. The source is wrapped in `ManuallyDrop`, which + // inhibits its destructor [2]. After this read, neither `src` nor + // `ptr` is used to access the inner value, and the wrapper's eventual + // drop does not drop it. Thus, only the returned `Dst` is subsequently + // used or dropped; there is no duplicate ownership use. + // + // [1] Per https://doc.rust-lang.org/1.56.0/std/ptr/fn.read_unaligned.html: + // + // Reads the value from `src` without moving it. This leaves the + // memory in `src` unchanged. + // + // Unlike `read`, `read_unaligned` works with unaligned pointers. + // + // Behavior is undefined if any of the following conditions are + // violated: + // + // * `src` must be valid for reads. + // + // * `src` must point to a properly initialized value of type `T`. + // + // Like `read`, `read_unaligned` creates a bitwise copy of `T`, + // regardless of whether `T` is `Copy`. If `T` is not `Copy`, using + // both the returned value and the value at `*src` can violate + // memory safety. + // + // Note that even if `T` has size `0`, the pointer must be non-null. + // + // [2] Per https://doc.rust-lang.org/1.56.0/std/mem/struct.ManuallyDrop.html: + // + // A wrapper to inhibit compiler from automatically calling `T`'s + // destructor. + let dst = unsafe { core::ptr::read_unaligned(ptr.as_inner().as_ptr()) }; + Ok(ReadOnly::into_inner(dst)) } else { - // SAFETY: `MaybeUninit` has no validity requirements. - let mu_src: mem::MaybeUninit = unsafe { crate::util::transmute_unchecked(mu_dst) }; - // SAFETY: `mu_dst`/`mu_src` was constructed from `src` and never - // modified, so it is still bit-valid. - Err(ValidityError::new(unsafe { mu_src.assume_init() })) + let src = ReadOnly::into_inner(mem::ManuallyDrop::into_inner(src)); + Err(ValidityError::new(src)) } } @@ -599,7 +633,7 @@ fn try_transmute_ref( let res = ptr.try_with(#[inline(always)] |ptr| { let ptr = ptr.recall_validity::(); let ptr = ptr.cast::<_, crate::layout::CastFrom, _>(); - ptr.try_into_valid() + ptr.try_into_safe() }); match res { Ok(ptr) => { @@ -662,7 +696,7 @@ fn try_transmute_mut( ptr.try_with_unchecked(#[inline(always)] |ptr| { let ptr = ptr.recall_validity::(); let ptr = ptr.cast::<_, crate::layout::CastFrom, _>(); - ptr.try_into_valid() + ptr.try_into_safe() }) }; match res { @@ -878,7 +912,7 @@ fn transmute_ref(self) -> &'a Dst { let ptr = Ptr::from_ref(self.0) .recall_validity::() .transmute_with::, (crate::pointer::BecauseMutationCompatible, _)>() - .recall_validity::(); + .recall_validity::(); static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() @@ -911,7 +945,7 @@ fn transmute_mut(self) -> &'a mut Dst { let ptr = Ptr::from_mut(self.0) .recall_validity::() .transmute_with::, _>() - .recall_validity::(); + .recall_validity::(); static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() @@ -946,6 +980,22 @@ pub mod mem { } } +/// Projects `Self` to itself. +/// +/// This is used by macros to have the compiler verify that two syntactically +/// similar types actually resolve to the same type. The blanket implementation +/// ensures that no other implementation can project a type to anything other +/// than itself. +#[doc(hidden)] +pub trait Identity { + /// `Self`. + type Type: ?Sized; +} + +impl Identity for T { + type Type = T; +} + #[cfg(test)] mod tests { use core::num::NonZeroUsize; diff --git a/rust/zerocopy/src/util/macros.rs b/rust/zerocopy/src/util/macros.rs index 7e63e3a54fc4..e514407ceeaa 100644 --- a/rust/zerocopy/src/util/macros.rs +++ b/rust/zerocopy/src/util/macros.rs @@ -15,21 +15,20 @@ /// The trait impl must be sound. /// /// When implementing `TryFromBytes`: -/// - If no `is_bit_valid` impl is provided, then it must be valid for -/// `is_bit_valid` to unconditionally return `true`. In other words, it must -/// be the case that any initialized sequence of bytes constitutes a valid -/// instance of `$ty`. -/// - If an `is_bit_valid` impl is provided, then the impl of `is_bit_valid` -/// must only return `true` if its argument refers to a valid `$ty`. +/// - If no `is_safe` impl is provided, then it must be valid for `is_safe` to +/// unconditionally return `true`. In other words, it must be the case that any +/// initialized sequence of bytes constitutes a valid instance of `$ty`. +/// - If an `is_safe` impl is provided, then the impl of `is_safe` must only +/// return `true` if its argument refers to a valid `$ty`. macro_rules! unsafe_impl { // Implement `$trait` for `$ty` with no bounds. - ($(#[$attr:meta])* $ty:ty: $trait:ident $(; |$candidate:ident| $is_bit_valid:expr)?) => {{ + ($(#[$attr:meta])* $ty:ty: $trait:ident $(; |$candidate:ident| $is_safe:expr)?) => {{ crate::util::macros::__unsafe(); $(#[$attr])* // SAFETY: The caller promises that this is sound. unsafe impl $trait for $ty { - unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + unsafe_impl!(@method $trait $(; |$candidate| $is_safe)?); } }}; @@ -92,26 +91,26 @@ unsafe impl $trait for $ty { $(#[$attr:meta])* const $constname:ident : $constty:ident $(,)? $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* - => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_safe:expr)? ) => { unsafe_impl!( @inner $(#[$attr])* @const $constname: $constty, $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* - => $trait for $ty $(; |$candidate| $is_bit_valid)? + => $trait for $ty $(; |$candidate| $is_safe)? ); }; ( $(#[$attr:meta])* $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* - => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_safe:expr)? ) => {{ unsafe_impl!( @inner $(#[$attr])* $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* - => $trait for $ty $(; |$candidate| $is_bit_valid)? + => $trait for $ty $(; |$candidate| $is_safe)? ); }}; ( @@ -119,7 +118,7 @@ unsafe impl $trait for $ty { $(#[$attr:meta])* $(@const $constname:ident : $constty:ident,)* $($tyvar:ident $(: $(? $optbound:ident +)* + $($bound:ident +)* )?,)* - => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_safe:expr)? ) => {{ crate::util::macros::__unsafe(); @@ -127,21 +126,21 @@ unsafe impl $trait for $ty { #[allow(non_local_definitions)] // SAFETY: The caller promises that this is sound. unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),* $(, const $constname: $constty,)*> $trait for $ty { - unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + unsafe_impl!(@method $trait $(; |$candidate| $is_safe)?); } }}; - (@method TryFromBytes ; |$candidate:ident| $is_bit_valid:expr) => { + (@method TryFromBytes ; |$candidate:ident| $is_safe:expr) => { #[allow(clippy::missing_inline_in_public_items, dead_code)] #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] fn only_derive_is_allowed_to_implement_this_trait() {} #[inline] - fn is_bit_valid($candidate: Maybe<'_, Self, Alignment>) -> bool + fn is_safe($candidate: Maybe<'_, Self, Alignment>) -> bool where Alignment: crate::invariant::Alignment, { - $is_bit_valid + $is_safe } }; (@method TryFromBytes) => { @@ -149,7 +148,7 @@ fn is_bit_valid($candidate: Maybe<'_, Self, Alignment>) -> bool #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] fn only_derive_is_allowed_to_implement_this_trait() {} #[inline(always)] - fn is_bit_valid(_candidate: Maybe<'_, Self, Alignment>) -> bool + fn is_safe(_candidate: Maybe<'_, Self, Alignment>) -> bool where Alignment: crate::invariant::Alignment, { @@ -161,16 +160,16 @@ fn is_bit_valid(_candidate: Maybe<'_, Self, Alignment>) -> bool #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] fn only_derive_is_allowed_to_implement_this_trait() {} }; - (@method $trait:ident; |$_candidate:ident| $_is_bit_valid:expr) => { - compile_error!("Can't provide `is_bit_valid` impl for trait other than `TryFromBytes`"); + (@method $trait:ident; |$_candidate:ident| $_is_safe:expr) => { + compile_error!("Can't provide `is_safe` impl for trait other than `TryFromBytes`"); }; } /// Implements `$trait` for `$ty` where `$ty: TransmuteFrom<$repr>` (and /// vice-versa). /// -/// Calling this macro is safe; the internals of the macro emit appropriate -/// trait bounds which ensure that the given impl is sound. +/// This macro is intended to be safe to call. The current proof is incomplete; +/// see FIXME(#3691) below. macro_rules! impl_for_transmute_from { ( $(#[$attr:meta])* @@ -181,26 +180,32 @@ macro_rules! impl_for_transmute_from { $(#[$attr])* #[allow(non_local_definitions)] - // SAFETY: `is_trait` (defined and used below) requires `T: - // TransmuteFrom`, `R: TransmuteFrom`, and `R: $trait`. It is - // called using `$ty` and `$repr`, ensuring that `$ty` and `$repr` - // have equivalent bit validity, and ensuring that `$repr: $trait`. - // The supported traits - `TryFromBytes`, `FromZeros`, `FromBytes`, - // and `IntoBytes` - are defined only in terms of the bit validity - // of a type. Therefore, `$repr: $trait` ensures that `$ty: $trait` - // is sound. + // SAFETY: `is_trait` (defined and used below) requires + // reciprocal `TransmuteFrom<_, Safe, Safe>` bounds and `R: $trait`. + // If `T` and `R` have the same size, the reciprocal bounds imply + // that they permit the same `Safe` bit patterns. The call below + // instantiates `T` with `$ty` and `R` with `$repr`, and establishes + // `$repr: $trait`. The supported traits - `TryFromBytes`, + // `FromZeros`, `FromBytes`, and `IntoBytes` - are defined only in + // terms of bit validity, so these premises are sufficient when + // `$ty` and `$repr` have the same size. + // + // FIXME(#3691): This macro does not establish that `$ty` and + // `$repr` have the same size. Without that premise, `TransmuteFrom` + // conveys no safety guarantee, so these bounds do not make + // arbitrary invocations of this macro sound. unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?> $trait for $ty { #[allow(dead_code, clippy::missing_inline_in_public_items)] #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] fn only_derive_is_allowed_to_implement_this_trait() { - use crate::pointer::{*, invariant::Valid}; + use crate::pointer::{*, invariant::Safe}; impl_for_transmute_from!(@assert_is_supported_trait $trait); fn is_trait() where - T: TransmuteFrom + ?Sized, - R: TransmuteFrom + ?Sized, + T: TransmuteFrom + ?Sized, + R: TransmuteFrom + ?Sized, R: $trait, { } @@ -212,7 +217,7 @@ fn f<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?>() { } impl_for_transmute_from!( - @is_bit_valid + @is_safe $(<$tyvar $(: $(? $optbound +)* $($bound +)*)?>)? $trait for $ty [$repr] ); @@ -224,27 +229,27 @@ fn f<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?>() { (@assert_is_supported_trait FromBytes) => {}; (@assert_is_supported_trait IntoBytes) => {}; ( - @is_bit_valid + @is_safe $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? TryFromBytes for $ty:ty [$repr:ty] ) => { #[inline(always)] - fn is_bit_valid(candidate: $crate::Maybe<'_, Self, Alignment>) -> bool + fn is_safe(candidate: $crate::Maybe<'_, Self, Alignment>) -> bool where Alignment: $crate::invariant::Alignment, { // SAFETY: This macro ensures that `$repr` and `Self` have the same // size and bit validity. Thus, a bit-valid instance of `$repr` is // also a bit-valid instance of `Self`. - <$repr as TryFromBytes>::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + <$repr as TryFromBytes>::is_safe(candidate.transmute::<_, _, BecauseImmutable>()) } }; ( - @is_bit_valid + @is_safe $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? $trait:ident for $ty:ty [$repr:ty] ) => { - // Trait other than `TryFromBytes`; no `is_bit_valid` impl. + // Trait other than `TryFromBytes`; no `is_safe` impl. }; } @@ -269,33 +274,33 @@ fn is_bit_valid(candidate: $crate::Maybe<'_, Self, Alignment>) -> boo macro_rules! unsafe_impl_for_power_set { ( $first:ident $(, $rest:ident)* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) - $(; |$candidate:ident| $is_bit_valid:expr)? + $(; |$candidate:ident| $is_safe:expr)? ) => { unsafe_impl_for_power_set!( $($rest),* $(-> $ret)? => $trait for $macro!(...) - $(; |$candidate| $is_bit_valid)? + $(; |$candidate| $is_safe)? ); unsafe_impl_for_power_set!( @impl $first $(, $rest)* $(-> $ret)? => $trait for $macro!(...) - $(; |$candidate| $is_bit_valid)? + $(; |$candidate| $is_safe)? ); }; ( $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) - $(; |$candidate:ident| $is_bit_valid:expr)? + $(; |$candidate:ident| $is_safe:expr)? ) => { unsafe_impl_for_power_set!( @impl $(-> $ret)? => $trait for $macro!(...) - $(; |$candidate| $is_bit_valid)? + $(; |$candidate| $is_safe)? ); }; ( @impl $($vars:ident),* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) - $(; |$candidate:ident| $is_bit_valid:expr)? + $(; |$candidate:ident| $is_safe:expr)? ) => { unsafe_impl!( $($vars,)* $($ret)? => $trait for $macro!($($vars),* $(-> $ret)?) - $(; |$candidate| $is_bit_valid)? + $(; |$candidate| $is_safe)? ); }; } @@ -355,8 +360,8 @@ macro_rules! opt_unsafe_fn { /// impl in this case, and also provides useful documentation for readers of the /// code. /// -/// Finally, if a `TryFromBytes::is_bit_valid` impl is provided, it must adhere -/// to the safety preconditions of [`unsafe_impl!`]. +/// Finally, if a `TryFromBytes::is_safe` impl is provided, it must adhere to the +/// safety preconditions of [`unsafe_impl!`]. /// /// ## Example /// @@ -396,11 +401,11 @@ impl $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_safe:expr)? ) => { impl_or_verify!(@impl { unsafe_impl!( $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty - $(; |$candidate| $is_bit_valid)? + $(; |$candidate| $is_safe)? ); }); impl_or_verify!(@verify $trait, { impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),*> Subtrait for $ty {} @@ -495,7 +500,7 @@ macro_rules! unsafe_impl_known_layout { #[allow(non_local_definitions)] // SAFETY: The caller promises that this is sound. unsafe impl<$($tyvar: ?Sized + KnownLayout)?> KnownLayout for $ty { - #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[allow(dead_code, clippy::missing_inline_in_public_items)] #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] fn only_derive_is_allowed_to_implement_this_trait() {} @@ -742,14 +747,14 @@ macro_rules! unsafe_impl_for_transparent_wrapper { ($vis:vis T $(: ?$optbound:ident)? => $wrapper:ident) => {{ crate::util::macros::__unsafe(); - use crate::pointer::{TransmuteFrom, cast::{CastExact, TransitiveProject}, SizeEq, invariant::Valid}; + use crate::pointer::{TransmuteFrom, cast::{CastExact, TransitiveProject}, SizeEq, invariant::Safe}; use crate::wrappers::ReadOnly; // SAFETY: The caller promises that `T` and `$wrapper` have the same // bit validity. - unsafe impl TransmuteFrom for $wrapper {} + unsafe impl TransmuteFrom for $wrapper {} // SAFETY: See previous safety comment. - unsafe impl TransmuteFrom<$wrapper, Valid, Valid> for T {} + unsafe impl TransmuteFrom<$wrapper, Safe, Safe> for T {} // SAFETY: The caller promises that a `T` to `$wrapper` cast is // size-preserving. define_cast!(unsafe { $vis CastToWrapper = T => $wrapper }); @@ -805,7 +810,7 @@ impl SizeEq>> for ReadOnly { macro_rules! impl_transitive_transmute_from { ($($tyvar:ident $(: ?$optbound:ident)?)? => $t:ty => $u:ty => $v:ty) => { const _: () = { - use crate::pointer::{TransmuteFrom, SizeEq, invariant::Valid}; + use crate::pointer::{TransmuteFrom, SizeEq, invariant::Safe}; impl<$($tyvar $(: ?$optbound)?)?> SizeEq<$t> for $v where @@ -819,14 +824,14 @@ impl<$($tyvar $(: ?$optbound)?)?> SizeEq<$t> for $v >; } - // SAFETY: Since `$u: TransmuteFrom<$t, Valid, Valid>`, it is sound - // to transmute a bit-valid `$t` to a bit-valid `$u`. Since `$v: - // TransmuteFrom<$u, Valid, Valid>`, it is sound to transmute that + // SAFETY: Since `$u: TransmuteFrom<$t, Safe, Safe>`, it is sound to + // transmute a bit-valid `$t` to a bit-valid `$u`. Since `$v: + // TransmuteFrom<$u, Safe, Safe>`, it is sound to transmute that // bit-valid `$u` to a bit-valid `$v`. - unsafe impl<$($tyvar $(: ?$optbound)?)?> TransmuteFrom<$t, Valid, Valid> for $v + unsafe impl<$($tyvar $(: ?$optbound)?)?> TransmuteFrom<$t, Safe, Safe> for $v where - $u: TransmuteFrom<$t, Valid, Valid>, - $v: TransmuteFrom<$u, Valid, Valid>, + $u: TransmuteFrom<$t, Safe, Safe>, + $v: TransmuteFrom<$u, Safe, Safe>, {} }; }; diff --git a/rust/zerocopy/src/util/mod.rs b/rust/zerocopy/src/util/mod.rs index 02fd4ed62741..646f45739802 100644 --- a/rust/zerocopy/src/util/mod.rs +++ b/rust/zerocopy/src/util/mod.rs @@ -23,7 +23,7 @@ use super::*; use crate::pointer::{ - invariant::{Exclusive, Shared, Valid}, + invariant::{Exclusive, Safe, Shared}, SizeEq, TransmuteFromPtr, }; @@ -108,7 +108,7 @@ fn addr(self) -> usize { #[allow(clippy::as_conversions)] #[cfg_attr( __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, - allow(lossy_provenance_casts) + allow(implicit_provenance_casts) )] return self.cast::<()>() as usize; } @@ -144,9 +144,10 @@ pub(crate) fn validate_aligned_to(t: T) -> Result<(), Alignment /// on the answer it gives if this is not the case. #[cfg_attr( kani, - kani::requires(len <= DstLayout::MAX_SIZE), kani::requires(align.is_power_of_two()), - kani::ensures(|&p| (len + p) % align.get() == 0), + // A power-of-two alignment divides the `usize` modulus, so wrapping + // preserves congruence even when the next aligned value exceeds `usize`. + kani::ensures(|&p| len.wrapping_add(p) % align.get() == 0), // Ensures that we add the minimum required padding. kani::ensures(|&p| p < align.get()), )] @@ -340,7 +341,7 @@ pub(crate) unsafe fn transmute_ref(src: &Src) -> &Dst where Src: ?Sized, Dst: SizeEq - + TransmuteFromPtr>::CastFrom, R> + + TransmuteFromPtr>::CastFrom, R> + ?Sized, { let dst = Ptr::from_ref(src).transmute(); @@ -357,7 +358,7 @@ pub(crate) unsafe fn transmute_mut(src: &mut Src) -> &mut Dst where Src: ?Sized, Dst: SizeEq - + TransmuteFromPtr>::CastFrom, R> + + TransmuteFromPtr>::CastFrom, R> + ?Sized, { let dst = Ptr::from_mut(src).transmute(); @@ -562,31 +563,6 @@ pub(crate) fn get(&self) -> T::PointerMetadata self.meta } - #[inline] - pub(crate) fn padding_needed_for(&self) -> usize - where - T: KnownLayout, - { - let trailing_slice_layout = crate::trailing_slice_layout::(); - - // FIXME(#67): Remove this allow. See NumExt for more details. - #[allow( - unstable_name_collisions, - clippy::incompatible_msrv, - clippy::multiple_unsafe_ops_per_block - )] - // SAFETY: By invariant on `self`, a `&T` with metadata `self.meta` - // describes an object of size `<= isize::MAX`. This computes the - // size of such a `&T` without any trailing padding, and so neither - // the multiplication nor the addition will overflow. - let unpadded_size = unsafe { - let trailing_size = self.meta.unchecked_mul(trailing_slice_layout.elem_size); - trailing_size.unchecked_add(trailing_slice_layout.offset) - }; - - util::padding_needed_for(unpadded_size, T::LAYOUT.align) - } - #[inline(always)] pub(crate) fn validate_cast_and_convert_metadata( addr: usize, @@ -895,6 +871,8 @@ mod tests { #[test] fn test_round_down_to_next_multiple_of_alignment() { + // Division by `NonZeroUsize` unsupported on our MSRV. + #[allow(clippy::needless_nonzero_get)] fn alt_impl(n: usize, align: NonZeroUsize) -> usize { let mul = n / align.get(); mul * align.get() diff --git a/rust/zerocopy/src/wrappers.rs b/rust/zerocopy/src/wrappers.rs index 1a8cf2b41d55..d0baa901f10c 100644 --- a/rust/zerocopy/src/wrappers.rs +++ b/rust/zerocopy/src/wrappers.rs @@ -8,10 +8,10 @@ // This file may not be copied, modified, or distributed except according to // those terms. -use core::{fmt, hash::Hash}; +use core::{borrow::Borrow, fmt, hash::Hash}; use super::*; -use crate::pointer::{invariant::Valid, SizeEq, TransmuteFrom}; +use crate::pointer::{invariant::Safe, SizeEq, TransmuteFrom}; /// A type with no alignment requirement. /// @@ -140,10 +140,10 @@ // Unaligned`. // - `Unalign` has the same bit validity as `T`, and so it is `FromZeros`, // `FromBytes`, or `IntoBytes` exactly when `T` is as well. -// - `Immutable`: `Unalign` has the same fields as `T`, so it permits -// interior mutation exactly when `T` does. +// - `Immutable`: `Unalign` has the same fields as `T`, so it permits interior +// mutation exactly when `T` does. // - `TryFromBytes`: `Unalign` has the same the same bit validity as `T`, so -// `T::is_bit_valid` is a sound implementation of `is_bit_valid`. +// `T::is_safe` is a sound implementation of `is_safe`. // #[allow(clippy::multiple_unsafe_ops_per_block)] const _: () = unsafe { @@ -151,7 +151,7 @@ impl_or_verify!(T: Immutable => Immutable for Unalign); impl_or_verify!( T: TryFromBytes => TryFromBytes for Unalign; - |c| T::is_bit_valid(c.transmute::<_, _, BecauseImmutable>()) + |c| T::is_safe(c.transmute::<_, _, BecauseImmutable>()) ); impl_or_verify!(T: FromZeros => FromZeros for Unalign); impl_or_verify!(T: FromBytes => FromBytes for Unalign); @@ -613,7 +613,14 @@ mod read_only_def { /// Note that `&mut ReadOnly` still permits mutation – the read-only /// property only applies to shared references. /// + /// `ReadOnly` implements [`Copy`] and [`Clone`] when `T: Copy`. Cloning + /// copies the value without calling `T::clone`. Trait implementations that + /// expose or operate on a shared reference to `T` require `T: Immutable`. + /// Constructing a wrapper or accessing its contents through [`AsMut`] + /// does not require `T: Immutable`. + /// /// [`Immutable`]: crate::Immutable + #[derive(Copy)] #[repr(transparent)] pub struct ReadOnly { // INVARIANT: `inner` is never mutated through a `&ReadOnly` @@ -665,7 +672,7 @@ pub(crate) fn as_mut(r: &mut ReadOnly) -> &mut T { // SAFETY: // - `ReadOnly` has the same alignment as `T`, and so it is `Unaligned` // exactly when `T` is as well. -// - `ReadOnly` has the same bit validity as `T`, and so this `is_bit_valid` +// - `ReadOnly` has the same bit validity as `T`, and so this `is_safe` // implementation is correct, and thus the `TryFromBytes` impl is sound. // - `ReadOnly` has the same bit validity as `T`, and so it is `FromZeros`, // `FromBytes`, and `IntoBytes` exactly when `T` is as well. @@ -673,7 +680,7 @@ pub(crate) fn as_mut(r: &mut ReadOnly) -> &mut T { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ReadOnly); unsafe_impl!( T: ?Sized + TryFromBytes => TryFromBytes for ReadOnly; - |c| T::is_bit_valid(c.cast::<_, as SizeEq>>>::CastFrom, _>()) + |c| T::is_safe(c.cast::<_, as SizeEq>>>::CastFrom, _>()) ); unsafe_impl!(T: ?Sized + FromZeros => FromZeros for ReadOnly); unsafe_impl!(T: ?Sized + FromBytes => FromBytes for ReadOnly); @@ -709,11 +716,34 @@ impl SizeEq for ReadOnly { // SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so // it has the same bit validity as `T`. -unsafe impl TransmuteFrom for ReadOnly {} +unsafe impl TransmuteFrom for ReadOnly {} // SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so // it has the same bit validity as `T`. -unsafe impl TransmuteFrom, Valid, Valid> for T {} +unsafe impl TransmuteFrom, Safe, Safe> for T {} + +impl Default for ReadOnly { + #[inline(always)] + fn default() -> Self { + Self::new(Default::default()) + } +} + +// Copying avoids calling `T::clone`, which could mutate through a shared +// reference to `T` and violate `ReadOnly`'s invariant. +impl Clone for ReadOnly { + #[inline(always)] + fn clone(&self) -> Self { + *self + } +} + +impl From for ReadOnly { + #[inline(always)] + fn from(t: T) -> Self { + Self::new(t) + } +} impl<'a, T: ?Sized + Immutable> From<&'a T> for &'a ReadOnly { #[inline(always)] @@ -743,15 +773,74 @@ fn deref_mut(&mut self) -> &mut Self::Target { } } -impl Debug for ReadOnly { +impl AsRef for ReadOnly { #[inline(always)] - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - self.deref().fmt(f) + fn as_ref(&self) -> &T { + self.deref() } } +impl Borrow for ReadOnly { + #[inline(always)] + fn borrow(&self) -> &T { + self.deref() + } +} + +impl AsMut for ReadOnly { + #[inline(always)] + fn as_mut(&mut self) -> &mut T { + ReadOnly::as_mut(self) + } +} + +// Delegate shared access through `Deref`, which requires `T: Immutable` and +// thus prevents interior mutation of the wrapped value by these trait methods. +impl PartialEq for ReadOnly { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl Eq for ReadOnly {} + +impl PartialOrd for ReadOnly { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option { + self.deref().partial_cmp(other.deref()) + } +} + +impl Ord for ReadOnly { + #[inline(always)] + fn cmp(&self, other: &Self) -> Ordering { + self.deref().cmp(other.deref()) + } +} + +impl Hash for ReadOnly { + #[inline(always)] + fn hash(&self, state: &mut H) { + self.deref().hash(state); + } +} + +macro_rules! impl_read_only_fmt { + ($($trait:ident),* $(,)?) => {$( + impl fmt::$trait for ReadOnly { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + fmt::$trait::fmt(self.deref(), f) + } + } + )*}; +} + +impl_read_only_fmt!(Debug, Display, Binary, Octal, LowerHex, UpperHex, LowerExp, UpperExp); + // SAFETY: See safety comment on `ProjectToTag`. -unsafe impl HasTag for ReadOnly { +unsafe impl + ?Sized, Client> HasTag for ReadOnly { #[allow(clippy::missing_inline_in_public_items)] fn only_derive_is_allowed_to_implement_this_trait() where @@ -759,30 +848,33 @@ fn only_derive_is_allowed_to_implement_this_trait() { } - type Tag = T::Tag; + type Tag = >::Tag; // SAFETY: `>>::CastFrom` is a no-op projection that - // produces a pointer with the same referent. By invariant, for any `Ptr<'_, - // T, I>` it is sound to use `T::ProjectToTag` to project to a `Ptr<'_, - // T::Tag, I>`. Since `ReadOnly` has the same layout and validity as `T`, - // the same is true of projecting from a `Ptr<'_, ReadOnly, I>`. + // produces a pointer with the same referent. By invariant, for any + // `Ptr<'_, T, I>` it is sound to use + // `>::ProjectToTag` to project to a + // shared `Ptr<'_, >::Tag, + // (_, I::Alignment, I::Validity)>`. Since `ReadOnly` has the same + // layout and validity as `T`, the same is true of projecting from a + // `Ptr<'_, ReadOnly, I>`. type ProjectToTag = crate::pointer::cast::TransitiveProject< T, >>::CastFrom, - T::ProjectToTag, + >::ProjectToTag, >; } // SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so // has the same fields at the same offsets. Thus, it satisfies the safety -// invariants of `HasField` for field `f` exactly -// when `T` does, as guaranteed by the `T: HasField` bound: -// - If `VARIANT_ID` is `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID`, then `T` has -// the layout of a struct or union type. Since `ReadOnly` is a transparent -// wrapper around `T`, it does too. Otherwise, if `VARIANT_ID` is an enum -// variant index, then `T` has the layout of an enum type, and `ReadOnly` -// does too. -// - By `T: HasField<_, _, FIELD_ID>`: +// invariants of `HasField` for field `f` +// exactly when `T` does, as guaranteed by the `T: HasField` bound: +// - If `VARIANT_ID` is `STRUCT_VARIANT_ID`, `UNION_VARIANT_ID`, or +// `REPR_C_UNION_VARIANT_ID`, then `T` has the layout of a struct, non-C union, +// or C union type respectively. Since `ReadOnly` is a transparent wrapper +// around `T`, it does too. Otherwise, `VARIANT_ID` is the identifier ID of an +// enum variant; `T` has the layout of an enum type, and `ReadOnly` does too. +// - By `T: HasField<_, _, _, FIELD_ID>`: // - `T` has a field `f` with name `n` such that // `FIELD_ID = zerocopy::ident_id!(n)` or at index `i` such that // `FIELD_ID = zerocopy::ident_id!(i)`. @@ -794,10 +886,10 @@ fn only_derive_is_allowed_to_implement_this_trait() // refers to a non-strict subset of the bytes of `slf`'s referent, and has the // same provenance as `slf` – because all intermediate operations satisfy those // same conditions. -unsafe impl - HasField for ReadOnly +unsafe impl + HasField for ReadOnly where - T: HasField + ?Sized, + T: HasField + ?Sized, { #[allow(clippy::missing_inline_in_public_items)] fn only_derive_is_allowed_to_implement_this_trait() @@ -811,7 +903,7 @@ fn only_derive_is_allowed_to_implement_this_trait() #[inline(always)] fn project(slf: PtrInner<'_, Self>) -> *mut ReadOnly { slf.project::<_, >>::CastFrom>() - .project::<_, crate::pointer::cast::Projection>() + .project::<_, crate::pointer::cast::Projection>() .project::<_, as SizeEq>::CastFrom>() .as_non_null() .as_ptr() @@ -822,10 +914,10 @@ fn project(slf: PtrInner<'_, Self>) -> *mut ReadOnly { // has the same fields at the same offsets. `is_projectable` simply delegates to // `T::is_projectable`, which is sound because a `Ptr<'_, ReadOnly, I>` will // be projectable exactly when a `Ptr<'_, T, I>` referent is. -unsafe impl - ProjectField for ReadOnly +unsafe impl + ProjectField for ReadOnly where - T: ProjectField + ?Sized, + T: ProjectField + ?Sized, I: invariant::Invariants, { #[allow(clippy::missing_inline_in_public_items)] @@ -835,13 +927,15 @@ fn only_derive_is_allowed_to_implement_this_trait() { } - type Invariants = T::Invariants; + type Invariants = >::Invariants; - type Error = T::Error; + type Error = >::Error; #[inline(always)] - fn is_projectable<'a>(ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { - T::is_projectable(ptr) + fn is_projectable<'a>( + ptr: Ptr<'a, >::Tag, (invariant::Shared, I::Alignment, I::Validity)>, + ) -> Result<(), Self::Error> { + >::is_projectable(ptr) } } @@ -852,6 +946,96 @@ mod tests { use super::*; use crate::util::testutil::*; + #[test] + #[allow(clippy::clone_on_copy, clippy::non_canonical_clone_impl)] + fn test_read_only_copy_clone() { + // This type deliberately does not implement `Immutable`. Copying the + // wrapper must not require it or invoke the inner `Clone` impl. + #[derive(Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] + struct PanickingClone(u16); + + impl Clone for PanickingClone { + fn clone(&self) -> Self { + panic!("ReadOnly must not call the inner Clone implementation") + } + } + + static_assertions::assert_not_impl_any!( + ReadOnly: Debug, PartialEq, Eq, PartialOrd, Ord, + Hash, AsRef, Borrow + ); + static_assertions::assert_not_impl_any!( + ReadOnly>: Copy, Clone, Deref, AsRef>, Borrow> + ); + + let value = ReadOnly::new(PanickingClone(42)); + let copy = value; + let clone = value.clone(); + assert_eq!(ReadOnly::into_inner(copy).0, 42); + assert_eq!(ReadOnly::into_inner(clone).0, 42); + } + + #[test] + fn test_read_only_mutable_access() { + // Construction and exclusive access also support interior mutability. + let mut value = ReadOnly::>::default(); + let inner: &mut Cell = value.as_mut(); + inner.set(42); + assert_eq!(ReadOnly::into_inner(value).get(), 42); + + let value = ReadOnly::from(Cell::new(7)); + assert_eq!(ReadOnly::into_inner(value).get(), 7); + } + + #[test] + fn test_read_only_borrowed_lookup() { + // Borrowed lookups require equality, hashing, and ordering to agree + // between the wrapper and the inner value. + let mut hash_map = std::collections::HashMap::new(); + hash_map.insert(ReadOnly::new(42u16), true); + assert_eq!(hash_map.get(&42u16), Some(&true)); + assert_eq!(hash_map.get(&43u16), None); + + let mut tree = std::collections::BTreeMap::new(); + tree.insert(ReadOnly::new(42u16), true); + assert_eq!(tree.get(&42u16), Some(&true)); + assert_eq!(tree.get(&43u16), None); + + // Preserve partial orders, including incomparable values. + let nan = ReadOnly::new(f32::NAN); + assert_eq!(nan.partial_cmp(&nan), None); + assert_ne!(nan, nan); + } + + #[test] + fn test_read_only_unsized() { + let bytes = [1u8, 2]; + let value: &ReadOnly<[u8]> = (&bytes[..]).into(); + let as_ref: &[u8] = value.as_ref(); + let borrowed: &[u8] = value.borrow(); + assert_eq!(as_ref, bytes); + assert_eq!(borrowed, bytes); + assert_eq!(value, value); + assert_eq!(value.cmp(value), Ordering::Equal); + + let text: &ReadOnly = "hello".into(); + assert_eq!(format!("{:.3}", text), "hel"); + } + + #[test] + fn test_read_only_formatting() { + let value = ReadOnly::new(42u16); + // Check that formatting flags are forwarded as well as the value. + assert_eq!(format!("{:04?}", value), "0042"); + assert_eq!(format!("{:04}", value), "0042"); + assert_eq!(format!("{:#010b}", value), "0b00101010"); + assert_eq!(format!("{:#06o}", value), "0o0052"); + assert_eq!(format!("{:#06x}", value), "0x002a"); + assert_eq!(format!("{:#06X}", value), "0x002A"); + assert_eq!(format!("{:.2e}", value), "4.20e1"); + assert_eq!(format!("{:.2E}", value), "4.20E1"); + } + #[test] fn test_unalign() { // Test methods that don't depend on alignment.