diff --git a/bindgen-cli/options.rs b/bindgen-cli/options.rs index 66bb3f1439..815a28f42a 100644 --- a/bindgen-cli/options.rs +++ b/bindgen-cli/options.rs @@ -400,6 +400,12 @@ struct BindgenCommand { /// Wrap unsafe operations in unsafe blocks. #[arg(long)] wrap_unsafe_ops: bool, + /// Enable fallback for clang macro parsing. + #[arg(long)] + clang_macro_fallback: bool, + /// Set path for temporary files generated by fallback for clang macro parsing. + #[arg(long)] + clang_macro_fallback_build_dir: Option, /// Derive custom traits on any kind of type. The CUSTOM value must be of the shape REGEX=DERIVE where DERIVE is a coma-separated list of derive macros. #[arg(long, value_name = "CUSTOM", value_parser = parse_custom_derive)] with_derive_custom: Vec<(Vec, String)>, @@ -554,6 +560,8 @@ where merge_extern_blocks, override_abi, wrap_unsafe_ops, + clang_macro_fallback, + clang_macro_fallback_build_dir, with_derive_custom, with_derive_custom_struct, with_derive_custom_enum, @@ -1023,6 +1031,14 @@ where builder = builder.wrap_unsafe_ops(true); } + if clang_macro_fallback { + builder = builder.clang_macro_fallback(); + } + + if let Some(path) = clang_macro_fallback_build_dir { + builder = builder.clang_macro_fallback_build_dir(path); + } + #[derive(Debug)] struct CustomDeriveCallback { derives: Vec, diff --git a/bindgen-tests/tests/expectations/tests/issue-753.rs b/bindgen-tests/tests/expectations/tests/issue-753.rs new file mode 100644 index 0000000000..3119ec569e --- /dev/null +++ b/bindgen-tests/tests/expectations/tests/issue-753.rs @@ -0,0 +1,4 @@ +#![allow(dead_code, non_snake_case, non_camel_case_types, non_upper_case_globals)] +pub const CONST: u32 = 5; +pub const OTHER_CONST: u32 = 6; +pub const LARGE_CONST: u32 = 1536; diff --git a/bindgen-tests/tests/headers/issue-753.h b/bindgen-tests/tests/headers/issue-753.h new file mode 100644 index 0000000000..94bb8e1c31 --- /dev/null +++ b/bindgen-tests/tests/headers/issue-753.h @@ -0,0 +1,7 @@ +// bindgen-flags: --clang-macro-fallback + +#define UINT32_C(c) c ## U + +#define CONST UINT32_C(5) +#define OTHER_CONST UINT32_C(6) +#define LARGE_CONST UINT32_C(6 << 8) diff --git a/bindgen/clang.rs b/bindgen/clang.rs index a9a0ec814f..a3f6174e66 100644 --- a/bindgen/clang.rs +++ b/bindgen/clang.rs @@ -10,6 +10,7 @@ use std::cmp; use std::ffi::{CStr, CString}; use std::fmt; +use std::fs::OpenOptions; use std::hash::Hash; use std::hash::Hasher; use std::os::raw::{c_char, c_int, c_longlong, c_uint, c_ulong, c_ulonglong}; @@ -1868,6 +1869,27 @@ impl TranslationUnit { } } + /// Save a translation unit to the given file. + pub(crate) fn save(&mut self, file: &str) -> Result<(), CXSaveError> { + let file = if let Ok(cstring) = CString::new(file) { + cstring + } else { + return Err(CXSaveError_Unknown); + }; + let ret = unsafe { + clang_saveTranslationUnit( + self.x, + file.as_ptr(), + clang_defaultSaveOptions(self.x), + ) + }; + if ret != 0 { + Err(ret) + } else { + Ok(()) + } + } + /// Is this the null translation unit? pub(crate) fn is_null(&self) -> bool { self.x.is_null() @@ -1882,6 +1904,80 @@ impl Drop for TranslationUnit { } } +/// Translation unit used for macro fallback parsing +pub(crate) struct FallbackTranslationUnit { + file_path: String, + idx: Box, + tu: TranslationUnit, +} + +impl fmt::Debug for FallbackTranslationUnit { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "FallbackTranslationUnit {{ }}") + } +} + +impl FallbackTranslationUnit { + /// Create a new fallback translation unit + pub(crate) fn new(file: String, c_args: &[Box]) -> Option { + // Create empty file + OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&file) + .ok()?; + + let f_index = Box::new(Index::new(true, false)); + let f_translation_unit = TranslationUnit::parse( + &f_index, + &file, + c_args, + &[], + CXTranslationUnit_None, + )?; + Some(FallbackTranslationUnit { + file_path: file, + tu: f_translation_unit, + idx: f_index, + }) + } + + /// Get reference to underlying translation unit. + pub(crate) fn translation_unit(&self) -> &TranslationUnit { + &self.tu + } + + /// Reparse a translation unit. + pub(crate) fn reparse( + &mut self, + unsaved_contents: &str, + ) -> Result<(), CXErrorCode> { + let unsaved = &[UnsavedFile::new(&self.file_path, unsaved_contents)]; + let mut c_unsaved: Vec = + unsaved.iter().map(|f| f.x).collect(); + let ret = unsafe { + clang_reparseTranslationUnit( + self.tu.x, + unsaved.len() as c_uint, + c_unsaved.as_mut_ptr(), + clang_defaultReparseOptions(self.tu.x), + ) + }; + if ret != 0 { + Err(ret) + } else { + Ok(()) + } + } +} + +impl Drop for FallbackTranslationUnit { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.file_path); + } +} + /// A diagnostic message generated while parsing a translation unit. pub(crate) struct Diagnostic { x: CXDiagnostic, diff --git a/bindgen/ir/context.rs b/bindgen/ir/context.rs index 26247cdcc5..3dad174f5d 100644 --- a/bindgen/ir/context.rs +++ b/bindgen/ir/context.rs @@ -30,6 +30,7 @@ use std::borrow::Cow; use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap as StdHashMap}; use std::mem; +use std::path::Path; /// An identifier for some kind of IR item. #[derive(Debug, Copy, Clone, Eq, PartialOrd, Ord, Hash)] @@ -251,9 +252,9 @@ where T: Copy + Into, { fn can_derive_partialord(&self, ctx: &BindgenContext) -> bool { - ctx.options().derive_partialord && - ctx.lookup_can_derive_partialeq_or_partialord(*self) == - CanDerive::Yes + ctx.options().derive_partialord + && ctx.lookup_can_derive_partialeq_or_partialord(*self) + == CanDerive::Yes } } @@ -262,9 +263,9 @@ where T: Copy + Into, { fn can_derive_partialeq(&self, ctx: &BindgenContext) -> bool { - ctx.options().derive_partialeq && - ctx.lookup_can_derive_partialeq_or_partialord(*self) == - CanDerive::Yes + ctx.options().derive_partialeq + && ctx.lookup_can_derive_partialeq_or_partialord(*self) + == CanDerive::Yes } } @@ -273,10 +274,10 @@ where T: Copy + Into, { fn can_derive_eq(&self, ctx: &BindgenContext) -> bool { - ctx.options().derive_eq && - ctx.lookup_can_derive_partialeq_or_partialord(*self) == - CanDerive::Yes && - !ctx.lookup_has_float(*self) + ctx.options().derive_eq + && ctx.lookup_can_derive_partialeq_or_partialord(*self) + == CanDerive::Yes + && !ctx.lookup_has_float(*self) } } @@ -285,10 +286,10 @@ where T: Copy + Into, { fn can_derive_ord(&self, ctx: &BindgenContext) -> bool { - ctx.options().derive_ord && - ctx.lookup_can_derive_partialeq_or_partialord(*self) == - CanDerive::Yes && - !ctx.lookup_has_float(*self) + ctx.options().derive_ord + && ctx.lookup_can_derive_partialeq_or_partialord(*self) + == CanDerive::Yes + && !ctx.lookup_has_float(*self) } } @@ -376,6 +377,9 @@ pub(crate) struct BindgenContext { /// The translation unit for parsing. translation_unit: clang::TranslationUnit, + /// The translation unit for macro fallback parsing. + fallback_tu: Option, + /// Target information that can be useful for some stuff. target_info: clang::TargetInfo, @@ -584,6 +588,7 @@ If you encounter an error missing from this list, please file an issue or a PR!" collected_typerefs: false, in_codegen: false, translation_unit, + fallback_tu: None, target_info, options, generated_bindgen_complex: Cell::new(false), @@ -696,11 +701,11 @@ If you encounter an error missing from this list, please file an issue or a PR!" item, declaration, location ); debug_assert!( - declaration.is_some() || - !item.kind().is_type() || - item.kind().expect_type().is_builtin_or_type_param() || - item.kind().expect_type().is_opaque(self, &item) || - item.kind().expect_type().is_unresolved_ref(), + declaration.is_some() + || !item.kind().is_type() + || item.kind().expect_type().is_builtin_or_type_param() + || item.kind().expect_type().is_opaque(self, &item) + || item.kind().expect_type().is_unresolved_ref(), "Adding a type without declaration?" ); @@ -1079,10 +1084,10 @@ If you encounter an error missing from this list, please file an issue or a PR!" }; match *ty.kind() { - TypeKind::Comp(..) | - TypeKind::TemplateAlias(..) | - TypeKind::Enum(..) | - TypeKind::Alias(..) => {} + TypeKind::Comp(..) + | TypeKind::TemplateAlias(..) + | TypeKind::Enum(..) + | TypeKind::Alias(..) => {} _ => continue, } @@ -1685,9 +1690,9 @@ If you encounter an error missing from this list, please file an issue or a PR!" for child in children.iter().rev() { match child.kind() { - clang_sys::CXCursor_TypeRef | - clang_sys::CXCursor_TypedefDecl | - clang_sys::CXCursor_TypeAliasDecl => { + clang_sys::CXCursor_TypeRef + | clang_sys::CXCursor_TypedefDecl + | clang_sys::CXCursor_TypeAliasDecl => { // The `with_id` ID will potentially end up unused if we give up // on this type (for example, because it has const value // template args), so if we pass `with_id` as the parent, it is @@ -1711,8 +1716,8 @@ If you encounter an error missing from this list, please file an issue or a PR!" child, )?; - if num_expected_template_args == 0 || - child.has_at_least_num_children( + if num_expected_template_args == 0 + || child.has_at_least_num_children( num_expected_template_args, ) { @@ -1890,8 +1895,8 @@ If you encounter an error missing from this list, please file an issue or a PR!" // * we have already parsed and resolved this type, and // there's nothing left to do. if let Some(location) = location { - if decl.cursor().is_template_like() && - *ty != decl.cursor().cur_type() + if decl.cursor().is_template_like() + && *ty != decl.cursor().cur_type() { // For specialized type aliases, there's no way to get the // template parameters as of this writing (for a struct @@ -1903,10 +1908,10 @@ If you encounter an error missing from this list, please file an issue or a PR!" // exposed. // // This is _tricky_, I know :( - if decl.cursor().kind() == - CXCursor_TypeAliasTemplateDecl && - !location.contains_cursor(CXCursor_TypeRef) && - ty.canonical_type().is_valid_and_exposed() + if decl.cursor().kind() + == CXCursor_TypeAliasTemplateDecl + && !location.contains_cursor(CXCursor_TypeRef) + && ty.canonical_type().is_valid_and_exposed() { return None; } @@ -2060,6 +2065,76 @@ If you encounter an error missing from this list, please file an issue or a PR!" &self.translation_unit } + /// Initialize fallback translation unit if it does not exist and + /// then return a mutable reference to the fallback translation unit. + pub(crate) fn try_ensure_fallback_translation_unit( + &mut self, + ) -> Option<&mut clang::FallbackTranslationUnit> { + if self.fallback_tu.is_none() { + let file = format!( + "{}/.macro_eval.c", + match self.options().clang_macro_fallback_build_dir { + Some(ref path) => path.as_os_str().to_str()?, + None => ".", + } + ); + + let index = clang::Index::new(false, false); + + let mut c_args = Vec::new(); + for input_header in self.options().input_headers.iter() { + let path = Path::new(input_header.as_ref()); + let header_name = path + .file_name() + .and_then(|hn| hn.to_str()) + .map(|s| s.to_owned()); + let header_path = path + .parent() + .and_then(|hp| hp.to_str()) + .map(|s| s.to_owned()); + + let (header, pch) = if let (Some(ref hp), Some(hn)) = + (header_path, header_name) + { + let header_path = if hp.is_empty() { "." } else { hp }; + let header = format!("{header_path}/{hn}"); + let pch_path = if let Some(ref path) = + self.options().clang_macro_fallback_build_dir + { + path.as_os_str().to_str()? + } else { + header_path + }; + (header, format!("{pch_path}/{hn}.pch")) + } else { + return None; + }; + + if !Path::new(&pch).exists() { + let mut tu = clang::TranslationUnit::parse( + &index, + &header, + &[ + "-x".to_owned().into_boxed_str(), + "c-header".to_owned().into_boxed_str(), + ], + &[], + clang_sys::CXTranslationUnit_ForSerialization, + )?; + tu.save(&pch).ok()?; + } + + c_args.push("-include-pch".to_string().into_boxed_str()); + c_args.push(pch.into_boxed_str()); + } + + self.fallback_tu = + Some(clang::FallbackTranslationUnit::new(file, &c_args)?); + } + + self.fallback_tu.as_mut() + } + /// Have we parsed the macro named `macro_name` already? pub(crate) fn parsed_macro(&self, macro_name: &[u8]) -> bool { self.parsed_macros.contains_key(macro_name) @@ -2320,9 +2395,9 @@ If you encounter an error missing from this list, please file an issue or a PR!" /// Is the given type a type from that corresponds to a Rust primitive type? pub(crate) fn is_stdint_type(&self, name: &str) -> bool { match name { - "int8_t" | "uint8_t" | "int16_t" | "uint16_t" | "int32_t" | - "uint32_t" | "int64_t" | "uint64_t" | "uintptr_t" | - "intptr_t" | "ptrdiff_t" => true, + "int8_t" | "uint8_t" | "int16_t" | "uint16_t" | "int32_t" + | "uint32_t" | "int64_t" | "uint64_t" | "uintptr_t" + | "intptr_t" | "ptrdiff_t" => true, "size_t" | "ssize_t" => self.options.size_t_is_usize, _ => false, } @@ -2350,11 +2425,11 @@ If you encounter an error missing from this list, please file an issue or a PR!" .filter(|&(_, item)| { // If nothing is explicitly allowlisted, then everything is fair // game. - if self.options().allowlisted_types.is_empty() && - self.options().allowlisted_functions.is_empty() && - self.options().allowlisted_vars.is_empty() && - self.options().allowlisted_files.is_empty() && - self.options().allowlisted_items.is_empty() + if self.options().allowlisted_types.is_empty() + && self.options().allowlisted_functions.is_empty() + && self.options().allowlisted_vars.is_empty() + && self.options().allowlisted_files.is_empty() + && self.options().allowlisted_items.is_empty() { return true; } @@ -2407,19 +2482,19 @@ If you encounter an error missing from this list, please file an issue or a PR!" // make the #[derive] analysis not be lame. if !self.options().allowlist_recursively { match *ty.kind() { - TypeKind::Void | - TypeKind::NullPtr | - TypeKind::Int(..) | - TypeKind::Float(..) | - TypeKind::Complex(..) | - TypeKind::Array(..) | - TypeKind::Vector(..) | - TypeKind::Pointer(..) | - TypeKind::Reference(..) | - TypeKind::Function(..) | - TypeKind::ResolvedTypeRef(..) | - TypeKind::Opaque | - TypeKind::TypeParam => return true, + TypeKind::Void + | TypeKind::NullPtr + | TypeKind::Int(..) + | TypeKind::Float(..) + | TypeKind::Complex(..) + | TypeKind::Array(..) + | TypeKind::Vector(..) + | TypeKind::Pointer(..) + | TypeKind::Reference(..) + | TypeKind::Function(..) + | TypeKind::ResolvedTypeRef(..) + | TypeKind::Opaque + | TypeKind::TypeParam => return true, _ => {} } if self.is_stdint_type(&name) { @@ -2717,9 +2792,9 @@ If you encounter an error missing from this list, please file an issue or a PR!" fn compute_cannot_derive_partialord_partialeq_or_eq(&mut self) { let _t = self.timer("compute_cannot_derive_partialord_partialeq_or_eq"); assert!(self.cannot_derive_partialeq_or_partialord.is_none()); - if self.options.derive_partialord || - self.options.derive_partialeq || - self.options.derive_eq + if self.options.derive_partialord + || self.options.derive_partialeq + || self.options.derive_eq { self.cannot_derive_partialeq_or_partialord = Some(analyze::(( @@ -2766,8 +2841,8 @@ If you encounter an error missing from this list, please file an issue or a PR!" // derive `Copy` or not. let id = id.into(); - !self.lookup_has_type_param_in_array(id) && - !self.cannot_derive_copy.as_ref().unwrap().contains(&id) + !self.lookup_has_type_param_in_array(id) + && !self.cannot_derive_copy.as_ref().unwrap().contains(&id) } /// Compute whether the type has type parameter in array. @@ -2994,15 +3069,15 @@ impl TemplateParameters for PartialType { // Wouldn't it be nice if libclang would reliably give us this // information‽ match self.decl().kind() { - clang_sys::CXCursor_ClassTemplate | - clang_sys::CXCursor_FunctionTemplate | - clang_sys::CXCursor_TypeAliasTemplateDecl => { + clang_sys::CXCursor_ClassTemplate + | clang_sys::CXCursor_FunctionTemplate + | clang_sys::CXCursor_TypeAliasTemplateDecl => { let mut num_params = 0; self.decl().visit(|c| { match c.kind() { - clang_sys::CXCursor_TemplateTypeParameter | - clang_sys::CXCursor_TemplateTemplateParameter | - clang_sys::CXCursor_NonTypeTemplateParameter => { + clang_sys::CXCursor_TemplateTypeParameter + | clang_sys::CXCursor_TemplateTemplateParameter + | clang_sys::CXCursor_NonTypeTemplateParameter => { num_params += 1; } _ => {} diff --git a/bindgen/ir/var.rs b/bindgen/ir/var.rs index a548ec881b..a970cf1588 100644 --- a/bindgen/ir/var.rs +++ b/bindgen/ir/var.rs @@ -389,9 +389,51 @@ impl ClangSubItemParser for Var { } } +/// This function uses a [`FallbackTranslationUnit`][clang::FallbackTranslationUnit] to parse each +/// macro that cannot be parsed by the normal bindgen process for `#define`s. +/// +/// To construct the [`FallbackTranslationUnit`][clang::FallbackTranslationUnit], first precompiled +/// headers are generated for all input headers. An empty temporary `.c` file is generated to pass +/// to the translation unit. On the evaluation of each macro, a [`String`] is generated with the +/// new contents of the empty file and passed in for reparsing. The precompiled headers and +/// preservation of the [`FallbackTranslationUnit`][clang::FallbackTranslationUnit] across macro +/// evaluations are both optimizations that have significantly improved the performance. +fn parse_macro_clang_fallback( + ctx: &mut BindgenContext, + cursor: &clang::Cursor, +) -> Option<(Vec, cexpr::expr::EvalResult)> { + if !ctx.options().clang_macro_fallback { + return None; + } + + let ftu = ctx.try_ensure_fallback_translation_unit()?; + let contents = format!("int main() {{ {}; }}", cursor.spelling(),); + ftu.reparse(&contents).ok()?; + // Children of root node of AST + let root_children = ftu.translation_unit().cursor().collect_children(); + // Last child in root is function declaration + // Should be FunctionDecl + let main_func = root_children.last()?; + // Children should all be statements in function declaration + let all_stmts = main_func.collect_children(); + // First child in all_stmts should be the statement containing the macro to evaluate + // Should be CompoundStmt + let macro_stmt = all_stmts.first()?; + // Children should all be expressions from the compound statement + let paren_exprs = macro_stmt.collect_children(); + // First child in all_exprs is the expression utilizing the given macro to be evaluated + // Should be ParenExpr + let paren = paren_exprs.first()?; + + Some(( + cursor.spelling().into_bytes(), + cexpr::expr::EvalResult::Int(Wrapping(paren.evaluate()?.as_int()?)), + )) +} + /// Try and parse a macro using all the macros parsed until now. fn parse_macro( - ctx: &BindgenContext, + ctx: &mut BindgenContext, cursor: &clang::Cursor, ) -> Option<(Vec, cexpr::expr::EvalResult)> { use cexpr::expr; @@ -402,7 +444,7 @@ fn parse_macro( match parser.macro_definition(&cexpr_tokens) { Ok((_, (id, val))) => Some((id.into(), val)), - _ => None, + _ => parse_macro_clang_fallback(ctx, cursor), } } diff --git a/bindgen/options/mod.rs b/bindgen/options/mod.rs index 3b3224caf0..dc453b13aa 100644 --- a/bindgen/options/mod.rs +++ b/bindgen/options/mod.rs @@ -21,9 +21,7 @@ use crate::HashMap; use crate::DEFAULT_ANON_FIELDS_PREFIX; use std::env; -#[cfg(feature = "experimental")] -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::rc::Rc; use as_args::AsArgs; @@ -2107,5 +2105,38 @@ options! { } }, as_args: "--emit-diagnostics", + }, + /// Whether to use Clang evaluation on temporary files as a fallback for macros that fail to + /// parse. + clang_macro_fallback: bool { + methods: { + /// Use Clang as a fallback for macros that fail to parse using `CExpr`. + /// + /// This uses a workaround to evaluate each macro in a temporary file. Because this + /// results in slower compilation, this option is opt-in. + pub fn clang_macro_fallback(mut self) -> Self { + self.options.clang_macro_fallback = true; + self + } + }, + as_args: "--clang-macro-fallback", + } + /// Path to use for temporary files created by clang macro fallback code like precompiled + /// headers. + clang_macro_fallback_build_dir: Option { + methods: { + /// Set a path to a directory to which `.c` and `.h.pch` files should be written for the + /// purpose of using clang to evaluate macros that can't be easily parsed. + /// + /// The default location for `.h.pch` files is the directory that the corresponding + /// `.h` file is located in. The default for the temporary `.c` file used for clang + /// parsing is the current working directory. Both of these defaults are overridden + /// by this option. + pub fn clang_macro_fallback_build_dir>(mut self, path: P) -> Self { + self.options.clang_macro_fallback_build_dir = Some(path.as_ref().to_owned()); + self + } + }, + as_args: "--clang-macro-fallback-build-dir", } }