Merge #3920

3920: Implement expand_task and list_macros in proc_macro_srv r=matklad a=edwin0cheng

This PR finish up the remain `proc_macro_srv` implementation :

1. Added dylib loading code for proc-macro crate dylib. Note that we have to add some special flags for unix loading because of a bug in old version of glibc, see fedochet/rust-proc-macro-panic-inside-panic-expample#1 and rust-lang/rust#60593 for details.

2. Added tests for proc-macro expansion: We use a trick here by adding `serde_derive` to dev-dependencies and calling `cargo-metadata` for searching its dylib path, and expand it in our tests. 

[EDIT]
Note that this PR **DO NOT** implement the final glue code with rust-analzyer and proc-macro-srv yet.
 

Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>

Author: bors[bot]

Cargo.lock

@@ -12,9 +12,12 @@ doctest = false
 ra_tt = { path = "../ra_tt" }
 ra_mbe = { path = "../ra_mbe" }
 ra_proc_macro = { path = "../ra_proc_macro" }
+goblin = "0.2.1"
+libloading = "0.6.0"
+test_utils = { path = "../test_utils" }
 
 [dev-dependencies]
 cargo_metadata = "0.9.1"
 difference = "2.0.0"
 # used as proc macro test target
-serde_derive = "=1.0.104"
+serde_derive = "=1.0.104"

crates/ra_proc_macro_srv/Cargo.toml

@@ -0,0 +1,211 @@
+//! Handles dynamic library loading for proc macro
+
+use crate::{proc_macro::bridge, rustc_server::TokenStream};
+use std::path::Path;
+
+use goblin::{mach::Mach, Object};
+use libloading::Library;
+use ra_proc_macro::ProcMacroKind;
+
+use std::io::Error as IoError;
+use std::io::ErrorKind as IoErrorKind;
+
+const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_";
+
+fn invalid_data_err(e: impl Into<Box<dyn std::error::Error + Send + Sync>>) -> IoError {
+    IoError::new(IoErrorKind::InvalidData, e)
+}
+
+fn get_symbols_from_lib(file: &Path) -> Result<Vec<String>, IoError> {
+    let buffer = std::fs::read(file)?;
+    let object = Object::parse(&buffer).map_err(invalid_data_err)?;
+
+    match object {
+        Object::Elf(elf) => {
+            let symbols = elf.dynstrtab.to_vec().map_err(invalid_data_err)?;
+            let names = symbols.iter().map(|s| s.to_string()).collect();
+            Ok(names)
+        }
+        Object::PE(pe) => {
+            let symbol_names =
+                pe.exports.iter().flat_map(|s| s.name).map(|n| n.to_string()).collect();
+            Ok(symbol_names)
+        }
+        Object::Mach(mach) => match mach {
+            Mach::Binary(binary) => {
+                let exports = binary.exports().map_err(invalid_data_err)?;
+                let names = exports
+                    .into_iter()
+                    .map(|s| {
+                        // In macos doc:
+                        // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dlsym.3.html
+                        // Unlike other dyld API's, the symbol name passed to dlsym() must NOT be
+                        // prepended with an underscore.
+                        if s.name.starts_with("_") {
+                            s.name[1..].to_string()
+                        } else {
+                            s.name
+                        }
+                    })
+                    .collect();
+                Ok(names)
+            }
+            Mach::Fat(_) => Ok(vec![]),
+        },
+        Object::Archive(_) | Object::Unknown(_) => Ok(vec![]),
+    }
+}
+
+fn is_derive_registrar_symbol(symbol: &str) -> bool {
+    symbol.contains(NEW_REGISTRAR_SYMBOL)
+}
+
+fn find_registrar_symbol(file: &Path) -> Result<Option<String>, IoError> {
+    let symbols = get_symbols_from_lib(file)?;
+    Ok(symbols.into_iter().find(|s| is_derive_registrar_symbol(s)))
+}
+
+/// Loads dynamic library in platform dependent manner.
+///
+/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described
+/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample)
+/// and [here](https://github.com/rust-lang/rust/issues/60593).
+///
+/// Usage of RTLD_DEEPBIND
+/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1)
+///
+/// It seems that on Windows that behaviour is default, so we do nothing in that case.
+#[cfg(windows)]
+fn load_library(file: &Path) -> Result<Library, libloading::Error> {
+    Library::new(file)
+}
+
+#[cfg(unix)]
+fn load_library(file: &Path) -> Result<Library, libloading::Error> {
+    use libloading::os::unix::Library as UnixLibrary;
+    use std::os::raw::c_int;
+
+    const RTLD_NOW: c_int = 0x00002;
+    const RTLD_DEEPBIND: c_int = 0x00008;
+
+    UnixLibrary::open(Some(file), RTLD_NOW | RTLD_DEEPBIND).map(|lib| lib.into())
+}
+
+struct ProcMacroLibraryLibloading {
+    // Hold the dylib to prevent it for unloadeding
+    _lib: Library,
+    exported_macros: Vec<bridge::client::ProcMacro>,
+}
+
+impl ProcMacroLibraryLibloading {
+    fn open(file: &Path) -> Result<Self, IoError> {
+        let symbol_name = find_registrar_symbol(file)?
+            .ok_or(invalid_data_err(format!("Cannot find registrar symbol in file {:?}", file)))?;
+
+        let lib = load_library(file).map_err(invalid_data_err)?;
+        let exported_macros = {
+            let macros: libloading::Symbol<&&[bridge::client::ProcMacro]> =
+                unsafe { lib.get(symbol_name.as_bytes()) }.map_err(invalid_data_err)?;
+            macros.to_vec()
+        };
+
+        Ok(ProcMacroLibraryLibloading { _lib: lib, exported_macros })
+    }
+}
+
+type ProcMacroLibraryImpl = ProcMacroLibraryLibloading;
+
+pub struct Expander {
+    libs: Vec<ProcMacroLibraryImpl>,
+}
+
+impl Expander {
+    pub fn new<P: AsRef<Path>>(lib: &P) -> Result<Expander, String> {
+        let mut libs = vec![];
+        /* Some libraries for dynamic loading require canonicalized path (even when it is
+        already absolute
+        */
+        let lib =
+            lib.as_ref().canonicalize().expect(&format!("Cannot canonicalize {:?}", lib.as_ref()));
+
+        let library = ProcMacroLibraryImpl::open(&lib).map_err(|e| e.to_string())?;
+        libs.push(library);
+
+        Ok(Expander { libs })
+    }
+
+    pub fn expand(
+        &self,
+        macro_name: &str,
+        macro_body: &ra_tt::Subtree,
+        attributes: Option<&ra_tt::Subtree>,
+    ) -> Result<ra_tt::Subtree, bridge::PanicMessage> {
+        let parsed_body = TokenStream::with_subtree(macro_body.clone());
+
+        let parsed_attributes = attributes
+            .map_or(crate::rustc_server::TokenStream::new(), |attr| {
+                TokenStream::with_subtree(attr.clone())
+            });
+
+        for lib in &self.libs {
+            for proc_macro in &lib.exported_macros {
+                match proc_macro {
+                    bridge::client::ProcMacro::CustomDerive { trait_name, client, .. }
+                        if *trait_name == macro_name =>
+                    {
+                        let res = client.run(
+                            &crate::proc_macro::bridge::server::SameThread,
+                            crate::rustc_server::Rustc::default(),
+                            parsed_body,
+                        );
+                        return res.map(|it| it.subtree);
+                    }
+                    bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => {
+                        let res = client.run(
+                            &crate::proc_macro::bridge::server::SameThread,
+                            crate::rustc_server::Rustc::default(),
+                            parsed_body,
+                        );
+                        return res.map(|it| it.subtree);
+                    }
+                    bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => {
+                        let res = client.run(
+                            &crate::proc_macro::bridge::server::SameThread,
+                            crate::rustc_server::Rustc::default(),
+                            parsed_attributes,
+                            parsed_body,
+                        );
+
+                        return res.map(|it| it.subtree);
+                    }
+                    _ => continue,
+                }
+            }
+        }
+
+        Err(bridge::PanicMessage::String("Nothing to expand".to_string()))
+    }
+
+    pub fn list_macros(&self) -> Result<Vec<(String, ProcMacroKind)>, bridge::PanicMessage> {
+        let mut result = vec![];
+
+        for lib in &self.libs {
+            for proc_macro in &lib.exported_macros {
+                let res = match proc_macro {
+                    bridge::client::ProcMacro::CustomDerive { trait_name, .. } => {
+                        (trait_name.to_string(), ProcMacroKind::CustomDerive)
+                    }
+                    bridge::client::ProcMacro::Bang { name, .. } => {
+                        (name.to_string(), ProcMacroKind::FuncLike)
+                    }
+                    bridge::client::ProcMacro::Attr { name, .. } => {
+                        (name.to_string(), ProcMacroKind::Attr)
+                    }
+                };
+                result.push(res);
+            }
+        }
+
+        Ok(result)
+    }
+}

crates/ra_proc_macro_srv/src/dylib.rs

@@ -17,13 +17,41 @@ mod proc_macro;
 #[doc(hidden)]
 mod rustc_server;
 
+mod dylib;
+
 use proc_macro::bridge::client::TokenStream;
 use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask};
 
-pub fn expand_task(_task: &ExpansionTask) -> Result<ExpansionResult, String> {
-    unimplemented!()
+pub fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
+    let expander = dylib::Expander::new(&task.lib)
+        .expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib));
+
+    match expander.expand(&task.macro_name, &task.macro_body, task.attributes.as_ref()) {
+        Ok(expansion) => Ok(ExpansionResult { expansion }),
+        Err(msg) => {
+            let reason = format!(
+                "Cannot perform expansion for {}: error {:?}!",
+                &task.macro_name,
+                msg.as_str()
+            );
+            Err(reason)
+        }
+    }
 }
 
-pub fn list_macros(_task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
-    unimplemented!()
+pub fn list_macros(task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
+    let expander = dylib::Expander::new(&task.lib)
+        .expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib));
+
+    match expander.list_macros() {
+        Ok(macros) => Ok(ListMacrosResult { macros }),
+        Err(msg) => {
+            let reason =
+                format!("Cannot perform expansion for {:?}: error {:?}!", &task.lib, msg.as_str());
+            Err(reason)
+        }
+    }
 }
+
+#[cfg(test)]
+mod tests;

crates/ra_proc_macro_srv/src/lib.rs

@@ -34,6 +34,10 @@ impl TokenStream {
         TokenStream { subtree: Default::default() }
     }
 
+    pub fn with_subtree(subtree: tt::Subtree) -> Self {
+        TokenStream { subtree }
+    }
+
     pub fn is_empty(&self) -> bool {
         self.subtree.token_trees.is_empty()
     }