Represent lifetimes as two joint tokens in proc macros

Author: petrochenkov

src/libproc_macro/lib.rs

@@ -487,7 +487,7 @@ impl PartialEq<FileName> for SourceFile {
 pub enum TokenTree {
     /// A token stream surrounded by bracket delimiters.
     Group(Group),
-    /// An identifier or lifetime identifier.
+    /// An identifier.
     Ident(Ident),
     /// A single punctuation character (`+`, `,`, `$`, etc.).
     Punct(Punct),
@@ -702,9 +702,10 @@ impl !Sync for Punct {}
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 #[unstable(feature = "proc_macro", issue = "38356")]
 pub enum Spacing {
-    /// e.g. `+` is `Alone` in `+ =`, `+ident` or `+()`.
+    /// E.g. `+` is `Alone` in `+ =`, `+ident` or `+()`.
     Alone,
-    /// e.g. `+` is `Joint` in `+=` or `+#`.
+    /// E.g. `+` is `Joint` in `+=` or `'#`.
+    /// Additionally, single quote `'` can join with identifiers to form lifetimes `'ident`.
     Joint,
 }
 
@@ -717,8 +718,8 @@ impl Punct {
     /// which can be further configured with the `set_span` method below.
     #[unstable(feature = "proc_macro", issue = "38356")]
     pub fn new(ch: char, spacing: Spacing) -> Punct {
-        const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%',
-                                       '^', '&', '|', '@', '.', ',', ';', ':', '#', '$', '?'];
+        const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^',
+                                       '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\''];
         if !LEGAL_CHARS.contains(&ch) {
             panic!("unsupported character `{:?}`", ch)
         }
@@ -766,7 +767,7 @@ impl fmt::Display for Punct {
     }
 }
 
-/// An identifier (`ident`) or lifetime identifier (`'ident`).
+/// An identifier (`ident`).
 #[derive(Clone, Debug)]
 #[unstable(feature = "proc_macro", issue = "38356")]
 pub struct Ident {
@@ -783,7 +784,7 @@ impl !Sync for Ident {}
 impl Ident {
     /// Creates a new `Ident` with the given `string` as well as the specified
     /// `span`.
-    /// The `string` argument must be a valid identifier or lifetime identifier permitted by the
+    /// The `string` argument must be a valid identifier permitted by the
     /// language, otherwise the function will panic.
     ///
     /// Note that `span`, currently in rustc, configures the hygiene information
@@ -817,8 +818,7 @@ impl Ident {
     pub fn new_raw(string: &str, span: Span) -> Ident {
         let mut ident = Ident::new(string, span);
         if ident.sym == keywords::Underscore.name() ||
-           token::is_path_segment_keyword(ast::Ident::with_empty_ctxt(ident.sym)) ||
-           ident.sym.as_str().starts_with("\'") {
+           token::is_path_segment_keyword(ast::Ident::with_empty_ctxt(ident.sym)) {
             panic!("`{:?}` is not a valid raw identifier", string)
         }
         ident.is_raw = true;
@@ -1211,13 +1211,19 @@ impl TokenTree {
             Pound => op!('#'),
             Dollar => op!('$'),
             Question => op!('?'),
+            SingleQuote => op!('\''),
 
-            Ident(ident, false) | Lifetime(ident) => {
+            Ident(ident, false) => {
                 tt!(self::Ident::new(&ident.name.as_str(), Span(span)))
             }
             Ident(ident, true) => {
                 tt!(self::Ident::new_raw(&ident.name.as_str(), Span(span)))
             }
+            Lifetime(ident) => {
+                let ident = ident.without_first_quote();
+                stack.push(tt!(self::Ident::new(&ident.name.as_str(), Span(span))));
+                tt!(Punct::new('\'', Spacing::Joint))
+            }
             Literal(lit, suffix) => tt!(self::Literal { lit, suffix, span: Span(span) }),
             DocComment(c) => {
                 let style = comments::doc_comment_style(&c.as_str());
@@ -1260,12 +1266,7 @@ impl TokenTree {
                 }).into();
             },
             self::TokenTree::Ident(tt) => {
-                let ident = ast::Ident::new(tt.sym, tt.span.0);
-                let token = if tt.sym.as_str().starts_with("'") {
-                    Lifetime(ident)
-                } else {
-                    Ident(ident, tt.is_raw)
-                };
+                let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw);
                 return TokenTree::Token(tt.span.0, token).into();
             }
             self::TokenTree::Literal(self::Literal {
@@ -1324,6 +1325,7 @@ impl TokenTree {
             '#' => Pound,
             '$' => Dollar,
             '?' => Question,
+            '\'' => SingleQuote,
             _ => unreachable!(),
         };
 

src/librustc/ich/impls_syntax.rs

@@ -314,6 +314,7 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
         token::Token::Pound |
         token::Token::Dollar |
         token::Token::Question |
+        token::Token::SingleQuote |
         token::Token::Whitespace |
         token::Token::Comment |
         token::Token::Eof => {}

src/librustdoc/html/highlight.rs

@@ -353,7 +353,7 @@ impl<'a> Classifier<'a> {
             token::Lifetime(..) => Class::Lifetime,
 
             token::Eof | token::Interpolated(..) |
-            token::Tilde | token::At | token::DotEq => Class::None,
+            token::Tilde | token::At | token::DotEq | token::SingleQuote => Class::None,
         };
 
         // Anything that didn't return above is the simple case where we the

src/libsyntax/ext/quote.rs

@@ -711,6 +711,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
         token::Pound        => "Pound",
         token::Dollar       => "Dollar",
         token::Question     => "Question",
+        token::SingleQuote  => "SingleQuote",
         token::Eof          => "Eof",
 
         token::Whitespace | token::Comment | token::Shebang(_) => {

src/libsyntax/parse/lexer/mod.rs

@@ -1773,10 +1773,7 @@ fn ident_continue(c: Option<char>) -> bool {
 // The string is a valid identifier or a lifetime identifier.
 pub fn is_valid_ident(s: &str) -> bool {
     let mut chars = s.chars();
-    match chars.next() {
-        Some('\'') => ident_start(chars.next()) && chars.all(|ch| ident_continue(Some(ch))),
-        ch => ident_start(ch) && chars.all(|ch| ident_continue(Some(ch)))
-    }
+    ident_start(chars.next()) && chars.all(|ch| ident_continue(Some(ch)))
 }
 
 #[cfg(test)]

src/libsyntax/parse/token.rs

@@ -210,6 +210,8 @@ pub enum Token {
     Pound,
     Dollar,
     Question,
+    /// Used by proc macros for representing lifetimes, not generated by lexer right now.
+    SingleQuote,
     /// An opening delimiter, eg. `{`
     OpenDelim(DelimToken),
     /// A closing delimiter, eg. `}`
@@ -513,6 +515,10 @@ impl Token {
                 Colon => ModSep,
                 _ => return None,
             },
+            SingleQuote => match joint {
+                Ident(ident, false) => Lifetime(ident),
+                _ => return None,
+            },
 
             Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotEq |
             DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar |

src/libsyntax/print/pprust.rs

@@ -224,6 +224,7 @@ pub fn token_to_string(tok: &Token) -> String {
         token::Pound                => "#".to_string(),
         token::Dollar               => "$".to_string(),
         token::Question             => "?".to_string(),
+        token::SingleQuote          => "'".to_string(),
 
         /* Literals */
         token::Literal(lit, suf) => {

src/test/run-pass-fulldeps/proc-macro/auxiliary/lifetimes.rs

@@ -0,0 +1,36 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+
+#[proc_macro]
+pub fn lifetimes_bang(input: TokenStream) -> TokenStream {
+    // Roundtrip through token trees
+    input.into_iter().collect()
+}
+
+#[proc_macro_attribute]
+pub fn lifetimes_attr(_: TokenStream, input: TokenStream) -> TokenStream {
+    // Roundtrip through AST
+    input
+}
+
+#[proc_macro_derive(Lifetimes)]
+pub fn lifetimes_derive(input: TokenStream) -> TokenStream {
+    // Roundtrip through a string
+    format!("mod m {{ {} }}", input).parse().unwrap()
+}

src/test/run-pass-fulldeps/proc-macro/lifetimes.rs

@@ -0,0 +1,36 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:lifetimes.rs
+// ignore-stage1
+
+#![feature(proc_macro)]
+
+extern crate lifetimes;
+use lifetimes::*;
+
+lifetimes_bang! {
+    fn bang<'a>() -> &'a u8 { &0 }
+}
+
+#[lifetimes_attr]
+fn attr<'a>() -> &'a u8 { &1 }
+
+#[derive(Lifetimes)]
+pub struct Lifetimes<'a> {
+    pub field: &'a u8,
+}
+
+fn main() {
+    assert_eq!(bang::<'static>(), &0);
+    assert_eq!(attr::<'static>(), &1);
+    let l1 = Lifetimes { field: &0 };
+    let l2 = m::Lifetimes { field: &1 };
+}

src/test/ui-fulldeps/auxiliary/lifetimes.rs

@@ -0,0 +1,30 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+
+#[proc_macro]
+pub fn single_quote_alone(_: TokenStream) -> TokenStream {
+    // `&'a u8`, but the `'` token is not joint
+    let trees: Vec<TokenTree> = vec![
+        Punct::new('&', Spacing::Alone).into(),
+        Punct::new('\'', Spacing::Alone).into(),
+        Ident::new("a", Span::call_site()).into(),
+        Ident::new("u8", Span::call_site()).into(),
+    ];
+    trees.into_iter().collect()
+}

src/test/ui-fulldeps/lifetimes.rs

@@ -0,0 +1,19 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:lifetimes.rs
+
+#![feature(proc_macro, proc_macro_non_items)]
+
+extern crate lifetimes;
+
+use lifetimes::*;
+
+type A = single_quote_alone!(); //~ ERROR expected type, found `'`

src/test/ui-fulldeps/lifetimes.stderr

@@ -0,0 +1,8 @@
+error: expected type, found `'`
+  --> $DIR/lifetimes.rs:19:10
+   |
+LL | type A = single_quote_alone!(); //~ ERROR expected type, found `'`
+   |          ^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+