Skip to content

Commit 6b63e9b

Browse files
petrochenkovpetrochenkov
committed
Do not remove tokens before AST json serialization
1 parent 4f7612a commit 6b63e9b

10 files changed

+30
-145
lines changed

‎compiler/rustc_ast/src/tokenstream.rs

+3-2
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,9 @@ impl fmt::Debug for LazyTokenStream {
153153
}
154154

155155
impl<S: Encoder> Encodable<S> for LazyTokenStream {
156-
fn encode(&self, _s: &mut S) -> Result<(), S::Error> {
157-
panic!("Attempted to encode LazyTokenStream");
156+
fn encode(&self, s: &mut S) -> Result<(), S::Error> {
157+
// Used by AST json printing.
158+
Encodable::encode(&self.create_token_stream(), s)
158159
}
159160
}
160161

‎compiler/rustc_interface/src/passes.rs

+2-88
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@ use crate::interface::{Compiler, Result};
22
use crate::proc_macro_decls;
33
use crate::util;
44

5-
use rustc_ast::mut_visit::{self, MutVisitor};
6-
use rustc_ast::ptr::P;
7-
use rustc_ast::{self as ast, token, visit};
5+
use rustc_ast::mut_visit::MutVisitor;
6+
use rustc_ast::{self as ast, visit};
87
use rustc_codegen_ssa::back::link::emit_metadata;
98
use rustc_codegen_ssa::traits::CodegenBackend;
109
use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal};
@@ -37,7 +36,6 @@ use rustc_span::symbol::Symbol;
3736
use rustc_span::{FileName, RealFileName};
3837
use rustc_trait_selection::traits;
3938
use rustc_typeck as typeck;
40-
use smallvec::SmallVec;
4139
use tracing::{info, warn};
4240

4341
use rustc_serialize::json;
@@ -52,82 +50,6 @@ use std::path::PathBuf;
5250
use std::rc::Rc;
5351
use std::{env, fs, iter, mem};
5452

55-
/// Remove alls `LazyTokenStreams` from an AST struct
56-
/// Normally, this is done during AST lowering. However,
57-
/// printing the AST JSON requires us to serialize
58-
/// the entire AST, and we don't want to serialize
59-
/// a `LazyTokenStream`.
60-
struct TokenStripper;
61-
impl mut_visit::MutVisitor for TokenStripper {
62-
fn flat_map_item(&mut self, mut i: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
63-
i.tokens = None;
64-
mut_visit::noop_flat_map_item(i, self)
65-
}
66-
fn flat_map_foreign_item(
67-
&mut self,
68-
mut i: P<ast::ForeignItem>,
69-
) -> SmallVec<[P<ast::ForeignItem>; 1]> {
70-
i.tokens = None;
71-
mut_visit::noop_flat_map_foreign_item(i, self)
72-
}
73-
fn flat_map_trait_item(
74-
&mut self,
75-
mut i: P<ast::AssocItem>,
76-
) -> SmallVec<[P<ast::AssocItem>; 1]> {
77-
i.tokens = None;
78-
mut_visit::noop_flat_map_assoc_item(i, self)
79-
}
80-
fn flat_map_impl_item(&mut self, mut i: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
81-
i.tokens = None;
82-
mut_visit::noop_flat_map_assoc_item(i, self)
83-
}
84-
fn visit_block(&mut self, b: &mut P<ast::Block>) {
85-
b.tokens = None;
86-
mut_visit::noop_visit_block(b, self);
87-
}
88-
fn flat_map_stmt(&mut self, mut stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
89-
stmt.tokens = None;
90-
mut_visit::noop_flat_map_stmt(stmt, self)
91-
}
92-
fn visit_pat(&mut self, p: &mut P<ast::Pat>) {
93-
p.tokens = None;
94-
mut_visit::noop_visit_pat(p, self);
95-
}
96-
fn visit_ty(&mut self, ty: &mut P<ast::Ty>) {
97-
ty.tokens = None;
98-
mut_visit::noop_visit_ty(ty, self);
99-
}
100-
fn visit_attribute(&mut self, attr: &mut ast::Attribute) {
101-
attr.tokens = None;
102-
if let ast::AttrKind::Normal(ast::AttrItem { tokens, .. }) = &mut attr.kind {
103-
*tokens = None;
104-
}
105-
mut_visit::noop_visit_attribute(attr, self);
106-
}
107-
108-
fn visit_interpolated(&mut self, nt: &mut token::Nonterminal) {
109-
if let token::Nonterminal::NtMeta(meta) = nt {
110-
meta.tokens = None;
111-
}
112-
// Handles all of the other cases
113-
mut_visit::noop_visit_interpolated(nt, self);
114-
}
115-
116-
fn visit_path(&mut self, p: &mut ast::Path) {
117-
p.tokens = None;
118-
mut_visit::noop_visit_path(p, self);
119-
}
120-
fn visit_vis(&mut self, vis: &mut ast::Visibility) {
121-
vis.tokens = None;
122-
mut_visit::noop_visit_vis(vis, self);
123-
}
124-
fn visit_expr(&mut self, e: &mut P<ast::Expr>) {
125-
e.tokens = None;
126-
mut_visit::noop_visit_expr(e, self);
127-
}
128-
fn visit_mac(&mut self, _mac: &mut ast::MacCall) {}
129-
}
130-
13153
pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
13254
let krate = sess.time("parse_crate", || match input {
13355
Input::File(file) => parse_crate_from_file(file, &sess.parse_sess),
@@ -137,10 +59,6 @@ pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
13759
})?;
13860

13961
if sess.opts.debugging_opts.ast_json_noexpand {
140-
// Set any `token` fields to `None` before
141-
// we display the AST.
142-
let mut krate = krate.clone();
143-
TokenStripper.visit_crate(&mut krate);
14462
println!("{}", json::as_json(&krate));
14563
}
14664

@@ -461,10 +379,6 @@ fn configure_and_expand_inner<'a>(
461379
}
462380

463381
if sess.opts.debugging_opts.ast_json {
464-
// Set any `token` fields to `None` before
465-
// we display the AST.
466-
let mut krate = krate.clone();
467-
TokenStripper.visit_crate(&mut krate);
468382
println!("{}", json::as_json(&krate));
469383
}
470384

‎src/test/ui/ast-json/ast-json-ice.rs

+23
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,26 @@ fn main() {
3939

4040
struct A;
4141
}
42+
43+
// Regressions tests for issues #78398 and #78510 (captured tokens in associated and foreign items)
44+
45+
struct S;
46+
47+
macro_rules! mac_extern {
48+
($i:item) => {
49+
extern "C" { $i }
50+
}
51+
}
52+
macro_rules! mac_assoc {
53+
($i:item) => {
54+
impl S { $i }
55+
trait Bar { $i }
56+
}
57+
}
58+
59+
mac_extern! {
60+
fn foo();
61+
}
62+
mac_assoc! {
63+
fn foo() {}
64+
}

‎src/test/ui/ast-json/ast-json-noexpand-output.stdout

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":null}],"span":{"lo":0,"hi":0},"proc_macros":[]}
1+
{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

‎src/test/ui/ast-json/ast-json-output.stdout

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":null}],"span":{"lo":0,"hi":0},"proc_macros":[]}
1+
{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

‎src/test/ui/ast-json/issue-78398-foreign-ice.rs

-18
This file was deleted.

‎src/test/ui/ast-json/issue-78398-foreign-ice.stdout

-1
This file was deleted.

‎src/test/ui/ast-json/issue-78510-assoc-ice.rs

-18
This file was deleted.

‎src/test/ui/ast-json/issue-78510-assoc-ice.stderr

-15
This file was deleted.

0 commit comments

Comments
 (0)