Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make ast::TokenKind more like lexer::TokenKind #137902

Merged
merged 2 commits into from
Mar 4, 2025
Merged

Make ast::TokenKind more like lexer::TokenKind #137902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a1e2e9
Replace `ast::TokenKind::BinOp{,Eq}` and remove `BinOpToken`.
nnethercote Dec 19, 2024
53167c0
Rename `ast::TokenKind::Not` as `ast::TokenKind::Bang`.
nnethercote Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
296 changes: 170 additions & 126 deletions compiler/rustc_ast/src/token.rs
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion compiler/rustc_ast/src/tokenstream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,7 @@ impl TokenStream {
if attr_style == AttrStyle::Inner {
vec![
TokenTree::token_joint(token::Pound, span),
TokenTree::token_joint_hidden(token::Not, span),
TokenTree::token_joint_hidden(token::Bang, span),
body,
]
} else {
Expand Down
42 changes: 21 additions & 21 deletions compiler/rustc_ast/src/util/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use rustc_span::kw;

use crate::ast::{self, BinOpKind, RangeLimits};
use crate::token::{self, BinOpToken, Token};
use crate::token::{self, Token};

/// Associative operator.
#[derive(Copy, Clone, PartialEq, Debug)]
Expand Down Expand Up @@ -34,26 +34,26 @@ impl AssocOp {
use AssocOp::*;
match t.kind {
token::Eq => Some(Assign),
token::BinOp(BinOpToken::Plus) => Some(Binary(BinOpKind::Add)),
token::BinOp(BinOpToken::Minus) => Some(Binary(BinOpKind::Sub)),
token::BinOp(BinOpToken::Star) => Some(Binary(BinOpKind::Mul)),
token::BinOp(BinOpToken::Slash) => Some(Binary(BinOpKind::Div)),
token::BinOp(BinOpToken::Percent) => Some(Binary(BinOpKind::Rem)),
token::BinOp(BinOpToken::Caret) => Some(Binary(BinOpKind::BitXor)),
token::BinOp(BinOpToken::And) => Some(Binary(BinOpKind::BitAnd)),
token::BinOp(BinOpToken::Or) => Some(Binary(BinOpKind::BitOr)),
token::BinOp(BinOpToken::Shl) => Some(Binary(BinOpKind::Shl)),
token::BinOp(BinOpToken::Shr) => Some(Binary(BinOpKind::Shr)),
token::BinOpEq(BinOpToken::Plus) => Some(AssignOp(BinOpKind::Add)),
token::BinOpEq(BinOpToken::Minus) => Some(AssignOp(BinOpKind::Sub)),
token::BinOpEq(BinOpToken::Star) => Some(AssignOp(BinOpKind::Mul)),
token::BinOpEq(BinOpToken::Slash) => Some(AssignOp(BinOpKind::Div)),
token::BinOpEq(BinOpToken::Percent) => Some(AssignOp(BinOpKind::Rem)),
token::BinOpEq(BinOpToken::Caret) => Some(AssignOp(BinOpKind::BitXor)),
token::BinOpEq(BinOpToken::And) => Some(AssignOp(BinOpKind::BitAnd)),
token::BinOpEq(BinOpToken::Or) => Some(AssignOp(BinOpKind::BitOr)),
token::BinOpEq(BinOpToken::Shl) => Some(AssignOp(BinOpKind::Shl)),
token::BinOpEq(BinOpToken::Shr) => Some(AssignOp(BinOpKind::Shr)),
token::Plus => Some(Binary(BinOpKind::Add)),
token::Minus => Some(Binary(BinOpKind::Sub)),
token::Star => Some(Binary(BinOpKind::Mul)),
token::Slash => Some(Binary(BinOpKind::Div)),
token::Percent => Some(Binary(BinOpKind::Rem)),
token::Caret => Some(Binary(BinOpKind::BitXor)),
token::And => Some(Binary(BinOpKind::BitAnd)),
token::Or => Some(Binary(BinOpKind::BitOr)),
token::Shl => Some(Binary(BinOpKind::Shl)),
token::Shr => Some(Binary(BinOpKind::Shr)),
token::PlusEq => Some(AssignOp(BinOpKind::Add)),
token::MinusEq => Some(AssignOp(BinOpKind::Sub)),
token::StarEq => Some(AssignOp(BinOpKind::Mul)),
token::SlashEq => Some(AssignOp(BinOpKind::Div)),
token::PercentEq => Some(AssignOp(BinOpKind::Rem)),
token::CaretEq => Some(AssignOp(BinOpKind::BitXor)),
token::AndEq => Some(AssignOp(BinOpKind::BitAnd)),
token::OrEq => Some(AssignOp(BinOpKind::BitOr)),
token::ShlEq => Some(AssignOp(BinOpKind::Shl)),
token::ShrEq => Some(AssignOp(BinOpKind::Shr)),
token::Lt => Some(Binary(BinOpKind::Lt)),
token::Le => Some(Binary(BinOpKind::Le)),
token::Ge => Some(Binary(BinOpKind::Ge)),
Expand Down
45 changes: 23 additions & 22 deletions compiler/rustc_ast_pretty/src/pprust/state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ use std::sync::Arc;

use rustc_ast::attr::AttrIdGenerator;
use rustc_ast::ptr::P;
use rustc_ast::token::{
self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind,
};
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
use rustc_ast::util::classify;
use rustc_ast::util::comments::{Comment, CommentStyle};
Expand Down Expand Up @@ -319,7 +317,7 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
(tt1, Tok(Token { kind: Comma | Semi | Dot, .. }, _)) if !is_punct(tt1) => false,

// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Bang, .. }, _))
if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, IdentIsRaw::Yes) =>
{
false
Expand All @@ -344,21 +342,6 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
}
}

fn binop_to_string(op: BinOpToken) -> &'static str {
match op {
token::Plus => "+",
token::Minus => "-",
token::Star => "*",
token::Slash => "/",
token::Percent => "%",
token::Caret => "^",
token::And => "&",
token::Or => "|",
token::Shl => "<<",
token::Shr => ">>",
}
}

pub fn doc_comment_to_string(
comment_kind: CommentKind,
attr_style: ast::AttrStyle,
Expand Down Expand Up @@ -913,12 +896,30 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
token::Ne => "!=".into(),
token::Ge => ">=".into(),
token::Gt => ">".into(),
token::Not => "!".into(),
token::Bang => "!".into(),
token::Tilde => "~".into(),
token::OrOr => "||".into(),
token::AndAnd => "&&".into(),
token::BinOp(op) => binop_to_string(op).into(),
token::BinOpEq(op) => format!("{}=", binop_to_string(op)).into(),
token::Plus => "+".into(),
token::Minus => "-".into(),
token::Star => "*".into(),
token::Slash => "/".into(),
token::Percent => "%".into(),
token::Caret => "^".into(),
token::And => "&".into(),
token::Or => "|".into(),
token::Shl => "<<".into(),
token::Shr => ">>".into(),
token::PlusEq => "+=".into(),
token::MinusEq => "-=".into(),
token::StarEq => "*=".into(),
token::SlashEq => "/=".into(),
token::PercentEq => "%=".into(),
token::CaretEq => "^=".into(),
token::AndEq => "&=".into(),
token::OrEq => "|=".into(),
token::ShlEq => "<<=".into(),
token::ShrEq => ">>=".into(),

/* Structural symbols */
token::At => "@".into(),
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_expand/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ impl<'a> StripUnconfigured<'a> {

// For inner attributes, we do the same thing for the `!` in `#![attr]`.
let mut trees = if cfg_attr.style == AttrStyle::Inner {
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _)) =
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Bang, .. }, _)) =
orig_trees.next()
else {
panic!("Bad tokens for attribute {cfg_attr:?}");
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_expand/src/mbe/macro_check.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ fn check_nested_occurrences(
}
(
NestedMacroState::MacroRules,
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
&TokenTree::Token(Token { kind: TokenKind::Bang, .. }),
) => {
state = NestedMacroState::MacroRulesNot;
}
Expand Down
12 changes: 6 additions & 6 deletions compiler/rustc_expand/src/mbe/macro_rules.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
&& let TokenKind::Ident(ident, _) = ident.kind
&& ident == sym::compile_error
&& let mbe::TokenTree::Token(bang) = bang
&& let TokenKind::Not = bang.kind
&& let TokenKind::Bang = bang.kind
&& let mbe::TokenTree::Delimited(.., del) = args
&& !del.delim.skip()
{
Expand Down Expand Up @@ -1135,7 +1135,7 @@ fn check_matcher_core<'tt>(
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
&& matches!(
next_token,
TokenTree::Token(token) if *token == BinOp(token::BinOpToken::Or)
TokenTree::Token(token) if *token == token::Or
)
{
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
Expand Down Expand Up @@ -1177,7 +1177,7 @@ fn check_matcher_core<'tt>(

if kind == NonterminalKind::Pat(PatWithOr)
&& sess.psess.edition.at_least_rust_2021()
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
&& next_token.is_token(&token::Or)
{
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
span,
Expand Down Expand Up @@ -1296,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
match tok {
TokenTree::Token(token) => match token.kind {
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
FatArrow | Comma | Eq | Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
IsInFollow::Yes
}
Expand Down Expand Up @@ -1332,9 +1332,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
| Colon
| Eq
| Gt
| BinOp(token::Shr)
| Shr
| Semi
| BinOp(token::Or) => IsInFollow::Yes,
| Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
IsInFollow::Yes
}
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_expand/src/mbe/quoted.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,8 @@ fn parse_tree<'a>(
/// `None`.
fn kleene_op(token: &Token) -> Option<KleeneOp> {
match token.kind {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
token::Star => Some(KleeneOp::ZeroOrMore),
token::Plus => Some(KleeneOp::OneOrMore),
token::Question => Some(KleeneOp::ZeroOrOne),
_ => None,
}
Expand Down
71 changes: 33 additions & 38 deletions compiler/rustc_expand/src/proc_macro_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,28 +180,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
Gt => op(">"),
AndAnd => op("&&"),
OrOr => op("||"),
Not => op("!"),
Bang => op("!"),
Tilde => op("~"),
BinOp(Plus) => op("+"),
BinOp(Minus) => op("-"),
BinOp(Star) => op("*"),
BinOp(Slash) => op("/"),
BinOp(Percent) => op("%"),
BinOp(Caret) => op("^"),
BinOp(And) => op("&"),
BinOp(Or) => op("|"),
BinOp(Shl) => op("<<"),
BinOp(Shr) => op(">>"),
BinOpEq(Plus) => op("+="),
BinOpEq(Minus) => op("-="),
BinOpEq(Star) => op("*="),
BinOpEq(Slash) => op("/="),
BinOpEq(Percent) => op("%="),
BinOpEq(Caret) => op("^="),
BinOpEq(And) => op("&="),
BinOpEq(Or) => op("|="),
BinOpEq(Shl) => op("<<="),
BinOpEq(Shr) => op(">>="),
Plus => op("+"),
Minus => op("-"),
Star => op("*"),
Slash => op("/"),
Percent => op("%"),
Caret => op("^"),
And => op("&"),
Or => op("|"),
Shl => op("<<"),
Shr => op(">>"),
PlusEq => op("+="),
MinusEq => op("-="),
StarEq => op("*="),
SlashEq => op("/="),
PercentEq => op("%="),
CaretEq => op("^="),
AndEq => op("&="),
OrEq => op("|="),
ShlEq => op("<<="),
ShrEq => op(">>="),
At => op("@"),
Dot => op("."),
DotDot => op(".."),
Expand Down Expand Up @@ -322,16 +322,16 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
b'=' => Eq,
b'<' => Lt,
b'>' => Gt,
b'!' => Not,
b'!' => Bang,
b'~' => Tilde,
b'+' => BinOp(Plus),
b'-' => BinOp(Minus),
b'*' => BinOp(Star),
b'/' => BinOp(Slash),
b'%' => BinOp(Percent),
b'^' => BinOp(Caret),
b'&' => BinOp(And),
b'|' => BinOp(Or),
b'+' => Plus,
b'-' => Minus,
b'*' => Star,
b'/' => Slash,
b'%' => Percent,
b'^' => Caret,
b'&' => And,
b'|' => Or,
b'@' => At,
b'.' => Dot,
b',' => Comma,
Expand Down Expand Up @@ -372,10 +372,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(integer, span);
smallvec![a, b]
}
Expand All @@ -385,10 +384,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(float, span);
smallvec![a, b]
}
Expand Down Expand Up @@ -599,10 +597,7 @@ impl server::TokenStream for Rustc<'_, '_> {
Ok(Self::TokenStream::from_iter([
// FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token_joint_hidden(
token::BinOp(token::Minus),
e.span,
),
tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span),
tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
]))
}
Expand Down
18 changes: 9 additions & 9 deletions compiler/rustc_parse/src/lexer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,17 +384,17 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
rustc_lexer::TokenKind::Colon => token::Colon,
rustc_lexer::TokenKind::Dollar => token::Dollar,
rustc_lexer::TokenKind::Eq => token::Eq,
rustc_lexer::TokenKind::Bang => token::Not,
rustc_lexer::TokenKind::Bang => token::Bang,
rustc_lexer::TokenKind::Lt => token::Lt,
rustc_lexer::TokenKind::Gt => token::Gt,
rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
rustc_lexer::TokenKind::And => token::BinOp(token::And),
rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
rustc_lexer::TokenKind::Minus => token::Minus,
rustc_lexer::TokenKind::And => token::And,
rustc_lexer::TokenKind::Or => token::Or,
rustc_lexer::TokenKind::Plus => token::Plus,
rustc_lexer::TokenKind::Star => token::Star,
rustc_lexer::TokenKind::Slash => token::Slash,
rustc_lexer::TokenKind::Caret => token::Caret,
rustc_lexer::TokenKind::Percent => token::Percent,

rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
// Don't emit diagnostics for sequences of the same invalid token
Expand Down
12 changes: 6 additions & 6 deletions compiler/rustc_parse/src/lexer/unicode_chars.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,11 +308,11 @@ pub(super) static UNICODE_ARRAY: &[(char, &str, &str)] = &[
const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
(" ", "Space", None),
("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentIsRaw::No))),
("-", "Minus/Hyphen", Some(token::BinOp(token::Minus))),
("-", "Minus/Hyphen", Some(token::Minus)),
(",", "Comma", Some(token::Comma)),
(";", "Semicolon", Some(token::Semi)),
(":", "Colon", Some(token::Colon)),
("!", "Exclamation Mark", Some(token::Not)),
("!", "Exclamation Mark", Some(token::Bang)),
("?", "Question Mark", Some(token::Question)),
(".", "Period", Some(token::Dot)),
("(", "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))),
Expand All @@ -321,11 +321,11 @@ const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
("]", "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))),
("{", "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))),
("}", "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))),
("*", "Asterisk", Some(token::BinOp(token::Star))),
("/", "Slash", Some(token::BinOp(token::Slash))),
("*", "Asterisk", Some(token::Star)),
("/", "Slash", Some(token::Slash)),
("\\", "Backslash", None),
("&", "Ampersand", Some(token::BinOp(token::And))),
("+", "Plus Sign", Some(token::BinOp(token::Plus))),
("&", "Ampersand", Some(token::And)),
("+", "Plus Sign", Some(token::Plus)),
("<", "Less-Than Sign", Some(token::Lt)),
("=", "Equals Sign", Some(token::Eq)),
("==", "Double Equals Sign", Some(token::EqEq)),
Expand Down
Loading
Loading