Rollup merge of #145354 - cache-proc-derive-macros, r=petrochenkov

Cache derive proc macro expansion with incremental query

This is a revival of #129102, originally implemented by @futile. Since it looks like they are not active currently, I'd like to push this work forward.

The first commit is squashed and rebased work from the original PR, with author attribution to futile. The rest of the commits are some additional comments that I created mostly for myself to understand what happens here. I also did some cleanups based on Vadim's review comments on the original PR, plus I refactored the TLS access a bit using `scoped_tls`.

The biggest issue, as usually, are tests... I tried using `#[rustc_clean(..., loaded_from_disk = "derive_macro_expansion")]`, but the problem is that since this query cannot recover the original key from its hash, and thus its fingerprintstyle is `FingerprintStyle::Opaque`, [this](https://github.com/rust-lang/rust/blob/2fef0a30ae6b2687dfb286cb544d2a542f7e2335/compiler/rustc_incremental/src/persist/dirty_clean.rs#L388) crashes when I try to use `loaded_from_disk`. Any suggestions from someone who actually understands the query system would be welcome 😅

TODO: document the new unstable flag

On a no-op change re-check of `octocrab 0.49` (which has a ton of `serde` derive proc macro invocations), this saves ~0.6s out of ~6s (so a ~10% win) on my PC.

r? @petrochenkov

Author: matthiaskrgr

Cargo.lock

@@ -3887,11 +3887,13 @@ dependencies = [
  "rustc_lexer",
  "rustc_lint_defs",
  "rustc_macros",
+ "rustc_middle",
  "rustc_parse",
  "rustc_proc_macro",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
+ "scoped-tls",
  "smallvec",
  "thin-vec",
  "tracing",

compiler/rustc_ast/src/ast.rs

@@ -3348,7 +3348,8 @@ impl UseTree {
 /// Distinguishes between `Attribute`s that decorate items and Attributes that
 /// are contained as statements within items. These two cases need to be
 /// distinguished for pretty-printing.
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy, HashStable_Generic, Walkable)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, Copy)]
+#[derive(Encodable, Decodable, HashStable_Generic, Walkable)]
 pub enum AttrStyle {
     Outer,
     Inner,

compiler/rustc_ast/src/token.rs

@@ -40,13 +40,13 @@ impl DocFragmentKind {
     }
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable, Debug, HashStable_Generic)]
 pub enum CommentKind {
     Line,
     Block,
 }
 
-#[derive(Copy, Clone, PartialEq, Debug, Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable, HashStable_Generic)]
 pub enum InvisibleOrigin {
     // From the expansion of a metavariable in a declarative macro.
     MetaVar(MetaVarKind),
@@ -123,7 +123,7 @@ impl fmt::Display for MetaVarKind {
 /// Describes how a sequence of token trees is delimited.
 /// Cannot use `proc_macro::Delimiter` directly because this
 /// structure should implement some additional traits.
-#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable, HashStable_Generic)]
 pub enum Delimiter {
     /// `( ... )`
     Parenthesis,
@@ -186,7 +186,7 @@ impl Delimiter {
 // type. This means that float literals like `1f32` are classified by this type
 // as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
 // given the `Float` kind.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable, Debug, HashStable_Generic)]
 pub enum LitKind {
     Bool, // AST only, must never appear in a `Token`
     Byte,
@@ -203,7 +203,7 @@ pub enum LitKind {
 }
 
 /// A literal token.
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable, Debug, HashStable_Generic)]
 pub struct Lit {
     pub kind: LitKind,
     pub symbol: Symbol,
@@ -349,7 +349,7 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool {
             .contains(&name)
 }
 
-#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic)]
+#[derive(PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy, Clone, HashStable_Generic)]
 pub enum IdentIsRaw {
     No,
     Yes,
@@ -376,7 +376,7 @@ impl From<bool> for IdentIsRaw {
     }
 }
 
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable, Debug, HashStable_Generic)]
 pub enum TokenKind {
     /* Expression-operator symbols. */
     /// `=`
@@ -526,7 +526,7 @@ pub enum TokenKind {
     Eof,
 }
 
-#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Encodable, Decodable, Debug, HashStable_Generic)]
 pub struct Token {
     pub kind: TokenKind,
     pub span: Span,

compiler/rustc_ast/src/tokenstream.rs

@@ -5,6 +5,7 @@
 //! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens.
 
 use std::borrow::Cow;
+use std::hash::Hash;
 use std::ops::Range;
 use std::sync::Arc;
 use std::{cmp, fmt, iter, mem};
@@ -22,7 +23,7 @@ use crate::token::{self, Delimiter, Token, TokenKind};
 use crate::{AttrVec, Attribute};
 
 /// Part of a `TokenStream`.
-#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
     /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
     /// delimiters are implicitly represented by `Delimited`.
@@ -538,7 +539,7 @@ pub struct AttrsTarget {
 /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
 /// guide pretty-printing, which is where the `JointHidden` value (which isn't
 /// part of `proc_macro::Spacing`) comes in useful.
-#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encodable, Decodable, HashStable_Generic)]
 pub enum Spacing {
     /// The token cannot join with the following token to form a compound
     /// token.
@@ -595,7 +596,7 @@ pub enum Spacing {
 }
 
 /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
-#[derive(Clone, Debug, Default, Encodable, Decodable)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
 
 impl TokenStream {
@@ -811,14 +812,6 @@ impl TokenStream {
     }
 }
 
-impl PartialEq<TokenStream> for TokenStream {
-    fn eq(&self, other: &TokenStream) -> bool {
-        self.iter().eq(other.iter())
-    }
-}
-
-impl Eq for TokenStream {}
-
 impl FromIterator<TokenTree> for TokenStream {
     fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
         TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
@@ -970,7 +963,8 @@ impl TokenCursor {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic, Walkable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+#[derive(Encodable, Decodable, HashStable_Generic, Walkable)]
 pub struct DelimSpan {
     pub open: Span,
     pub close: Span,
@@ -994,7 +988,7 @@ impl DelimSpan {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable, HashStable_Generic)]
 pub struct DelimSpacing {
     pub open: Spacing,
     pub close: Spacing,

compiler/rustc_expand/Cargo.toml

@@ -21,13 +21,15 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_parse = { path = "../rustc_parse" }
 # We must use the proc_macro version that we will compile proc-macros against,
 # not the one from our own sysroot.
 rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
+scoped-tls = "1.0"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"

compiler/rustc_expand/src/lib.rs

@@ -29,4 +29,8 @@ pub mod module;
 #[allow(rustc::untranslatable_diagnostic)]
 pub mod proc_macro;
 
+pub fn provide(providers: &mut rustc_middle::query::Providers) {
+    providers.derive_macro_expansion = proc_macro::provide_derive_macro_expansion;
+}
+
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }