1 // Copyright (C) 2020-2023 Free Software Foundation, Inc.
3 // This file is part of GCC.
5 // GCC is free software; you can redistribute it and/or modify it under
6 // the terms of the GNU General Public License as published by the Free
7 // Software Foundation; either version 3, or (at your option) any later
10 // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 // You should have received a copy of the GNU General Public License
16 // along with GCC; see the file COPYING3. If not see
17 // <http://www.gnu.org/licenses/>.
19 #include "rust-macro-expand.h"
20 #include "rust-macro-substitute-ctx.h"
21 #include "rust-ast-full.h"
22 #include "rust-ast-visitor.h"
23 #include "rust-diagnostics.h"
24 #include "rust-parse.h"
25 #include "rust-attribute-visitor.h"
26 #include "rust-early-name-resolver.h"
30 MacroExpander::expand_decl_macro (Location invoc_locus
,
31 AST::MacroInvocData
&invoc
,
32 AST::MacroRulesDefinition
&rules_def
,
35 // ensure that both invocation and rules are in a valid state
36 rust_assert (!invoc
.is_marked_for_strip ());
37 rust_assert (!rules_def
.is_marked_for_strip ());
38 rust_assert (rules_def
.get_macro_rules ().size () > 0);
40 /* probably something here about parsing invoc and rules def token trees to
41 * token stream. if not, how would parser handle the captures of exprs and
42 * stuff? on the other hand, token trees may be kind of useful in rules def as
43 * creating a point where recursion can occur (like having
44 * "compare_macro_match" and then it calling itself when it finds delimiters)
47 /* find matching rule to invoc token tree, based on macro rule's matcher. if
49 * - specifically, check each matcher in order. if one fails to match, move
51 /* TODO: does doing this require parsing expressions and whatever in the
52 * invoc? if so, might as well save the results if referenced using $ or
53 * whatever. If not, do another pass saving them. Except this is probably
54 * useless as different rules could have different starting points for exprs
55 * or whatever. Decision trees could avoid this, but they have their own
57 /* TODO: will need to modify the parser so that it can essentially "catch"
58 * errors - maybe "try_parse_expr" or whatever methods. */
59 // this technically creates a back-tracking parser - this will be the
60 // implementation style
62 /* then, after results are saved, generate the macro output from the
63 * transcriber token tree. if i understand this correctly, the macro
64 * invocation gets replaced by the transcriber tokens, except with
65 * substitutions made (e.g. for $i variables) */
67 /* TODO: it is probably better to modify AST::Token to store a pointer to a
68 * Lexer::Token (rather than being converted) - i.e. not so much have
69 * AST::Token as a Token but rather a TokenContainer (as it is another type of
70 * TokenTree). This will prevent re-conversion of Tokens between each type
71 * all the time, while still allowing the heterogenous storage of token trees.
74 AST::DelimTokenTree
&invoc_token_tree
= invoc
.get_delim_tok_tree ();
77 AST::MacroRule
*matched_rule
= nullptr;
78 std::map
<std::string
, MatchedFragmentContainer
> matched_fragments
;
79 for (auto &rule
: rules_def
.get_rules ())
82 bool did_match_rule
= try_match_rule (rule
, invoc_token_tree
);
83 matched_fragments
= sub_stack
.pop ();
88 // for (auto &kv : matched_fragments)
89 // rust_debug ("[fragment]: %s (%ld - %s)", kv.first.c_str (),
90 // kv.second.get_fragments ().size (),
91 // kv.second.get_kind ()
92 // == MatchedFragmentContainer::Kind::Repetition
101 if (matched_rule
== nullptr)
103 RichLocation
r (invoc_locus
);
104 r
.add_range (rules_def
.get_locus ());
105 rust_error_at (r
, "Failed to match any rule within macro");
106 return AST::ASTFragment::create_error ();
109 return transcribe_rule (*matched_rule
, invoc_token_tree
, matched_fragments
,
110 semicolon
, peek_context ());
114 MacroExpander::expand_invoc (AST::MacroInvocation
&invoc
, bool has_semicolon
)
116 if (depth_exceeds_recursion_limit ())
118 rust_error_at (invoc
.get_locus (), "reached recursion limit");
122 AST::MacroInvocData
&invoc_data
= invoc
.get_invoc_data ();
125 // switch on type of macro:
126 // - '!' syntax macro (inner switch)
127 // - procedural macro - "A token-based function-like macro"
128 // - 'macro_rules' (by example/pattern-match) macro? or not? "an
129 // AST-based function-like macro"
130 // - else is unreachable
131 // - attribute syntax macro (inner switch)
132 // - procedural macro attribute syntax - "A token-based attribute
134 // - legacy macro attribute syntax? - "an AST-based attribute macro"
135 // - non-macro attribute: mark known
136 // - else is unreachable
137 // - derive macro (inner switch)
138 // - derive or legacy derive - "token-based" vs "AST-based"
139 // - else is unreachable
140 // - derive container macro - unreachable
142 auto fragment
= AST::ASTFragment::create_error ();
143 invoc_data
.set_expander (this);
146 AST::MacroRulesDefinition
*rules_def
= nullptr;
147 bool ok
= mappings
->lookup_macro_invocation (invoc
, &rules_def
);
149 // If there's no rule associated with the invocation, we can simply return
150 // early. The early name resolver will have already emitted an error.
154 if (rules_def
->is_builtin ())
156 = rules_def
->get_builtin_transcriber () (invoc
.get_locus (), invoc_data
);
158 fragment
= expand_decl_macro (invoc
.get_locus (), invoc_data
, *rules_def
,
161 set_expanded_fragment (std::move (fragment
));
164 /* Determines whether any cfg predicate is false and hence item with attributes
165 * should be stripped. Note that attributes must be expanded before calling. */
167 MacroExpander::fails_cfg (const AST::AttrVec
&attrs
) const
169 for (const auto &attr
: attrs
)
171 if (attr
.get_path () == "cfg" && !attr
.check_cfg_predicate (session
))
177 /* Determines whether any cfg predicate is false and hence item with attributes
178 * should be stripped. Will expand attributes as well. */
180 MacroExpander::fails_cfg_with_expand (AST::AttrVec
&attrs
) const
182 // TODO: maybe have something that strips cfg attributes that evaluate true?
183 for (auto &attr
: attrs
)
185 if (attr
.get_path () == "cfg")
187 if (!attr
.is_parsed_to_meta_item ())
188 attr
.parse_attr_to_meta_item ();
191 if (!attr
.is_parsed_to_meta_item ())
192 rust_debug ("failed to parse attr to meta item, right before "
193 "cfg predicate check");
195 rust_debug ("attr has been successfully parsed to meta item, "
196 "right before cfg predicate check");
198 if (!attr
.check_cfg_predicate (session
))
202 "cfg predicate failed for attribute: \033[0;31m'%s'\033[0m",
203 attr
.as_string ().c_str ());
210 rust_debug ("cfg predicate succeeded for attribute: "
211 "\033[0;31m'%s'\033[0m",
212 attr
.as_string ().c_str ());
219 // Expands cfg_attr attributes.
221 MacroExpander::expand_cfg_attrs (AST::AttrVec
&attrs
)
223 for (std::size_t i
= 0; i
< attrs
.size (); i
++)
225 auto &attr
= attrs
[i
];
226 if (attr
.get_path () == "cfg_attr")
228 if (!attr
.is_parsed_to_meta_item ())
229 attr
.parse_attr_to_meta_item ();
231 if (attr
.check_cfg_predicate (session
))
233 // split off cfg_attr
234 AST::AttrVec new_attrs
= attr
.separate_cfg_attrs ();
236 // remove attr from vector
237 attrs
.erase (attrs
.begin () + i
);
239 // add new attrs to vector
240 attrs
.insert (attrs
.begin () + i
,
241 std::make_move_iterator (new_attrs
.begin ()),
242 std::make_move_iterator (new_attrs
.end ()));
245 /* do something - if feature (first token in tree) is in fact enabled,
246 * make tokens listed afterwards into attributes. i.e.: for
247 * [cfg_attr(feature = "wow", wow1, wow2)], if "wow" is true, then add
248 * attributes [wow1] and [wow2] to attribute list. This can also be
249 * recursive, so check for expanded attributes being recursive and
250 * possibly recursively call the expand_attrs? */
257 attrs
.shrink_to_fit ();
261 MacroExpander::expand_crate ()
263 NodeId scope_node_id
= crate
.get_node_id ();
264 resolver
->get_macro_scope ().push (scope_node_id
);
266 /* fill macro/decorator map from init list? not sure where init list comes
269 // TODO: does cfg apply for inner attributes? research.
270 // the apparent answer (from playground test) is yes
272 // expand crate cfg_attr attributes
273 expand_cfg_attrs (crate
.inner_attrs
);
275 if (fails_cfg_with_expand (crate
.inner_attrs
))
277 // basically, delete whole crate
278 crate
.strip_crate ();
279 // TODO: maybe create warning here? probably not desired behaviour
281 // expand module attributes?
285 // expand attributes recursively and strip items if required
286 AttrVisitor
attr_visitor (*this);
287 auto &items
= crate
.items
;
288 for (auto it
= items
.begin (); it
!= items
.end ();)
292 // mark for stripping if required
293 item
->accept_vis (attr_visitor
);
295 auto fragment
= take_expanded_fragment (attr_visitor
);
296 if (fragment
.should_expand ())
298 // Remove the current expanded invocation
299 it
= items
.erase (it
);
300 for (auto &node
: fragment
.get_nodes ())
302 it
= items
.insert (it
, node
.take_item ());
306 else if (item
->is_marked_for_strip ())
307 it
= items
.erase (it
);
314 // TODO: should recursive attribute and macro expansion be done in the same
315 // transversal? Or in separate ones like currently?
317 // expand module tree recursively
321 // extract exported macros?
325 MacroExpander::depth_exceeds_recursion_limit () const
327 return expansion_depth
>= cfg
.recursion_limit
;
331 MacroExpander::try_match_rule (AST::MacroRule
&match_rule
,
332 AST::DelimTokenTree
&invoc_token_tree
)
334 MacroInvocLexer
lex (invoc_token_tree
.to_token_stream ());
335 Parser
<MacroInvocLexer
> parser (lex
);
337 AST::MacroMatcher
&matcher
= match_rule
.get_matcher ();
340 if (!match_matcher (parser
, matcher
))
347 bool used_all_input_tokens
= parser
.skip_token (END_OF_FILE
);
348 return used_all_input_tokens
;
352 MacroExpander::match_fragment (Parser
<MacroInvocLexer
> &parser
,
353 AST::MacroMatchFragment
&fragment
)
355 switch (fragment
.get_frag_spec ().get_kind ())
357 case AST::MacroFragSpec::EXPR
:
358 parser
.parse_expr ();
361 case AST::MacroFragSpec::BLOCK
:
362 parser
.parse_block_expr ();
365 case AST::MacroFragSpec::IDENT
:
366 parser
.parse_identifier_pattern ();
369 case AST::MacroFragSpec::LITERAL
:
370 parser
.parse_literal_expr ();
373 case AST::MacroFragSpec::ITEM
:
374 parser
.parse_item (false);
377 case AST::MacroFragSpec::TY
:
378 parser
.parse_type ();
381 case AST::MacroFragSpec::PAT
:
382 parser
.parse_pattern ();
385 case AST::MacroFragSpec::PATH
:
386 parser
.parse_path_in_expression ();
389 case AST::MacroFragSpec::VIS
:
390 parser
.parse_visibility ();
393 case AST::MacroFragSpec::STMT
: {
394 auto restrictions
= ParseRestrictions ();
395 restrictions
.consume_semi
= false;
396 parser
.parse_stmt (restrictions
);
400 case AST::MacroFragSpec::LIFETIME
:
401 parser
.parse_lifetime_params ();
404 // is meta attributes?
405 case AST::MacroFragSpec::META
:
406 parser
.parse_attribute_body ();
409 case AST::MacroFragSpec::TT
:
410 parser
.parse_token_tree ();
413 // i guess we just ignore invalid and just error out
414 case AST::MacroFragSpec::INVALID
:
418 // it matches if the parser did not produce errors trying to parse that type
420 return !parser
.has_errors ();
424 MacroExpander::match_matcher (Parser
<MacroInvocLexer
> &parser
,
425 AST::MacroMatcher
&matcher
, bool in_repetition
)
427 if (depth_exceeds_recursion_limit ())
429 rust_error_at (matcher
.get_match_locus (), "reached recursion limit");
433 auto delimiter
= parser
.peek_current_token ();
435 // this is used so we can check that we delimit the stream correctly.
436 switch (delimiter
->get_id ())
439 if (!parser
.skip_token (LEFT_PAREN
))
445 if (!parser
.skip_token (LEFT_SQUARE
))
451 if (!parser
.skip_token (LEFT_CURLY
))
459 const MacroInvocLexer
&source
= parser
.get_token_source ();
461 for (auto &match
: matcher
.get_matches ())
463 size_t offs_begin
= source
.get_offs ();
465 switch (match
->get_macro_match_type ())
467 case AST::MacroMatch::MacroMatchType::Fragment
: {
468 AST::MacroMatchFragment
*fragment
469 = static_cast<AST::MacroMatchFragment
*> (match
.get ());
470 if (!match_fragment (parser
, *fragment
))
473 // matched fragment get the offset in the token stream
474 size_t offs_end
= source
.get_offs ();
476 sub_stack
.append_fragment (
477 MatchedFragment (fragment
->get_ident (), offs_begin
, offs_end
));
479 sub_stack
.insert_metavar (
480 MatchedFragment (fragment
->get_ident (), offs_begin
, offs_end
));
484 case AST::MacroMatch::MacroMatchType::Tok
: {
485 AST::Token
*tok
= static_cast<AST::Token
*> (match
.get ());
486 if (!match_token (parser
, *tok
))
491 case AST::MacroMatch::MacroMatchType::Repetition
: {
492 AST::MacroMatchRepetition
*rep
493 = static_cast<AST::MacroMatchRepetition
*> (match
.get ());
494 if (!match_repetition (parser
, *rep
))
499 case AST::MacroMatch::MacroMatchType::Matcher
: {
501 = static_cast<AST::MacroMatcher
*> (match
.get ());
503 if (!match_matcher (parser
, *m
, in_repetition
))
514 switch (delimiter
->get_id ())
517 if (!parser
.skip_token (RIGHT_PAREN
))
523 if (!parser
.skip_token (RIGHT_SQUARE
))
529 if (!parser
.skip_token (RIGHT_CURLY
))
541 MacroExpander::match_token (Parser
<MacroInvocLexer
> &parser
, AST::Token
&token
)
543 // FIXME this needs to actually match the content and the type
544 return parser
.skip_token (token
.get_id ());
548 MacroExpander::match_n_matches (Parser
<MacroInvocLexer
> &parser
,
549 AST::MacroMatchRepetition
&rep
,
550 size_t &match_amount
, size_t lo_bound
,
554 auto &matches
= rep
.get_matches ();
556 const MacroInvocLexer
&source
= parser
.get_token_source ();
559 // If the current token is a closing macro delimiter, break away.
560 // TODO: Is this correct?
561 auto t_id
= parser
.peek_current_token ()->get_id ();
562 if (t_id
== RIGHT_PAREN
|| t_id
== RIGHT_SQUARE
|| t_id
== RIGHT_CURLY
)
565 // Skip parsing a separator on the first match, otherwise consume it.
566 // If it isn't present, this is an error
567 if (rep
.has_sep () && match_amount
> 0)
568 if (!match_token (parser
, *rep
.get_sep ()))
571 bool valid_current_match
= false;
572 for (auto &match
: matches
)
574 size_t offs_begin
= source
.get_offs ();
575 switch (match
->get_macro_match_type ())
577 case AST::MacroMatch::MacroMatchType::Fragment
: {
578 AST::MacroMatchFragment
*fragment
579 = static_cast<AST::MacroMatchFragment
*> (match
.get ());
580 valid_current_match
= match_fragment (parser
, *fragment
);
582 // matched fragment get the offset in the token stream
583 size_t offs_end
= source
.get_offs ();
585 // The main difference with match_matcher happens here: Instead
586 // of inserting a new fragment, we append to one. If that
587 // fragment does not exist, then the operation is similar to
588 // `insert_fragment` with the difference that we are not
589 // creating a metavariable, but a repetition of one, which is
591 sub_stack
.append_fragment (
592 MatchedFragment (fragment
->get_ident (), offs_begin
,
597 case AST::MacroMatch::MacroMatchType::Tok
: {
598 AST::Token
*tok
= static_cast<AST::Token
*> (match
.get ());
599 valid_current_match
= match_token (parser
, *tok
);
603 case AST::MacroMatch::MacroMatchType::Repetition
: {
604 AST::MacroMatchRepetition
*rep
605 = static_cast<AST::MacroMatchRepetition
*> (match
.get ());
606 valid_current_match
= match_repetition (parser
, *rep
);
610 case AST::MacroMatch::MacroMatchType::Matcher
: {
612 = static_cast<AST::MacroMatcher
*> (match
.get ());
613 valid_current_match
= match_matcher (parser
, *m
, true);
618 // If we've encountered an error once, stop trying to match more
620 if (!valid_current_match
)
625 // Break early if we notice there's too many expressions already
626 if (hi_bound
&& match_amount
> hi_bound
)
630 // Check if the amount of matches we got is valid: Is it more than the lower
631 // bound and less than the higher bound?
632 bool did_meet_lo_bound
= match_amount
>= lo_bound
;
633 bool did_meet_hi_bound
= hi_bound
? match_amount
<= hi_bound
: true;
635 // If the end-result is valid, then we can clear the parse errors: Since
636 // repetitions are parsed eagerly, it is okay to fail in some cases
637 auto res
= did_meet_lo_bound
&& did_meet_hi_bound
;
639 parser
.clear_errors ();
645 MacroExpander::match_repetition (Parser
<MacroInvocLexer
> &parser
,
646 AST::MacroMatchRepetition
&rep
)
648 size_t match_amount
= 0;
653 switch (rep
.get_op ())
655 case AST::MacroMatchRepetition::MacroRepOp::ANY
:
658 res
= match_n_matches (parser
, rep
, match_amount
);
660 case AST::MacroMatchRepetition::MacroRepOp::ONE_OR_MORE
:
663 res
= match_n_matches (parser
, rep
, match_amount
, 1);
665 case AST::MacroMatchRepetition::MacroRepOp::ZERO_OR_ONE
:
668 res
= match_n_matches (parser
, rep
, match_amount
, 0, 1);
675 rust_error_at (rep
.get_match_locus (),
676 "invalid amount of matches for macro invocation. Expected "
677 "between %s and %s, got %lu",
678 lo_str
.c_str (), hi_str
.c_str (),
679 (unsigned long) match_amount
);
681 rust_debug_loc (rep
.get_match_locus (), "%s matched %lu times",
682 res
? "successfully" : "unsuccessfully",
683 (unsigned long) match_amount
);
685 // We have to handle zero fragments differently: They will not have been
686 // "matched" but they are still valid and should be inserted as a special
687 // case. So we go through the stack map, and for every fragment which doesn't
688 // exist, insert a zero-matched fragment.
689 auto &stack_map
= sub_stack
.peek ();
690 for (auto &match
: rep
.get_matches ())
692 if (match
->get_macro_match_type ()
693 == AST::MacroMatch::MacroMatchType::Fragment
)
695 auto fragment
= static_cast<AST::MacroMatchFragment
*> (match
.get ());
696 auto it
= stack_map
.find (fragment
->get_ident ());
698 if (it
== stack_map
.end ())
699 sub_stack
.insert_matches (fragment
->get_ident (),
700 MatchedFragmentContainer::zero ());
708 * Helper function to refactor calling a parsing function 0 or more times
710 static AST::ASTFragment
711 parse_many (Parser
<MacroInvocLexer
> &parser
, TokenId
&delimiter
,
712 std::function
<AST::SingleASTNode ()> parse_fn
)
714 std::vector
<AST::SingleASTNode
> nodes
;
717 if (parser
.peek_current_token ()->get_id () == delimiter
)
720 auto node
= parse_fn ();
721 if (node
.is_error ())
723 for (auto err
: parser
.get_errors ())
726 return AST::ASTFragment::create_error ();
729 nodes
.emplace_back (std::move (node
));
732 return AST::ASTFragment (std::move (nodes
));
736 * Transcribe 0 or more items from a macro invocation
738 * @param parser Parser to extract items from
739 * @param delimiter Id of the token on which parsing should stop
741 static AST::ASTFragment
742 transcribe_many_items (Parser
<MacroInvocLexer
> &parser
, TokenId
&delimiter
)
744 return parse_many (parser
, delimiter
, [&parser
] () {
745 auto item
= parser
.parse_item (true);
746 return AST::SingleASTNode (std::move (item
));
751 * Transcribe 0 or more external items from a macro invocation
753 * @param parser Parser to extract items from
754 * @param delimiter Id of the token on which parsing should stop
756 static AST::ASTFragment
757 transcribe_many_ext (Parser
<MacroInvocLexer
> &parser
, TokenId
&delimiter
)
759 return parse_many (parser
, delimiter
, [&parser
] () {
760 auto item
= parser
.parse_external_item ();
761 return AST::SingleASTNode (std::move (item
));
766 * Transcribe 0 or more trait items from a macro invocation
768 * @param parser Parser to extract items from
769 * @param delimiter Id of the token on which parsing should stop
771 static AST::ASTFragment
772 transcribe_many_trait_items (Parser
<MacroInvocLexer
> &parser
,
775 return parse_many (parser
, delimiter
, [&parser
] () {
776 auto item
= parser
.parse_trait_item ();
777 return AST::SingleASTNode (std::move (item
));
782 * Transcribe 0 or more impl items from a macro invocation
784 * @param parser Parser to extract items from
785 * @param delimiter Id of the token on which parsing should stop
787 static AST::ASTFragment
788 transcribe_many_impl_items (Parser
<MacroInvocLexer
> &parser
, TokenId
&delimiter
)
790 return parse_many (parser
, delimiter
, [&parser
] () {
791 auto item
= parser
.parse_inherent_impl_item ();
792 return AST::SingleASTNode (std::move (item
));
797 * Transcribe 0 or more trait impl items from a macro invocation
799 * @param parser Parser to extract items from
800 * @param delimiter Id of the token on which parsing should stop
802 static AST::ASTFragment
803 transcribe_many_trait_impl_items (Parser
<MacroInvocLexer
> &parser
,
806 return parse_many (parser
, delimiter
, [&parser
] () {
807 auto item
= parser
.parse_trait_impl_item ();
808 return AST::SingleASTNode (std::move (item
));
813 * Transcribe 0 or more statements from a macro invocation
815 * @param parser Parser to extract statements from
816 * @param delimiter Id of the token on which parsing should stop
818 static AST::ASTFragment
819 transcribe_many_stmts (Parser
<MacroInvocLexer
> &parser
, TokenId
&delimiter
)
821 auto restrictions
= ParseRestrictions ();
822 restrictions
.consume_semi
= false;
824 // FIXME: This is invalid! It needs to also handle cases where the macro
825 // transcriber is an expression, but since the macro call is followed by
826 // a semicolon, it's a valid ExprStmt
827 return parse_many (parser
, delimiter
, [&parser
, restrictions
] () {
828 auto stmt
= parser
.parse_stmt (restrictions
);
829 return AST::SingleASTNode (std::move (stmt
));
834 * Transcribe one expression from a macro invocation
836 * @param parser Parser to extract statements from
838 static AST::ASTFragment
839 transcribe_expression (Parser
<MacroInvocLexer
> &parser
)
841 auto expr
= parser
.parse_expr ();
843 return AST::ASTFragment ({std::move (expr
)});
847 * Transcribe one type from a macro invocation
849 * @param parser Parser to extract statements from
851 static AST::ASTFragment
852 transcribe_type (Parser
<MacroInvocLexer
> &parser
)
854 auto type
= parser
.parse_type (true);
855 for (auto err
: parser
.get_errors ())
858 return AST::ASTFragment ({std::move (type
)});
861 static AST::ASTFragment
862 transcribe_on_delimiter (Parser
<MacroInvocLexer
> &parser
, bool semicolon
,
863 AST::DelimType delimiter
, TokenId last_token_id
)
865 if (semicolon
|| delimiter
== AST::DelimType::CURLY
)
866 return transcribe_many_stmts (parser
, last_token_id
);
868 return transcribe_expression (parser
);
871 static AST::ASTFragment
872 transcribe_context (MacroExpander::ContextType ctx
,
873 Parser
<MacroInvocLexer
> &parser
, bool semicolon
,
874 AST::DelimType delimiter
, TokenId last_token_id
)
876 // The flow-chart in order to choose a parsing function is as follows:
878 // [switch special context]
879 // -- Item --> parser.parse_item();
880 // -- Trait --> parser.parse_trait_item();
881 // -- Impl --> parser.parse_impl_item();
882 // -- Extern --> parser.parse_extern_item();
883 // -- None --> [has semicolon?]
884 // -- Yes --> parser.parse_stmt();
885 // -- No --> [switch invocation.delimiter()]
886 // -- { } --> parser.parse_stmt();
887 // -- _ --> parser.parse_expr(); // once!
889 // If there is a semicolon OR we are expanding a MacroInvocationSemi, then
890 // we can parse multiple items. Otherwise, parse *one* expression
894 case MacroExpander::ContextType::ITEM
:
895 return transcribe_many_items (parser
, last_token_id
);
897 case MacroExpander::ContextType::TRAIT
:
898 return transcribe_many_trait_items (parser
, last_token_id
);
900 case MacroExpander::ContextType::IMPL
:
901 return transcribe_many_impl_items (parser
, last_token_id
);
903 case MacroExpander::ContextType::TRAIT_IMPL
:
904 return transcribe_many_trait_impl_items (parser
, last_token_id
);
906 case MacroExpander::ContextType::EXTERN
:
907 return transcribe_many_ext (parser
, last_token_id
);
909 case MacroExpander::ContextType::TYPE
:
910 return transcribe_type (parser
);
913 return transcribe_on_delimiter (parser
, semicolon
, delimiter
,
919 tokens_to_str (std::vector
<std::unique_ptr
<AST::Token
>> &tokens
)
922 if (!tokens
.empty ())
924 str
+= tokens
[0]->as_string ();
925 for (size_t i
= 1; i
< tokens
.size (); i
++)
926 str
+= " " + tokens
[i
]->as_string ();
933 MacroExpander::transcribe_rule (
934 AST::MacroRule
&match_rule
, AST::DelimTokenTree
&invoc_token_tree
,
935 std::map
<std::string
, MatchedFragmentContainer
> &matched_fragments
,
936 bool semicolon
, ContextType ctx
)
938 // we can manipulate the token tree to substitute the dollar identifiers so
939 // that when we call parse its already substituted for us
940 AST::MacroTranscriber
&transcriber
= match_rule
.get_transcriber ();
941 AST::DelimTokenTree
&transcribe_tree
= transcriber
.get_token_tree ();
943 auto invoc_stream
= invoc_token_tree
.to_token_stream ();
944 auto macro_rule_tokens
= transcribe_tree
.to_token_stream ();
946 auto substitute_context
947 = SubstituteCtx (invoc_stream
, macro_rule_tokens
, matched_fragments
);
948 std::vector
<std::unique_ptr
<AST::Token
>> substituted_tokens
949 = substitute_context
.substitute_tokens ();
951 rust_debug ("substituted tokens: %s",
952 tokens_to_str (substituted_tokens
).c_str ());
954 // parse it to an ASTFragment
955 MacroInvocLexer
lex (std::move (substituted_tokens
));
956 Parser
<MacroInvocLexer
> parser (lex
);
958 auto last_token_id
= TokenId::RIGHT_CURLY
;
960 // this is used so we can check that we delimit the stream correctly.
961 switch (transcribe_tree
.get_delim_type ())
963 case AST::DelimType::PARENS
:
964 last_token_id
= TokenId::RIGHT_PAREN
;
965 rust_assert (parser
.skip_token (LEFT_PAREN
));
968 case AST::DelimType::CURLY
:
969 rust_assert (parser
.skip_token (LEFT_CURLY
));
972 case AST::DelimType::SQUARE
:
973 last_token_id
= TokenId::RIGHT_SQUARE
;
974 rust_assert (parser
.skip_token (LEFT_SQUARE
));
978 // see https://github.com/Rust-GCC/gccrs/issues/22
980 // - Treat all macro invocations with parentheses, (), or square brackets,
981 // [], as expressions.
982 // - If the macro invocation has curly brackets, {}, it may be parsed as a
983 // statement depending on the context.
984 // - If the macro invocation has a semicolon at the end, it must be parsed
985 // as a statement (either via ExpressionStatement or
986 // MacroInvocationWithSemi)
989 = transcribe_context (ctx
, parser
, semicolon
,
990 invoc_token_tree
.get_delim_type (), last_token_id
);
993 if (parser
.has_errors ())
995 for (auto &err
: parser
.get_errors ())
996 rust_error_at (err
.locus
, "%s", err
.message
.c_str ());
997 return AST::ASTFragment::create_error ();
1000 // are all the tokens used?
1001 bool did_delimit
= parser
.skip_token (last_token_id
);
1003 bool reached_end_of_stream
= did_delimit
&& parser
.skip_token (END_OF_FILE
);
1004 if (!reached_end_of_stream
)
1006 const_TokenPtr current_token
= parser
.peek_current_token ();
1007 rust_error_at (current_token
->get_locus (),
1008 "tokens here and after are unparsed");