|
150
|
1 //===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 //
|
|
|
9 // This file implements pieces of the Preprocessor interface that manage the
|
|
|
10 // caching of lexed tokens.
|
|
|
11 //
|
|
|
12 //===----------------------------------------------------------------------===//
|
|
|
13
|
|
|
14 #include "clang/Lex/Preprocessor.h"
|
|
|
15 using namespace clang;
|
|
|
16
|
|
|
17 // EnableBacktrackAtThisPos - From the point that this method is called, and
|
|
|
18 // until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
|
|
|
19 // keeps track of the lexed tokens so that a subsequent Backtrack() call will
|
|
|
20 // make the Preprocessor re-lex the same tokens.
|
|
|
21 //
|
|
|
22 // Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
|
|
|
23 // be called multiple times and CommitBacktrackedTokens/Backtrack calls will
|
|
|
24 // be combined with the EnableBacktrackAtThisPos calls in reverse order.
|
|
|
25 void Preprocessor::EnableBacktrackAtThisPos() {
|
|
|
26 assert(LexLevel == 0 && "cannot use lookahead while lexing");
|
|
|
27 BacktrackPositions.push_back(CachedLexPos);
|
|
|
28 EnterCachingLexMode();
|
|
|
29 }
|
|
|
30
|
|
|
31 // Disable the last EnableBacktrackAtThisPos call.
|
|
|
32 void Preprocessor::CommitBacktrackedTokens() {
|
|
|
33 assert(!BacktrackPositions.empty()
|
|
|
34 && "EnableBacktrackAtThisPos was not called!");
|
|
|
35 BacktrackPositions.pop_back();
|
|
|
36 }
|
|
|
37
|
|
|
38 // Make Preprocessor re-lex the tokens that were lexed since
|
|
|
39 // EnableBacktrackAtThisPos() was previously called.
|
|
|
40 void Preprocessor::Backtrack() {
|
|
|
41 assert(!BacktrackPositions.empty()
|
|
|
42 && "EnableBacktrackAtThisPos was not called!");
|
|
|
43 CachedLexPos = BacktrackPositions.back();
|
|
|
44 BacktrackPositions.pop_back();
|
|
|
45 recomputeCurLexerKind();
|
|
|
46 }
|
|
|
47
|
|
|
48 void Preprocessor::CachingLex(Token &Result) {
|
|
|
49 if (!InCachingLexMode())
|
|
|
50 return;
|
|
|
51
|
|
|
52 // The assert in EnterCachingLexMode should prevent this from happening.
|
|
|
53 assert(LexLevel == 1 &&
|
|
|
54 "should not use token caching within the preprocessor");
|
|
|
55
|
|
|
56 if (CachedLexPos < CachedTokens.size()) {
|
|
|
57 Result = CachedTokens[CachedLexPos++];
|
|
|
58 Result.setFlag(Token::IsReinjected);
|
|
|
59 return;
|
|
|
60 }
|
|
|
61
|
|
|
62 ExitCachingLexMode();
|
|
|
63 Lex(Result);
|
|
|
64
|
|
|
65 if (isBacktrackEnabled()) {
|
|
|
66 // Cache the lexed token.
|
|
|
67 EnterCachingLexModeUnchecked();
|
|
|
68 CachedTokens.push_back(Result);
|
|
|
69 ++CachedLexPos;
|
|
|
70 return;
|
|
|
71 }
|
|
|
72
|
|
|
73 if (CachedLexPos < CachedTokens.size()) {
|
|
|
74 EnterCachingLexModeUnchecked();
|
|
|
75 } else {
|
|
|
76 // All cached tokens were consumed.
|
|
|
77 CachedTokens.clear();
|
|
|
78 CachedLexPos = 0;
|
|
|
79 }
|
|
|
80 }
|
|
|
81
|
|
|
82 void Preprocessor::EnterCachingLexMode() {
|
|
|
83 // The caching layer sits on top of all the other lexers, so it's incorrect
|
|
|
84 // to cache tokens while inside a nested lex action. The cached tokens would
|
|
|
85 // be retained after returning to the enclosing lex action and, at best,
|
|
|
86 // would appear at the wrong position in the token stream.
|
|
|
87 assert(LexLevel == 0 &&
|
|
|
88 "entered caching lex mode while lexing something else");
|
|
|
89
|
|
|
90 if (InCachingLexMode()) {
|
|
|
91 assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
|
|
|
92 return;
|
|
|
93 }
|
|
|
94
|
|
|
95 EnterCachingLexModeUnchecked();
|
|
|
96 }
|
|
|
97
|
|
|
98 void Preprocessor::EnterCachingLexModeUnchecked() {
|
|
|
99 assert(CurLexerKind != CLK_CachingLexer && "already in caching lex mode");
|
|
|
100 PushIncludeMacroStack();
|
|
|
101 CurLexerKind = CLK_CachingLexer;
|
|
|
102 }
|
|
|
103
|
|
|
104
|
|
|
105 const Token &Preprocessor::PeekAhead(unsigned N) {
|
|
|
106 assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
|
|
|
107 ExitCachingLexMode();
|
|
|
108 for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
|
|
|
109 CachedTokens.push_back(Token());
|
|
|
110 Lex(CachedTokens.back());
|
|
|
111 }
|
|
|
112 EnterCachingLexMode();
|
|
|
113 return CachedTokens.back();
|
|
|
114 }
|
|
|
115
|
|
|
116 void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
|
|
|
117 assert(Tok.isAnnotation() && "Expected annotation token");
|
|
|
118 assert(CachedLexPos != 0 && "Expected to have some cached tokens");
|
|
|
119 assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
|
|
|
120 && "The annotation should be until the most recent cached token");
|
|
|
121
|
|
|
122 // Start from the end of the cached tokens list and look for the token
|
|
|
123 // that is the beginning of the annotation token.
|
|
|
124 for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
|
|
|
125 CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
|
|
|
126 if (AnnotBegin->getLocation() == Tok.getLocation()) {
|
|
|
127 assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
|
|
|
128 "The backtrack pos points inside the annotated tokens!");
|
|
|
129 // Replace the cached tokens with the single annotation token.
|
|
|
130 if (i < CachedLexPos)
|
|
|
131 CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
|
|
|
132 *AnnotBegin = Tok;
|
|
|
133 CachedLexPos = i;
|
|
|
134 return;
|
|
|
135 }
|
|
|
136 }
|
|
|
137 }
|
|
|
138
|
|
|
139 bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
|
|
|
140 // There's currently no cached token...
|
|
|
141 if (!CachedLexPos)
|
|
|
142 return false;
|
|
|
143
|
|
|
144 const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
|
|
|
145 if (LastCachedTok.getKind() != Tok.getKind())
|
|
|
146 return false;
|
|
|
147
|
|
|
148 int RelOffset = 0;
|
|
|
149 if ((!getSourceManager().isInSameSLocAddrSpace(
|
|
|
150 Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
|
|
|
151 RelOffset)
|
|
|
152 return false;
|
|
|
153
|
|
|
154 return true;
|
|
|
155 }
|
|
|
156
|
|
|
157 void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
|
|
|
158 assert(CachedLexPos != 0 && "Expected to have some cached tokens");
|
|
|
159 CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
|
|
|
160 NewToks.end());
|
|
|
161 CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
|
|
|
162 CachedLexPos += NewToks.size() - 1;
|
|
|
163 }
|
