|
150
|
1 //===--- FormatTokenLexer.cpp - Lex FormatTokens -------------*- C++ ----*-===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 ///
|
|
|
9 /// \file
|
|
|
10 /// This file implements FormatTokenLexer, which tokenizes a source file
|
|
|
11 /// into a FormatToken stream suitable for ClangFormat.
|
|
|
12 ///
|
|
|
13 //===----------------------------------------------------------------------===//
|
|
|
14
|
|
|
15 #include "FormatTokenLexer.h"
|
|
|
16 #include "FormatToken.h"
|
|
|
17 #include "clang/Basic/SourceLocation.h"
|
|
|
18 #include "clang/Basic/SourceManager.h"
|
|
|
19 #include "clang/Format/Format.h"
|
|
|
20 #include "llvm/Support/Regex.h"
|
|
|
21
|
|
|
22 namespace clang {
|
|
|
23 namespace format {
|
|
|
24
|
|
|
25 FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
|
|
|
26 unsigned Column, const FormatStyle &Style,
|
|
|
27 encoding::Encoding Encoding)
|
|
|
28 : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
|
|
|
29 Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
|
|
|
30 Style(Style), IdentTable(getFormattingLangOpts(Style)),
|
|
|
31 Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
|
|
|
32 FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
|
|
|
33 MacroBlockEndRegex(Style.MacroBlockEnd) {
|
|
|
34 Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
|
|
|
35 getFormattingLangOpts(Style)));
|
|
|
36 Lex->SetKeepWhitespaceMode(true);
|
|
|
37
|
|
|
38 for (const std::string &ForEachMacro : Style.ForEachMacros)
|
|
|
39 Macros.insert({&IdentTable.get(ForEachMacro), TT_ForEachMacro});
|
|
|
40 for (const std::string &StatementMacro : Style.StatementMacros)
|
|
|
41 Macros.insert({&IdentTable.get(StatementMacro), TT_StatementMacro});
|
|
|
42 for (const std::string &TypenameMacro : Style.TypenameMacros)
|
|
|
43 Macros.insert({&IdentTable.get(TypenameMacro), TT_TypenameMacro});
|
|
|
44 for (const std::string &NamespaceMacro : Style.NamespaceMacros)
|
|
|
45 Macros.insert({&IdentTable.get(NamespaceMacro), TT_NamespaceMacro});
|
|
|
46 }
|
|
|
47
|
|
|
48 ArrayRef<FormatToken *> FormatTokenLexer::lex() {
|
|
|
49 assert(Tokens.empty());
|
|
|
50 assert(FirstInLineIndex == 0);
|
|
|
51 do {
|
|
|
52 Tokens.push_back(getNextToken());
|
|
|
53 if (Style.Language == FormatStyle::LK_JavaScript) {
|
|
|
54 tryParseJSRegexLiteral();
|
|
|
55 handleTemplateStrings();
|
|
|
56 }
|
|
|
57 if (Style.Language == FormatStyle::LK_TextProto)
|
|
|
58 tryParsePythonComment();
|
|
|
59 tryMergePreviousTokens();
|
|
|
60 if (Style.isCSharp())
|
|
|
61 // This needs to come after tokens have been merged so that C#
|
|
|
62 // string literals are correctly identified.
|
|
|
63 handleCSharpVerbatimAndInterpolatedStrings();
|
|
|
64 if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
|
|
|
65 FirstInLineIndex = Tokens.size() - 1;
|
|
|
66 } while (Tokens.back()->Tok.isNot(tok::eof));
|
|
|
67 return Tokens;
|
|
|
68 }
|
|
|
69
|
|
|
70 void FormatTokenLexer::tryMergePreviousTokens() {
|
|
|
71 if (tryMerge_TMacro())
|
|
|
72 return;
|
|
|
73 if (tryMergeConflictMarkers())
|
|
|
74 return;
|
|
|
75 if (tryMergeLessLess())
|
|
|
76 return;
|
|
173
|
77 if (tryMergeForEach())
|
|
|
78 return;
|
|
150
|
79
|
|
|
80 if (Style.isCSharp()) {
|
|
|
81 if (tryMergeCSharpKeywordVariables())
|
|
|
82 return;
|
|
|
83 if (tryMergeCSharpStringLiteral())
|
|
|
84 return;
|
|
|
85 if (tryMergeCSharpDoubleQuestion())
|
|
|
86 return;
|
|
173
|
87 if (tryMergeCSharpNullConditional())
|
|
150
|
88 return;
|
|
|
89 if (tryTransformCSharpForEach())
|
|
|
90 return;
|
|
|
91 static const tok::TokenKind JSRightArrow[] = {tok::equal, tok::greater};
|
|
|
92 if (tryMergeTokens(JSRightArrow, TT_JsFatArrow))
|
|
|
93 return;
|
|
|
94 }
|
|
|
95
|
|
|
96 if (tryMergeNSStringLiteral())
|
|
|
97 return;
|
|
|
98
|
|
|
99 if (Style.Language == FormatStyle::LK_JavaScript) {
|
|
|
100 static const tok::TokenKind JSIdentity[] = {tok::equalequal, tok::equal};
|
|
|
101 static const tok::TokenKind JSNotIdentity[] = {tok::exclaimequal,
|
|
|
102 tok::equal};
|
|
|
103 static const tok::TokenKind JSShiftEqual[] = {tok::greater, tok::greater,
|
|
|
104 tok::greaterequal};
|
|
|
105 static const tok::TokenKind JSRightArrow[] = {tok::equal, tok::greater};
|
|
|
106 static const tok::TokenKind JSExponentiation[] = {tok::star, tok::star};
|
|
|
107 static const tok::TokenKind JSExponentiationEqual[] = {tok::star,
|
|
|
108 tok::starequal};
|
|
|
109 static const tok::TokenKind JSNullPropagatingOperator[] = {tok::question,
|
|
|
110 tok::period};
|
|
|
111 static const tok::TokenKind JSNullishOperator[] = {tok::question,
|
|
|
112 tok::question};
|
|
|
113
|
|
|
114 // FIXME: Investigate what token type gives the correct operator priority.
|
|
|
115 if (tryMergeTokens(JSIdentity, TT_BinaryOperator))
|
|
|
116 return;
|
|
|
117 if (tryMergeTokens(JSNotIdentity, TT_BinaryOperator))
|
|
|
118 return;
|
|
|
119 if (tryMergeTokens(JSShiftEqual, TT_BinaryOperator))
|
|
|
120 return;
|
|
|
121 if (tryMergeTokens(JSRightArrow, TT_JsFatArrow))
|
|
|
122 return;
|
|
|
123 if (tryMergeTokens(JSExponentiation, TT_JsExponentiation))
|
|
|
124 return;
|
|
|
125 if (tryMergeTokens(JSExponentiationEqual, TT_JsExponentiationEqual)) {
|
|
|
126 Tokens.back()->Tok.setKind(tok::starequal);
|
|
|
127 return;
|
|
|
128 }
|
|
|
129 if (tryMergeTokens(JSNullishOperator, TT_JsNullishCoalescingOperator)) {
|
|
|
130 // Treat like the "||" operator (as opposed to the ternary ?).
|
|
|
131 Tokens.back()->Tok.setKind(tok::pipepipe);
|
|
|
132 return;
|
|
|
133 }
|
|
|
134 if (tryMergeTokens(JSNullPropagatingOperator,
|
|
|
135 TT_JsNullPropagatingOperator)) {
|
|
|
136 // Treat like a regular "." access.
|
|
|
137 Tokens.back()->Tok.setKind(tok::period);
|
|
|
138 return;
|
|
|
139 }
|
|
|
140 if (tryMergeJSPrivateIdentifier())
|
|
|
141 return;
|
|
|
142 }
|
|
|
143
|
|
|
144 if (Style.Language == FormatStyle::LK_Java) {
|
|
|
145 static const tok::TokenKind JavaRightLogicalShiftAssign[] = {
|
|
|
146 tok::greater, tok::greater, tok::greaterequal};
|
|
|
147 if (tryMergeTokens(JavaRightLogicalShiftAssign, TT_BinaryOperator))
|
|
|
148 return;
|
|
|
149 }
|
|
|
150 }
|
|
|
151
|
|
|
152 bool FormatTokenLexer::tryMergeNSStringLiteral() {
|
|
|
153 if (Tokens.size() < 2)
|
|
|
154 return false;
|
|
|
155 auto &At = *(Tokens.end() - 2);
|
|
|
156 auto &String = *(Tokens.end() - 1);
|
|
|
157 if (!At->is(tok::at) || !String->is(tok::string_literal))
|
|
|
158 return false;
|
|
|
159 At->Tok.setKind(tok::string_literal);
|
|
|
160 At->TokenText = StringRef(At->TokenText.begin(),
|
|
|
161 String->TokenText.end() - At->TokenText.begin());
|
|
|
162 At->ColumnWidth += String->ColumnWidth;
|
|
173
|
163 At->setType(TT_ObjCStringLiteral);
|
|
150
|
164 Tokens.erase(Tokens.end() - 1);
|
|
|
165 return true;
|
|
|
166 }
|
|
|
167
|
|
|
168 bool FormatTokenLexer::tryMergeJSPrivateIdentifier() {
|
|
|
169 // Merges #idenfier into a single identifier with the text #identifier
|
|
|
170 // but the token tok::identifier.
|
|
|
171 if (Tokens.size() < 2)
|
|
|
172 return false;
|
|
|
173 auto &Hash = *(Tokens.end() - 2);
|
|
|
174 auto &Identifier = *(Tokens.end() - 1);
|
|
|
175 if (!Hash->is(tok::hash) || !Identifier->is(tok::identifier))
|
|
|
176 return false;
|
|
|
177 Hash->Tok.setKind(tok::identifier);
|
|
|
178 Hash->TokenText =
|
|
|
179 StringRef(Hash->TokenText.begin(),
|
|
|
180 Identifier->TokenText.end() - Hash->TokenText.begin());
|
|
|
181 Hash->ColumnWidth += Identifier->ColumnWidth;
|
|
173
|
182 Hash->setType(TT_JsPrivateIdentifier);
|
|
150
|
183 Tokens.erase(Tokens.end() - 1);
|
|
|
184 return true;
|
|
|
185 }
|
|
|
186
|
|
|
187 // Search for verbatim or interpolated string literals @"ABC" or
|
|
|
188 // $"aaaaa{abc}aaaaa" i and mark the token as TT_CSharpStringLiteral, and to
|
|
|
189 // prevent splitting of @, $ and ".
|
|
|
190 // Merging of multiline verbatim strings with embedded '"' is handled in
|
|
|
191 // handleCSharpVerbatimAndInterpolatedStrings with lower-level lexing.
|
|
|
192 bool FormatTokenLexer::tryMergeCSharpStringLiteral() {
|
|
|
193 if (Tokens.size() < 2)
|
|
|
194 return false;
|
|
|
195
|
|
|
196 // Interpolated strings could contain { } with " characters inside.
|
|
|
197 // $"{x ?? "null"}"
|
|
|
198 // should not be split into $"{x ?? ", null, "}" but should treated as a
|
|
|
199 // single string-literal.
|
|
|
200 //
|
|
|
201 // We opt not to try and format expressions inside {} within a C#
|
|
|
202 // interpolated string. Formatting expressions within an interpolated string
|
|
|
203 // would require similar work as that done for JavaScript template strings
|
|
|
204 // in `handleTemplateStrings()`.
|
|
|
205 auto &CSharpInterpolatedString = *(Tokens.end() - 2);
|
|
173
|
206 if (CSharpInterpolatedString->getType() == TT_CSharpStringLiteral &&
|
|
150
|
207 (CSharpInterpolatedString->TokenText.startswith(R"($")") ||
|
|
|
208 CSharpInterpolatedString->TokenText.startswith(R"($@")"))) {
|
|
|
209 int UnmatchedOpeningBraceCount = 0;
|
|
|
210
|
|
|
211 auto TokenTextSize = CSharpInterpolatedString->TokenText.size();
|
|
|
212 for (size_t Index = 0; Index < TokenTextSize; ++Index) {
|
|
|
213 char C = CSharpInterpolatedString->TokenText[Index];
|
|
|
214 if (C == '{') {
|
|
|
215 // "{{" inside an interpolated string is an escaped '{' so skip it.
|
|
|
216 if (Index + 1 < TokenTextSize &&
|
|
|
217 CSharpInterpolatedString->TokenText[Index + 1] == '{') {
|
|
|
218 ++Index;
|
|
|
219 continue;
|
|
|
220 }
|
|
|
221 ++UnmatchedOpeningBraceCount;
|
|
|
222 } else if (C == '}') {
|
|
|
223 // "}}" inside an interpolated string is an escaped '}' so skip it.
|
|
|
224 if (Index + 1 < TokenTextSize &&
|
|
|
225 CSharpInterpolatedString->TokenText[Index + 1] == '}') {
|
|
|
226 ++Index;
|
|
|
227 continue;
|
|
|
228 }
|
|
|
229 --UnmatchedOpeningBraceCount;
|
|
|
230 }
|
|
|
231 }
|
|
|
232
|
|
|
233 if (UnmatchedOpeningBraceCount > 0) {
|
|
|
234 auto &NextToken = *(Tokens.end() - 1);
|
|
|
235 CSharpInterpolatedString->TokenText =
|
|
|
236 StringRef(CSharpInterpolatedString->TokenText.begin(),
|
|
|
237 NextToken->TokenText.end() -
|
|
|
238 CSharpInterpolatedString->TokenText.begin());
|
|
|
239 CSharpInterpolatedString->ColumnWidth += NextToken->ColumnWidth;
|
|
|
240 Tokens.erase(Tokens.end() - 1);
|
|
|
241 return true;
|
|
|
242 }
|
|
|
243 }
|
|
|
244
|
|
|
245 // Look for @"aaaaaa" or $"aaaaaa".
|
|
|
246 auto &String = *(Tokens.end() - 1);
|
|
|
247 if (!String->is(tok::string_literal))
|
|
|
248 return false;
|
|
|
249
|
|
|
250 auto &At = *(Tokens.end() - 2);
|
|
|
251 if (!(At->is(tok::at) || At->TokenText == "$"))
|
|
|
252 return false;
|
|
|
253
|
|
|
254 if (Tokens.size() > 2 && At->is(tok::at)) {
|
|
|
255 auto &Dollar = *(Tokens.end() - 3);
|
|
|
256 if (Dollar->TokenText == "$") {
|
|
|
257 // This looks like $@"aaaaa" so we need to combine all 3 tokens.
|
|
|
258 Dollar->Tok.setKind(tok::string_literal);
|
|
|
259 Dollar->TokenText =
|
|
|
260 StringRef(Dollar->TokenText.begin(),
|
|
|
261 String->TokenText.end() - Dollar->TokenText.begin());
|
|
|
262 Dollar->ColumnWidth += (At->ColumnWidth + String->ColumnWidth);
|
|
173
|
263 Dollar->setType(TT_CSharpStringLiteral);
|
|
150
|
264 Tokens.erase(Tokens.end() - 2);
|
|
|
265 Tokens.erase(Tokens.end() - 1);
|
|
|
266 return true;
|
|
|
267 }
|
|
|
268 }
|
|
|
269
|
|
|
270 // Convert back into just a string_literal.
|
|
|
271 At->Tok.setKind(tok::string_literal);
|
|
|
272 At->TokenText = StringRef(At->TokenText.begin(),
|
|
|
273 String->TokenText.end() - At->TokenText.begin());
|
|
|
274 At->ColumnWidth += String->ColumnWidth;
|
|
173
|
275 At->setType(TT_CSharpStringLiteral);
|
|
150
|
276 Tokens.erase(Tokens.end() - 1);
|
|
|
277 return true;
|
|
|
278 }
|
|
|
279
|
|
|
280 // Valid C# attribute targets:
|
|
|
281 // https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/attributes/#attribute-targets
|
|
|
282 const llvm::StringSet<> FormatTokenLexer::CSharpAttributeTargets = {
|
|
|
283 "assembly", "module", "field", "event", "method",
|
|
|
284 "param", "property", "return", "type",
|
|
|
285 };
|
|
|
286
|
|
|
287 bool FormatTokenLexer::tryMergeCSharpDoubleQuestion() {
|
|
|
288 if (Tokens.size() < 2)
|
|
|
289 return false;
|
|
|
290 auto &FirstQuestion = *(Tokens.end() - 2);
|
|
|
291 auto &SecondQuestion = *(Tokens.end() - 1);
|
|
|
292 if (!FirstQuestion->is(tok::question) || !SecondQuestion->is(tok::question))
|
|
|
293 return false;
|
|
173
|
294 FirstQuestion->Tok.setKind(tok::question); // no '??' in clang tokens.
|
|
150
|
295 FirstQuestion->TokenText = StringRef(FirstQuestion->TokenText.begin(),
|
|
|
296 SecondQuestion->TokenText.end() -
|
|
|
297 FirstQuestion->TokenText.begin());
|
|
|
298 FirstQuestion->ColumnWidth += SecondQuestion->ColumnWidth;
|
|
173
|
299 FirstQuestion->setType(TT_CSharpNullCoalescing);
|
|
|
300 Tokens.erase(Tokens.end() - 1);
|
|
|
301 return true;
|
|
|
302 }
|
|
|
303
|
|
|
304 // Merge '?[' and '?.' pairs into single tokens.
|
|
|
305 bool FormatTokenLexer::tryMergeCSharpNullConditional() {
|
|
|
306 if (Tokens.size() < 2)
|
|
|
307 return false;
|
|
|
308 auto &Question = *(Tokens.end() - 2);
|
|
|
309 auto &PeriodOrLSquare = *(Tokens.end() - 1);
|
|
|
310 if (!Question->is(tok::question) ||
|
|
|
311 !PeriodOrLSquare->isOneOf(tok::l_square, tok::period))
|
|
|
312 return false;
|
|
|
313 Question->TokenText =
|
|
|
314 StringRef(Question->TokenText.begin(),
|
|
|
315 PeriodOrLSquare->TokenText.end() - Question->TokenText.begin());
|
|
|
316 Question->ColumnWidth += PeriodOrLSquare->ColumnWidth;
|
|
|
317
|
|
|
318 if (PeriodOrLSquare->is(tok::l_square)) {
|
|
|
319 Question->Tok.setKind(tok::question); // no '?[' in clang tokens.
|
|
|
320 Question->setType(TT_CSharpNullConditionalLSquare);
|
|
|
321 } else {
|
|
|
322 Question->Tok.setKind(tok::question); // no '?.' in clang tokens.
|
|
|
323 Question->setType(TT_CSharpNullConditional);
|
|
|
324 }
|
|
|
325
|
|
150
|
326 Tokens.erase(Tokens.end() - 1);
|
|
|
327 return true;
|
|
|
328 }
|
|
|
329
|
|
|
330 bool FormatTokenLexer::tryMergeCSharpKeywordVariables() {
|
|
|
331 if (Tokens.size() < 2)
|
|
|
332 return false;
|
|
|
333 auto &At = *(Tokens.end() - 2);
|
|
|
334 auto &Keyword = *(Tokens.end() - 1);
|
|
|
335 if (!At->is(tok::at))
|
|
|
336 return false;
|
|
|
337 if (!Keywords.isCSharpKeyword(*Keyword))
|
|
|
338 return false;
|
|
|
339
|
|
|
340 At->Tok.setKind(tok::identifier);
|
|
|
341 At->TokenText = StringRef(At->TokenText.begin(),
|
|
|
342 Keyword->TokenText.end() - At->TokenText.begin());
|
|
|
343 At->ColumnWidth += Keyword->ColumnWidth;
|
|
173
|
344 At->setType(Keyword->getType());
|
|
150
|
345 Tokens.erase(Tokens.end() - 1);
|
|
|
346 return true;
|
|
|
347 }
|
|
|
348
|
|
|
349 // In C# transform identifier foreach into kw_foreach
|
|
|
350 bool FormatTokenLexer::tryTransformCSharpForEach() {
|
|
|
351 if (Tokens.size() < 1)
|
|
|
352 return false;
|
|
|
353 auto &Identifier = *(Tokens.end() - 1);
|
|
|
354 if (!Identifier->is(tok::identifier))
|
|
|
355 return false;
|
|
|
356 if (Identifier->TokenText != "foreach")
|
|
|
357 return false;
|
|
|
358
|
|
173
|
359 Identifier->setType(TT_ForEachMacro);
|
|
150
|
360 Identifier->Tok.setKind(tok::kw_for);
|
|
|
361 return true;
|
|
|
362 }
|
|
|
363
|
|
173
|
364 bool FormatTokenLexer::tryMergeForEach() {
|
|
|
365 if (Tokens.size() < 2)
|
|
|
366 return false;
|
|
|
367 auto &For = *(Tokens.end() - 2);
|
|
|
368 auto &Each = *(Tokens.end() - 1);
|
|
|
369 if (!For->is(tok::kw_for))
|
|
|
370 return false;
|
|
|
371 if (!Each->is(tok::identifier))
|
|
|
372 return false;
|
|
|
373 if (Each->TokenText != "each")
|
|
|
374 return false;
|
|
|
375
|
|
|
376 For->setType(TT_ForEachMacro);
|
|
|
377 For->Tok.setKind(tok::kw_for);
|
|
|
378
|
|
|
379 For->TokenText = StringRef(For->TokenText.begin(),
|
|
|
380 Each->TokenText.end() - For->TokenText.begin());
|
|
|
381 For->ColumnWidth += Each->ColumnWidth;
|
|
|
382 Tokens.erase(Tokens.end() - 1);
|
|
|
383 return true;
|
|
|
384 }
|
|
|
385
|
|
150
|
386 bool FormatTokenLexer::tryMergeLessLess() {
|
|
|
387 // Merge X,less,less,Y into X,lessless,Y unless X or Y is less.
|
|
|
388 if (Tokens.size() < 3)
|
|
|
389 return false;
|
|
|
390
|
|
|
391 bool FourthTokenIsLess = false;
|
|
|
392 if (Tokens.size() > 3)
|
|
|
393 FourthTokenIsLess = (Tokens.end() - 4)[0]->is(tok::less);
|
|
|
394
|
|
|
395 auto First = Tokens.end() - 3;
|
|
|
396 if (First[2]->is(tok::less) || First[1]->isNot(tok::less) ||
|
|
|
397 First[0]->isNot(tok::less) || FourthTokenIsLess)
|
|
|
398 return false;
|
|
|
399
|
|
|
400 // Only merge if there currently is no whitespace between the two "<".
|
|
|
401 if (First[1]->WhitespaceRange.getBegin() !=
|
|
|
402 First[1]->WhitespaceRange.getEnd())
|
|
|
403 return false;
|
|
|
404
|
|
|
405 First[0]->Tok.setKind(tok::lessless);
|
|
|
406 First[0]->TokenText = "<<";
|
|
|
407 First[0]->ColumnWidth += 1;
|
|
|
408 Tokens.erase(Tokens.end() - 2);
|
|
|
409 return true;
|
|
|
410 }
|
|
|
411
|
|
|
412 bool FormatTokenLexer::tryMergeTokens(ArrayRef<tok::TokenKind> Kinds,
|
|
|
413 TokenType NewType) {
|
|
|
414 if (Tokens.size() < Kinds.size())
|
|
|
415 return false;
|
|
|
416
|
|
|
417 SmallVectorImpl<FormatToken *>::const_iterator First =
|
|
|
418 Tokens.end() - Kinds.size();
|
|
|
419 if (!First[0]->is(Kinds[0]))
|
|
|
420 return false;
|
|
|
421 unsigned AddLength = 0;
|
|
|
422 for (unsigned i = 1; i < Kinds.size(); ++i) {
|
|
|
423 if (!First[i]->is(Kinds[i]) || First[i]->WhitespaceRange.getBegin() !=
|
|
|
424 First[i]->WhitespaceRange.getEnd())
|
|
|
425 return false;
|
|
|
426 AddLength += First[i]->TokenText.size();
|
|
|
427 }
|
|
|
428 Tokens.resize(Tokens.size() - Kinds.size() + 1);
|
|
|
429 First[0]->TokenText = StringRef(First[0]->TokenText.data(),
|
|
|
430 First[0]->TokenText.size() + AddLength);
|
|
|
431 First[0]->ColumnWidth += AddLength;
|
|
173
|
432 First[0]->setType(NewType);
|
|
150
|
433 return true;
|
|
|
434 }
|
|
|
435
|
|
|
436 // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
|
|
|
437 bool FormatTokenLexer::precedesOperand(FormatToken *Tok) {
|
|
|
438 // NB: This is not entirely correct, as an r_paren can introduce an operand
|
|
|
439 // location in e.g. `if (foo) /bar/.exec(...);`. That is a rare enough
|
|
|
440 // corner case to not matter in practice, though.
|
|
|
441 return Tok->isOneOf(tok::period, tok::l_paren, tok::comma, tok::l_brace,
|
|
|
442 tok::r_brace, tok::l_square, tok::semi, tok::exclaim,
|
|
|
443 tok::colon, tok::question, tok::tilde) ||
|
|
|
444 Tok->isOneOf(tok::kw_return, tok::kw_do, tok::kw_case, tok::kw_throw,
|
|
|
445 tok::kw_else, tok::kw_new, tok::kw_delete, tok::kw_void,
|
|
|
446 tok::kw_typeof, Keywords.kw_instanceof, Keywords.kw_in) ||
|
|
|
447 Tok->isBinaryOperator();
|
|
|
448 }
|
|
|
449
|
|
|
450 bool FormatTokenLexer::canPrecedeRegexLiteral(FormatToken *Prev) {
|
|
|
451 if (!Prev)
|
|
|
452 return true;
|
|
|
453
|
|
|
454 // Regex literals can only follow after prefix unary operators, not after
|
|
|
455 // postfix unary operators. If the '++' is followed by a non-operand
|
|
|
456 // introducing token, the slash here is the operand and not the start of a
|
|
|
457 // regex.
|
|
|
458 // `!` is an unary prefix operator, but also a post-fix operator that casts
|
|
|
459 // away nullability, so the same check applies.
|
|
|
460 if (Prev->isOneOf(tok::plusplus, tok::minusminus, tok::exclaim))
|
|
|
461 return (Tokens.size() < 3 || precedesOperand(Tokens[Tokens.size() - 3]));
|
|
|
462
|
|
|
463 // The previous token must introduce an operand location where regex
|
|
|
464 // literals can occur.
|
|
|
465 if (!precedesOperand(Prev))
|
|
|
466 return false;
|
|
|
467
|
|
|
468 return true;
|
|
|
469 }
|
|
|
470
|
|
|
471 // Tries to parse a JavaScript Regex literal starting at the current token,
|
|
|
472 // if that begins with a slash and is in a location where JavaScript allows
|
|
|
473 // regex literals. Changes the current token to a regex literal and updates
|
|
|
474 // its text if successful.
|
|
|
475 void FormatTokenLexer::tryParseJSRegexLiteral() {
|
|
|
476 FormatToken *RegexToken = Tokens.back();
|
|
|
477 if (!RegexToken->isOneOf(tok::slash, tok::slashequal))
|
|
|
478 return;
|
|
|
479
|
|
|
480 FormatToken *Prev = nullptr;
|
|
|
481 for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; ++I) {
|
|
|
482 // NB: Because previous pointers are not initialized yet, this cannot use
|
|
|
483 // Token.getPreviousNonComment.
|
|
|
484 if ((*I)->isNot(tok::comment)) {
|
|
|
485 Prev = *I;
|
|
|
486 break;
|
|
|
487 }
|
|
|
488 }
|
|
|
489
|
|
|
490 if (!canPrecedeRegexLiteral(Prev))
|
|
|
491 return;
|
|
|
492
|
|
|
493 // 'Manually' lex ahead in the current file buffer.
|
|
|
494 const char *Offset = Lex->getBufferLocation();
|
|
|
495 const char *RegexBegin = Offset - RegexToken->TokenText.size();
|
|
|
496 StringRef Buffer = Lex->getBuffer();
|
|
|
497 bool InCharacterClass = false;
|
|
|
498 bool HaveClosingSlash = false;
|
|
|
499 for (; !HaveClosingSlash && Offset != Buffer.end(); ++Offset) {
|
|
|
500 // Regular expressions are terminated with a '/', which can only be
|
|
|
501 // escaped using '\' or a character class between '[' and ']'.
|
|
|
502 // See http://www.ecma-international.org/ecma-262/5.1/#sec-7.8.5.
|
|
|
503 switch (*Offset) {
|
|
|
504 case '\\':
|
|
|
505 // Skip the escaped character.
|
|
|
506 ++Offset;
|
|
|
507 break;
|
|
|
508 case '[':
|
|
|
509 InCharacterClass = true;
|
|
|
510 break;
|
|
|
511 case ']':
|
|
|
512 InCharacterClass = false;
|
|
|
513 break;
|
|
|
514 case '/':
|
|
|
515 if (!InCharacterClass)
|
|
|
516 HaveClosingSlash = true;
|
|
|
517 break;
|
|
|
518 }
|
|
|
519 }
|
|
|
520
|
|
173
|
521 RegexToken->setType(TT_RegexLiteral);
|
|
150
|
522 // Treat regex literals like other string_literals.
|
|
|
523 RegexToken->Tok.setKind(tok::string_literal);
|
|
|
524 RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
|
|
|
525 RegexToken->ColumnWidth = RegexToken->TokenText.size();
|
|
|
526
|
|
|
527 resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
|
|
|
528 }
|
|
|
529
|
|
|
530 void FormatTokenLexer::handleCSharpVerbatimAndInterpolatedStrings() {
|
|
|
531 FormatToken *CSharpStringLiteral = Tokens.back();
|
|
|
532
|
|
173
|
533 if (CSharpStringLiteral->getType() != TT_CSharpStringLiteral)
|
|
150
|
534 return;
|
|
|
535
|
|
|
536 // Deal with multiline strings.
|
|
|
537 if (!(CSharpStringLiteral->TokenText.startswith(R"(@")") ||
|
|
|
538 CSharpStringLiteral->TokenText.startswith(R"($@")")))
|
|
|
539 return;
|
|
|
540
|
|
|
541 const char *StrBegin =
|
|
|
542 Lex->getBufferLocation() - CSharpStringLiteral->TokenText.size();
|
|
|
543 const char *Offset = StrBegin;
|
|
|
544 if (CSharpStringLiteral->TokenText.startswith(R"(@")"))
|
|
|
545 Offset += 2;
|
|
|
546 else // CSharpStringLiteral->TokenText.startswith(R"($@")")
|
|
|
547 Offset += 3;
|
|
|
548
|
|
|
549 // Look for a terminating '"' in the current file buffer.
|
|
|
550 // Make no effort to format code within an interpolated or verbatim string.
|
|
|
551 for (; Offset != Lex->getBuffer().end(); ++Offset) {
|
|
|
552 if (Offset[0] == '"') {
|
|
|
553 // "" within a verbatim string is an escaped double quote: skip it.
|
|
|
554 if (Offset + 1 < Lex->getBuffer().end() && Offset[1] == '"')
|
|
|
555 ++Offset;
|
|
|
556 else
|
|
|
557 break;
|
|
|
558 }
|
|
|
559 }
|
|
|
560
|
|
|
561 // Make no attempt to format code properly if a verbatim string is
|
|
|
562 // unterminated.
|
|
|
563 if (Offset == Lex->getBuffer().end())
|
|
|
564 return;
|
|
|
565
|
|
|
566 StringRef LiteralText(StrBegin, Offset - StrBegin + 1);
|
|
|
567 CSharpStringLiteral->TokenText = LiteralText;
|
|
|
568
|
|
|
569 // Adjust width for potentially multiline string literals.
|
|
|
570 size_t FirstBreak = LiteralText.find('\n');
|
|
|
571 StringRef FirstLineText = FirstBreak == StringRef::npos
|
|
|
572 ? LiteralText
|
|
|
573 : LiteralText.substr(0, FirstBreak);
|
|
|
574 CSharpStringLiteral->ColumnWidth = encoding::columnWidthWithTabs(
|
|
|
575 FirstLineText, CSharpStringLiteral->OriginalColumn, Style.TabWidth,
|
|
|
576 Encoding);
|
|
|
577 size_t LastBreak = LiteralText.rfind('\n');
|
|
|
578 if (LastBreak != StringRef::npos) {
|
|
|
579 CSharpStringLiteral->IsMultiline = true;
|
|
173
|
580 unsigned StartColumn = 0;
|
|
150
|
581 CSharpStringLiteral->LastLineColumnWidth = encoding::columnWidthWithTabs(
|
|
|
582 LiteralText.substr(LastBreak + 1, LiteralText.size()), StartColumn,
|
|
|
583 Style.TabWidth, Encoding);
|
|
|
584 }
|
|
|
585
|
|
|
586 SourceLocation loc = Offset < Lex->getBuffer().end()
|
|
|
587 ? Lex->getSourceLocation(Offset + 1)
|
|
|
588 : SourceMgr.getLocForEndOfFile(ID);
|
|
|
589 resetLexer(SourceMgr.getFileOffset(loc));
|
|
|
590 }
|
|
|
591
|
|
|
592 void FormatTokenLexer::handleTemplateStrings() {
|
|
|
593 FormatToken *BacktickToken = Tokens.back();
|
|
|
594
|
|
|
595 if (BacktickToken->is(tok::l_brace)) {
|
|
|
596 StateStack.push(LexerState::NORMAL);
|
|
|
597 return;
|
|
|
598 }
|
|
|
599 if (BacktickToken->is(tok::r_brace)) {
|
|
|
600 if (StateStack.size() == 1)
|
|
|
601 return;
|
|
|
602 StateStack.pop();
|
|
|
603 if (StateStack.top() != LexerState::TEMPLATE_STRING)
|
|
|
604 return;
|
|
|
605 // If back in TEMPLATE_STRING, fallthrough and continue parsing the
|
|
|
606 } else if (BacktickToken->is(tok::unknown) &&
|
|
|
607 BacktickToken->TokenText == "`") {
|
|
|
608 StateStack.push(LexerState::TEMPLATE_STRING);
|
|
|
609 } else {
|
|
|
610 return; // Not actually a template
|
|
|
611 }
|
|
|
612
|
|
|
613 // 'Manually' lex ahead in the current file buffer.
|
|
|
614 const char *Offset = Lex->getBufferLocation();
|
|
|
615 const char *TmplBegin = Offset - BacktickToken->TokenText.size(); // at "`"
|
|
|
616 for (; Offset != Lex->getBuffer().end(); ++Offset) {
|
|
|
617 if (Offset[0] == '`') {
|
|
|
618 StateStack.pop();
|
|
|
619 break;
|
|
|
620 }
|
|
|
621 if (Offset[0] == '\\') {
|
|
|
622 ++Offset; // Skip the escaped character.
|
|
|
623 } else if (Offset + 1 < Lex->getBuffer().end() && Offset[0] == '$' &&
|
|
|
624 Offset[1] == '{') {
|
|
|
625 // '${' introduces an expression interpolation in the template string.
|
|
|
626 StateStack.push(LexerState::NORMAL);
|
|
|
627 ++Offset;
|
|
|
628 break;
|
|
|
629 }
|
|
|
630 }
|
|
|
631
|
|
|
632 StringRef LiteralText(TmplBegin, Offset - TmplBegin + 1);
|
|
173
|
633 BacktickToken->setType(TT_TemplateString);
|
|
150
|
634 BacktickToken->Tok.setKind(tok::string_literal);
|
|
|
635 BacktickToken->TokenText = LiteralText;
|
|
|
636
|
|
|
637 // Adjust width for potentially multiline string literals.
|
|
|
638 size_t FirstBreak = LiteralText.find('\n');
|
|
|
639 StringRef FirstLineText = FirstBreak == StringRef::npos
|
|
|
640 ? LiteralText
|
|
|
641 : LiteralText.substr(0, FirstBreak);
|
|
|
642 BacktickToken->ColumnWidth = encoding::columnWidthWithTabs(
|
|
|
643 FirstLineText, BacktickToken->OriginalColumn, Style.TabWidth, Encoding);
|
|
|
644 size_t LastBreak = LiteralText.rfind('\n');
|
|
|
645 if (LastBreak != StringRef::npos) {
|
|
|
646 BacktickToken->IsMultiline = true;
|
|
|
647 unsigned StartColumn = 0; // The template tail spans the entire line.
|
|
|
648 BacktickToken->LastLineColumnWidth = encoding::columnWidthWithTabs(
|
|
|
649 LiteralText.substr(LastBreak + 1, LiteralText.size()), StartColumn,
|
|
|
650 Style.TabWidth, Encoding);
|
|
|
651 }
|
|
|
652
|
|
|
653 SourceLocation loc = Offset < Lex->getBuffer().end()
|
|
|
654 ? Lex->getSourceLocation(Offset + 1)
|
|
|
655 : SourceMgr.getLocForEndOfFile(ID);
|
|
|
656 resetLexer(SourceMgr.getFileOffset(loc));
|
|
|
657 }
|
|
|
658
|
|
|
659 void FormatTokenLexer::tryParsePythonComment() {
|
|
|
660 FormatToken *HashToken = Tokens.back();
|
|
|
661 if (!HashToken->isOneOf(tok::hash, tok::hashhash))
|
|
|
662 return;
|
|
|
663 // Turn the remainder of this line into a comment.
|
|
|
664 const char *CommentBegin =
|
|
|
665 Lex->getBufferLocation() - HashToken->TokenText.size(); // at "#"
|
|
|
666 size_t From = CommentBegin - Lex->getBuffer().begin();
|
|
|
667 size_t To = Lex->getBuffer().find_first_of('\n', From);
|
|
|
668 if (To == StringRef::npos)
|
|
|
669 To = Lex->getBuffer().size();
|
|
|
670 size_t Len = To - From;
|
|
173
|
671 HashToken->setType(TT_LineComment);
|
|
150
|
672 HashToken->Tok.setKind(tok::comment);
|
|
|
673 HashToken->TokenText = Lex->getBuffer().substr(From, Len);
|
|
|
674 SourceLocation Loc = To < Lex->getBuffer().size()
|
|
|
675 ? Lex->getSourceLocation(CommentBegin + Len)
|
|
|
676 : SourceMgr.getLocForEndOfFile(ID);
|
|
|
677 resetLexer(SourceMgr.getFileOffset(Loc));
|
|
|
678 }
|
|
|
679
|
|
|
680 bool FormatTokenLexer::tryMerge_TMacro() {
|
|
|
681 if (Tokens.size() < 4)
|
|
|
682 return false;
|
|
|
683 FormatToken *Last = Tokens.back();
|
|
|
684 if (!Last->is(tok::r_paren))
|
|
|
685 return false;
|
|
|
686
|
|
|
687 FormatToken *String = Tokens[Tokens.size() - 2];
|
|
|
688 if (!String->is(tok::string_literal) || String->IsMultiline)
|
|
|
689 return false;
|
|
|
690
|
|
|
691 if (!Tokens[Tokens.size() - 3]->is(tok::l_paren))
|
|
|
692 return false;
|
|
|
693
|
|
|
694 FormatToken *Macro = Tokens[Tokens.size() - 4];
|
|
|
695 if (Macro->TokenText != "_T")
|
|
|
696 return false;
|
|
|
697
|
|
|
698 const char *Start = Macro->TokenText.data();
|
|
|
699 const char *End = Last->TokenText.data() + Last->TokenText.size();
|
|
|
700 String->TokenText = StringRef(Start, End - Start);
|
|
|
701 String->IsFirst = Macro->IsFirst;
|
|
|
702 String->LastNewlineOffset = Macro->LastNewlineOffset;
|
|
|
703 String->WhitespaceRange = Macro->WhitespaceRange;
|
|
|
704 String->OriginalColumn = Macro->OriginalColumn;
|
|
|
705 String->ColumnWidth = encoding::columnWidthWithTabs(
|
|
|
706 String->TokenText, String->OriginalColumn, Style.TabWidth, Encoding);
|
|
|
707 String->NewlinesBefore = Macro->NewlinesBefore;
|
|
|
708 String->HasUnescapedNewline = Macro->HasUnescapedNewline;
|
|
|
709
|
|
|
710 Tokens.pop_back();
|
|
|
711 Tokens.pop_back();
|
|
|
712 Tokens.pop_back();
|
|
|
713 Tokens.back() = String;
|
|
|
714 return true;
|
|
|
715 }
|
|
|
716
|
|
|
717 bool FormatTokenLexer::tryMergeConflictMarkers() {
|
|
|
718 if (Tokens.back()->NewlinesBefore == 0 && Tokens.back()->isNot(tok::eof))
|
|
|
719 return false;
|
|
|
720
|
|
|
721 // Conflict lines look like:
|
|
|
722 // <marker> <text from the vcs>
|
|
|
723 // For example:
|
|
|
724 // >>>>>>> /file/in/file/system at revision 1234
|
|
|
725 //
|
|
|
726 // We merge all tokens in a line that starts with a conflict marker
|
|
|
727 // into a single token with a special token type that the unwrapped line
|
|
|
728 // parser will use to correctly rebuild the underlying code.
|
|
|
729
|
|
|
730 FileID ID;
|
|
|
731 // Get the position of the first token in the line.
|
|
|
732 unsigned FirstInLineOffset;
|
|
|
733 std::tie(ID, FirstInLineOffset) = SourceMgr.getDecomposedLoc(
|
|
|
734 Tokens[FirstInLineIndex]->getStartOfNonWhitespace());
|
|
|
735 StringRef Buffer = SourceMgr.getBuffer(ID)->getBuffer();
|
|
|
736 // Calculate the offset of the start of the current line.
|
|
|
737 auto LineOffset = Buffer.rfind('\n', FirstInLineOffset);
|
|
|
738 if (LineOffset == StringRef::npos) {
|
|
|
739 LineOffset = 0;
|
|
|
740 } else {
|
|
|
741 ++LineOffset;
|
|
|
742 }
|
|
|
743
|
|
|
744 auto FirstSpace = Buffer.find_first_of(" \n", LineOffset);
|
|
|
745 StringRef LineStart;
|
|
|
746 if (FirstSpace == StringRef::npos) {
|
|
|
747 LineStart = Buffer.substr(LineOffset);
|
|
|
748 } else {
|
|
|
749 LineStart = Buffer.substr(LineOffset, FirstSpace - LineOffset);
|
|
|
750 }
|
|
|
751
|
|
|
752 TokenType Type = TT_Unknown;
|
|
|
753 if (LineStart == "<<<<<<<" || LineStart == ">>>>") {
|
|
|
754 Type = TT_ConflictStart;
|
|
|
755 } else if (LineStart == "|||||||" || LineStart == "=======" ||
|
|
|
756 LineStart == "====") {
|
|
|
757 Type = TT_ConflictAlternative;
|
|
|
758 } else if (LineStart == ">>>>>>>" || LineStart == "<<<<") {
|
|
|
759 Type = TT_ConflictEnd;
|
|
|
760 }
|
|
|
761
|
|
|
762 if (Type != TT_Unknown) {
|
|
|
763 FormatToken *Next = Tokens.back();
|
|
|
764
|
|
|
765 Tokens.resize(FirstInLineIndex + 1);
|
|
|
766 // We do not need to build a complete token here, as we will skip it
|
|
|
767 // during parsing anyway (as we must not touch whitespace around conflict
|
|
|
768 // markers).
|
|
173
|
769 Tokens.back()->setType(Type);
|
|
150
|
770 Tokens.back()->Tok.setKind(tok::kw___unknown_anytype);
|
|
|
771
|
|
|
772 Tokens.push_back(Next);
|
|
|
773 return true;
|
|
|
774 }
|
|
|
775
|
|
|
776 return false;
|
|
|
777 }
|
|
|
778
|
|
|
779 FormatToken *FormatTokenLexer::getStashedToken() {
|
|
|
780 // Create a synthesized second '>' or '<' token.
|
|
|
781 Token Tok = FormatTok->Tok;
|
|
|
782 StringRef TokenText = FormatTok->TokenText;
|
|
|
783
|
|
|
784 unsigned OriginalColumn = FormatTok->OriginalColumn;
|
|
|
785 FormatTok = new (Allocator.Allocate()) FormatToken;
|
|
|
786 FormatTok->Tok = Tok;
|
|
|
787 SourceLocation TokLocation =
|
|
|
788 FormatTok->Tok.getLocation().getLocWithOffset(Tok.getLength() - 1);
|
|
|
789 FormatTok->Tok.setLocation(TokLocation);
|
|
|
790 FormatTok->WhitespaceRange = SourceRange(TokLocation, TokLocation);
|
|
|
791 FormatTok->TokenText = TokenText;
|
|
|
792 FormatTok->ColumnWidth = 1;
|
|
|
793 FormatTok->OriginalColumn = OriginalColumn + 1;
|
|
|
794
|
|
|
795 return FormatTok;
|
|
|
796 }
|
|
|
797
|
|
|
798 FormatToken *FormatTokenLexer::getNextToken() {
|
|
|
799 if (StateStack.top() == LexerState::TOKEN_STASHED) {
|
|
|
800 StateStack.pop();
|
|
|
801 return getStashedToken();
|
|
|
802 }
|
|
|
803
|
|
|
804 FormatTok = new (Allocator.Allocate()) FormatToken;
|
|
|
805 readRawToken(*FormatTok);
|
|
|
806 SourceLocation WhitespaceStart =
|
|
|
807 FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
|
|
|
808 FormatTok->IsFirst = IsFirstToken;
|
|
|
809 IsFirstToken = false;
|
|
|
810
|
|
|
811 // Consume and record whitespace until we find a significant token.
|
|
|
812 unsigned WhitespaceLength = TrailingWhitespace;
|
|
|
813 while (FormatTok->Tok.is(tok::unknown)) {
|
|
|
814 StringRef Text = FormatTok->TokenText;
|
|
|
815 auto EscapesNewline = [&](int pos) {
|
|
|
816 // A '\r' here is just part of '\r\n'. Skip it.
|
|
|
817 if (pos >= 0 && Text[pos] == '\r')
|
|
|
818 --pos;
|
|
|
819 // See whether there is an odd number of '\' before this.
|
|
|
820 // FIXME: This is wrong. A '\' followed by a newline is always removed,
|
|
|
821 // regardless of whether there is another '\' before it.
|
|
|
822 // FIXME: Newlines can also be escaped by a '?' '?' '/' trigraph.
|
|
|
823 unsigned count = 0;
|
|
|
824 for (; pos >= 0; --pos, ++count)
|
|
|
825 if (Text[pos] != '\\')
|
|
|
826 break;
|
|
|
827 return count & 1;
|
|
|
828 };
|
|
|
829 // FIXME: This miscounts tok:unknown tokens that are not just
|
|
|
830 // whitespace, e.g. a '`' character.
|
|
|
831 for (int i = 0, e = Text.size(); i != e; ++i) {
|
|
|
832 switch (Text[i]) {
|
|
|
833 case '\n':
|
|
|
834 ++FormatTok->NewlinesBefore;
|
|
|
835 FormatTok->HasUnescapedNewline = !EscapesNewline(i - 1);
|
|
|
836 FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
|
|
|
837 Column = 0;
|
|
|
838 break;
|
|
|
839 case '\r':
|
|
|
840 FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
|
|
|
841 Column = 0;
|
|
|
842 break;
|
|
|
843 case '\f':
|
|
|
844 case '\v':
|
|
|
845 Column = 0;
|
|
|
846 break;
|
|
|
847 case ' ':
|
|
|
848 ++Column;
|
|
|
849 break;
|
|
|
850 case '\t':
|
|
|
851 Column +=
|
|
|
852 Style.TabWidth - (Style.TabWidth ? Column % Style.TabWidth : 0);
|
|
|
853 break;
|
|
|
854 case '\\':
|
|
|
855 if (i + 1 == e || (Text[i + 1] != '\r' && Text[i + 1] != '\n'))
|
|
173
|
856 FormatTok->setType(TT_ImplicitStringLiteral);
|
|
150
|
857 break;
|
|
|
858 default:
|
|
173
|
859 FormatTok->setType(TT_ImplicitStringLiteral);
|
|
150
|
860 break;
|
|
|
861 }
|
|
173
|
862 if (FormatTok->getType() == TT_ImplicitStringLiteral)
|
|
150
|
863 break;
|
|
|
864 }
|
|
|
865
|
|
|
866 if (FormatTok->is(TT_ImplicitStringLiteral))
|
|
|
867 break;
|
|
|
868 WhitespaceLength += FormatTok->Tok.getLength();
|
|
|
869
|
|
|
870 readRawToken(*FormatTok);
|
|
|
871 }
|
|
|
872
|
|
|
873 // JavaScript and Java do not allow to escape the end of the line with a
|
|
|
874 // backslash. Backslashes are syntax errors in plain source, but can occur in
|
|
|
875 // comments. When a single line comment ends with a \, it'll cause the next
|
|
|
876 // line of code to be lexed as a comment, breaking formatting. The code below
|
|
|
877 // finds comments that contain a backslash followed by a line break, truncates
|
|
|
878 // the comment token at the backslash, and resets the lexer to restart behind
|
|
|
879 // the backslash.
|
|
|
880 if ((Style.Language == FormatStyle::LK_JavaScript ||
|
|
|
881 Style.Language == FormatStyle::LK_Java) &&
|
|
|
882 FormatTok->is(tok::comment) && FormatTok->TokenText.startswith("//")) {
|
|
|
883 size_t BackslashPos = FormatTok->TokenText.find('\\');
|
|
|
884 while (BackslashPos != StringRef::npos) {
|
|
|
885 if (BackslashPos + 1 < FormatTok->TokenText.size() &&
|
|
|
886 FormatTok->TokenText[BackslashPos + 1] == '\n') {
|
|
|
887 const char *Offset = Lex->getBufferLocation();
|
|
|
888 Offset -= FormatTok->TokenText.size();
|
|
|
889 Offset += BackslashPos + 1;
|
|
|
890 resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
|
|
|
891 FormatTok->TokenText = FormatTok->TokenText.substr(0, BackslashPos + 1);
|
|
|
892 FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
|
|
|
893 FormatTok->TokenText, FormatTok->OriginalColumn, Style.TabWidth,
|
|
|
894 Encoding);
|
|
|
895 break;
|
|
|
896 }
|
|
|
897 BackslashPos = FormatTok->TokenText.find('\\', BackslashPos + 1);
|
|
|
898 }
|
|
|
899 }
|
|
|
900
|
|
|
901 // In case the token starts with escaped newlines, we want to
|
|
|
902 // take them into account as whitespace - this pattern is quite frequent
|
|
|
903 // in macro definitions.
|
|
|
904 // FIXME: Add a more explicit test.
|
|
|
905 while (FormatTok->TokenText.size() > 1 && FormatTok->TokenText[0] == '\\') {
|
|
|
906 unsigned SkippedWhitespace = 0;
|
|
|
907 if (FormatTok->TokenText.size() > 2 &&
|
|
|
908 (FormatTok->TokenText[1] == '\r' && FormatTok->TokenText[2] == '\n'))
|
|
|
909 SkippedWhitespace = 3;
|
|
|
910 else if (FormatTok->TokenText[1] == '\n')
|
|
|
911 SkippedWhitespace = 2;
|
|
|
912 else
|
|
|
913 break;
|
|
|
914
|
|
|
915 ++FormatTok->NewlinesBefore;
|
|
|
916 WhitespaceLength += SkippedWhitespace;
|
|
|
917 FormatTok->LastNewlineOffset = SkippedWhitespace;
|
|
|
918 Column = 0;
|
|
|
919 FormatTok->TokenText = FormatTok->TokenText.substr(SkippedWhitespace);
|
|
|
920 }
|
|
|
921
|
|
|
922 FormatTok->WhitespaceRange = SourceRange(
|
|
|
923 WhitespaceStart, WhitespaceStart.getLocWithOffset(WhitespaceLength));
|
|
|
924
|
|
|
925 FormatTok->OriginalColumn = Column;
|
|
|
926
|
|
|
927 TrailingWhitespace = 0;
|
|
|
928 if (FormatTok->Tok.is(tok::comment)) {
|
|
|
929 // FIXME: Add the trimmed whitespace to Column.
|
|
|
930 StringRef UntrimmedText = FormatTok->TokenText;
|
|
|
931 FormatTok->TokenText = FormatTok->TokenText.rtrim(" \t\v\f");
|
|
|
932 TrailingWhitespace = UntrimmedText.size() - FormatTok->TokenText.size();
|
|
|
933 } else if (FormatTok->Tok.is(tok::raw_identifier)) {
|
|
|
934 IdentifierInfo &Info = IdentTable.get(FormatTok->TokenText);
|
|
|
935 FormatTok->Tok.setIdentifierInfo(&Info);
|
|
|
936 FormatTok->Tok.setKind(Info.getTokenID());
|
|
|
937 if (Style.Language == FormatStyle::LK_Java &&
|
|
|
938 FormatTok->isOneOf(tok::kw_struct, tok::kw_union, tok::kw_delete,
|
|
|
939 tok::kw_operator)) {
|
|
|
940 FormatTok->Tok.setKind(tok::identifier);
|
|
|
941 FormatTok->Tok.setIdentifierInfo(nullptr);
|
|
|
942 } else if (Style.Language == FormatStyle::LK_JavaScript &&
|
|
|
943 FormatTok->isOneOf(tok::kw_struct, tok::kw_union,
|
|
|
944 tok::kw_operator)) {
|
|
|
945 FormatTok->Tok.setKind(tok::identifier);
|
|
|
946 FormatTok->Tok.setIdentifierInfo(nullptr);
|
|
|
947 }
|
|
|
948 } else if (FormatTok->Tok.is(tok::greatergreater)) {
|
|
|
949 FormatTok->Tok.setKind(tok::greater);
|
|
|
950 FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
|
|
|
951 ++Column;
|
|
|
952 StateStack.push(LexerState::TOKEN_STASHED);
|
|
|
953 } else if (FormatTok->Tok.is(tok::lessless)) {
|
|
|
954 FormatTok->Tok.setKind(tok::less);
|
|
|
955 FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
|
|
|
956 ++Column;
|
|
|
957 StateStack.push(LexerState::TOKEN_STASHED);
|
|
|
958 }
|
|
|
959
|
|
|
960 // Now FormatTok is the next non-whitespace token.
|
|
|
961
|
|
|
962 StringRef Text = FormatTok->TokenText;
|
|
|
963 size_t FirstNewlinePos = Text.find('\n');
|
|
|
964 if (FirstNewlinePos == StringRef::npos) {
|
|
|
965 // FIXME: ColumnWidth actually depends on the start column, we need to
|
|
|
966 // take this into account when the token is moved.
|
|
|
967 FormatTok->ColumnWidth =
|
|
|
968 encoding::columnWidthWithTabs(Text, Column, Style.TabWidth, Encoding);
|
|
|
969 Column += FormatTok->ColumnWidth;
|
|
|
970 } else {
|
|
|
971 FormatTok->IsMultiline = true;
|
|
|
972 // FIXME: ColumnWidth actually depends on the start column, we need to
|
|
|
973 // take this into account when the token is moved.
|
|
|
974 FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
|
|
|
975 Text.substr(0, FirstNewlinePos), Column, Style.TabWidth, Encoding);
|
|
|
976
|
|
|
977 // The last line of the token always starts in column 0.
|
|
|
978 // Thus, the length can be precomputed even in the presence of tabs.
|
|
|
979 FormatTok->LastLineColumnWidth = encoding::columnWidthWithTabs(
|
|
|
980 Text.substr(Text.find_last_of('\n') + 1), 0, Style.TabWidth, Encoding);
|
|
|
981 Column = FormatTok->LastLineColumnWidth;
|
|
|
982 }
|
|
|
983
|
|
|
984 if (Style.isCpp()) {
|
|
|
985 auto it = Macros.find(FormatTok->Tok.getIdentifierInfo());
|
|
|
986 if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() &&
|
|
|
987 Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() ==
|
|
|
988 tok::pp_define) &&
|
|
|
989 it != Macros.end()) {
|
|
173
|
990 FormatTok->setType(it->second);
|
|
150
|
991 } else if (FormatTok->is(tok::identifier)) {
|
|
|
992 if (MacroBlockBeginRegex.match(Text)) {
|
|
173
|
993 FormatTok->setType(TT_MacroBlockBegin);
|
|
150
|
994 } else if (MacroBlockEndRegex.match(Text)) {
|
|
173
|
995 FormatTok->setType(TT_MacroBlockEnd);
|
|
150
|
996 }
|
|
|
997 }
|
|
|
998 }
|
|
|
999
|
|
|
1000 return FormatTok;
|
|
|
1001 }
|
|
|
1002
|
|
|
1003 void FormatTokenLexer::readRawToken(FormatToken &Tok) {
|
|
|
1004 Lex->LexFromRawLexer(Tok.Tok);
|
|
|
1005 Tok.TokenText = StringRef(SourceMgr.getCharacterData(Tok.Tok.getLocation()),
|
|
|
1006 Tok.Tok.getLength());
|
|
|
1007 // For formatting, treat unterminated string literals like normal string
|
|
|
1008 // literals.
|
|
|
1009 if (Tok.is(tok::unknown)) {
|
|
|
1010 if (!Tok.TokenText.empty() && Tok.TokenText[0] == '"') {
|
|
|
1011 Tok.Tok.setKind(tok::string_literal);
|
|
|
1012 Tok.IsUnterminatedLiteral = true;
|
|
|
1013 } else if (Style.Language == FormatStyle::LK_JavaScript &&
|
|
|
1014 Tok.TokenText == "''") {
|
|
|
1015 Tok.Tok.setKind(tok::string_literal);
|
|
|
1016 }
|
|
|
1017 }
|
|
|
1018
|
|
|
1019 if ((Style.Language == FormatStyle::LK_JavaScript ||
|
|
|
1020 Style.Language == FormatStyle::LK_Proto ||
|
|
|
1021 Style.Language == FormatStyle::LK_TextProto) &&
|
|
|
1022 Tok.is(tok::char_constant)) {
|
|
|
1023 Tok.Tok.setKind(tok::string_literal);
|
|
|
1024 }
|
|
|
1025
|
|
|
1026 if (Tok.is(tok::comment) && (Tok.TokenText == "// clang-format on" ||
|
|
|
1027 Tok.TokenText == "/* clang-format on */")) {
|
|
|
1028 FormattingDisabled = false;
|
|
|
1029 }
|
|
|
1030
|
|
|
1031 Tok.Finalized = FormattingDisabled;
|
|
|
1032
|
|
|
1033 if (Tok.is(tok::comment) && (Tok.TokenText == "// clang-format off" ||
|
|
|
1034 Tok.TokenText == "/* clang-format off */")) {
|
|
|
1035 FormattingDisabled = true;
|
|
|
1036 }
|
|
|
1037 }
|
|
|
1038
|
|
|
1039 void FormatTokenLexer::resetLexer(unsigned Offset) {
|
|
|
1040 StringRef Buffer = SourceMgr.getBufferData(ID);
|
|
|
1041 Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID),
|
|
|
1042 getFormattingLangOpts(Style), Buffer.begin(),
|
|
|
1043 Buffer.begin() + Offset, Buffer.end()));
|
|
|
1044 Lex->SetKeepWhitespaceMode(true);
|
|
|
1045 TrailingWhitespace = 0;
|
|
|
1046 }
|
|
|
1047
|
|
|
1048 } // namespace format
|
|
|
1049 } // namespace clang
|
