xref: /freebsd/contrib/llvm-project/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp (revision cb14a3fe5122c879eae1fb480ed7ce82a699ddb6)
1 //===- VerifyDiagnosticConsumer.cpp - Verifying Diagnostic Client ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This is a concrete diagnostic client, which buffers the diagnostic messages.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Frontend/VerifyDiagnosticConsumer.h"
14 #include "clang/Basic/CharInfo.h"
15 #include "clang/Basic/Diagnostic.h"
16 #include "clang/Basic/DiagnosticOptions.h"
17 #include "clang/Basic/FileManager.h"
18 #include "clang/Basic/LLVM.h"
19 #include "clang/Basic/SourceLocation.h"
20 #include "clang/Basic/SourceManager.h"
21 #include "clang/Basic/TokenKinds.h"
22 #include "clang/Frontend/FrontendDiagnostic.h"
23 #include "clang/Frontend/TextDiagnosticBuffer.h"
24 #include "clang/Lex/HeaderSearch.h"
25 #include "clang/Lex/Lexer.h"
26 #include "clang/Lex/PPCallbacks.h"
27 #include "clang/Lex/Preprocessor.h"
28 #include "clang/Lex/Token.h"
29 #include "llvm/ADT/STLExtras.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/SmallString.h"
32 #include "llvm/ADT/StringRef.h"
33 #include "llvm/ADT/Twine.h"
34 #include "llvm/Support/ErrorHandling.h"
35 #include "llvm/Support/Regex.h"
36 #include "llvm/Support/raw_ostream.h"
37 #include <algorithm>
38 #include <cassert>
39 #include <cstddef>
40 #include <cstring>
41 #include <iterator>
42 #include <memory>
43 #include <string>
44 #include <utility>
45 #include <vector>
46 
47 using namespace clang;
48 
49 using Directive = VerifyDiagnosticConsumer::Directive;
50 using DirectiveList = VerifyDiagnosticConsumer::DirectiveList;
51 using ExpectedData = VerifyDiagnosticConsumer::ExpectedData;
52 
53 #ifndef NDEBUG
54 
55 namespace {
56 
57 class VerifyFileTracker : public PPCallbacks {
58   VerifyDiagnosticConsumer &Verify;
59   SourceManager &SM;
60 
61 public:
62   VerifyFileTracker(VerifyDiagnosticConsumer &Verify, SourceManager &SM)
63       : Verify(Verify), SM(SM) {}
64 
65   /// Hook into the preprocessor and update the list of parsed
66   /// files when the preprocessor indicates a new file is entered.
67   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
68                    SrcMgr::CharacteristicKind FileType,
69                    FileID PrevFID) override {
70     Verify.UpdateParsedFileStatus(SM, SM.getFileID(Loc),
71                                   VerifyDiagnosticConsumer::IsParsed);
72   }
73 };
74 
75 } // namespace
76 
77 #endif
78 
79 //===----------------------------------------------------------------------===//
80 // Checking diagnostics implementation.
81 //===----------------------------------------------------------------------===//
82 
83 using DiagList = TextDiagnosticBuffer::DiagList;
84 using const_diag_iterator = TextDiagnosticBuffer::const_iterator;
85 
86 namespace {
87 
88 /// StandardDirective - Directive with string matching.
89 class StandardDirective : public Directive {
90 public:
91   StandardDirective(SourceLocation DirectiveLoc, SourceLocation DiagnosticLoc,
92                     bool MatchAnyFileAndLine, bool MatchAnyLine, StringRef Text,
93                     unsigned Min, unsigned Max)
94       : Directive(DirectiveLoc, DiagnosticLoc, MatchAnyFileAndLine,
95                   MatchAnyLine, Text, Min, Max) {}
96 
97   bool isValid(std::string &Error) override {
98     // all strings are considered valid; even empty ones
99     return true;
100   }
101 
102   bool match(StringRef S) override { return S.contains(Text); }
103 };
104 
105 /// RegexDirective - Directive with regular-expression matching.
106 class RegexDirective : public Directive {
107 public:
108   RegexDirective(SourceLocation DirectiveLoc, SourceLocation DiagnosticLoc,
109                  bool MatchAnyFileAndLine, bool MatchAnyLine, StringRef Text,
110                  unsigned Min, unsigned Max, StringRef RegexStr)
111       : Directive(DirectiveLoc, DiagnosticLoc, MatchAnyFileAndLine,
112                   MatchAnyLine, Text, Min, Max),
113         Regex(RegexStr) {}
114 
115   bool isValid(std::string &Error) override {
116     return Regex.isValid(Error);
117   }
118 
119   bool match(StringRef S) override {
120     return Regex.match(S);
121   }
122 
123 private:
124   llvm::Regex Regex;
125 };
126 
127 class ParseHelper
128 {
129 public:
130   ParseHelper(StringRef S)
131       : Begin(S.begin()), End(S.end()), C(Begin), P(Begin) {}
132 
133   // Return true if string literal is next.
134   bool Next(StringRef S) {
135     P = C;
136     PEnd = C + S.size();
137     if (PEnd > End)
138       return false;
139     return memcmp(P, S.data(), S.size()) == 0;
140   }
141 
142   // Return true if number is next.
143   // Output N only if number is next.
144   bool Next(unsigned &N) {
145     unsigned TMP = 0;
146     P = C;
147     PEnd = P;
148     for (; PEnd < End && *PEnd >= '0' && *PEnd <= '9'; ++PEnd) {
149       TMP *= 10;
150       TMP += *PEnd - '0';
151     }
152     if (PEnd == C)
153       return false;
154     N = TMP;
155     return true;
156   }
157 
158   // Return true if a marker is next.
159   // A marker is the longest match for /#[A-Za-z0-9_-]+/.
160   bool NextMarker() {
161     P = C;
162     if (P == End || *P != '#')
163       return false;
164     PEnd = P;
165     ++PEnd;
166     while ((isAlphanumeric(*PEnd) || *PEnd == '-' || *PEnd == '_') &&
167            PEnd < End)
168       ++PEnd;
169     return PEnd > P + 1;
170   }
171 
172   // Return true if string literal S is matched in content.
173   // When true, P marks begin-position of the match, and calling Advance sets C
174   // to end-position of the match.
175   // If S is the empty string, then search for any letter instead (makes sense
176   // with FinishDirectiveToken=true).
177   // If EnsureStartOfWord, then skip matches that don't start a new word.
178   // If FinishDirectiveToken, then assume the match is the start of a comment
179   // directive for -verify, and extend the match to include the entire first
180   // token of that directive.
181   bool Search(StringRef S, bool EnsureStartOfWord = false,
182               bool FinishDirectiveToken = false) {
183     do {
184       if (!S.empty()) {
185         P = std::search(C, End, S.begin(), S.end());
186         PEnd = P + S.size();
187       }
188       else {
189         P = C;
190         while (P != End && !isLetter(*P))
191           ++P;
192         PEnd = P + 1;
193       }
194       if (P == End)
195         break;
196       // If not start of word but required, skip and search again.
197       if (EnsureStartOfWord
198                // Check if string literal starts a new word.
199           && !(P == Begin || isWhitespace(P[-1])
200                // Or it could be preceded by the start of a comment.
201                || (P > (Begin + 1) && (P[-1] == '/' || P[-1] == '*')
202                                    &&  P[-2] == '/')))
203         continue;
204       if (FinishDirectiveToken) {
205         while (PEnd != End && (isAlphanumeric(*PEnd)
206                                || *PEnd == '-' || *PEnd == '_'))
207           ++PEnd;
208         // Put back trailing digits and hyphens to be parsed later as a count
209         // or count range.  Because -verify prefixes must start with letters,
210         // we know the actual directive we found starts with a letter, so
211         // we won't put back the entire directive word and thus record an empty
212         // string.
213         assert(isLetter(*P) && "-verify prefix must start with a letter");
214         while (isDigit(PEnd[-1]) || PEnd[-1] == '-')
215           --PEnd;
216       }
217       return true;
218     } while (Advance());
219     return false;
220   }
221 
222   // Return true if a CloseBrace that closes the OpenBrace at the current nest
223   // level is found. When true, P marks begin-position of CloseBrace.
224   bool SearchClosingBrace(StringRef OpenBrace, StringRef CloseBrace) {
225     unsigned Depth = 1;
226     P = C;
227     while (P < End) {
228       StringRef S(P, End - P);
229       if (S.starts_with(OpenBrace)) {
230         ++Depth;
231         P += OpenBrace.size();
232       } else if (S.starts_with(CloseBrace)) {
233         --Depth;
234         if (Depth == 0) {
235           PEnd = P + CloseBrace.size();
236           return true;
237         }
238         P += CloseBrace.size();
239       } else {
240         ++P;
241       }
242     }
243     return false;
244   }
245 
246   // Advance 1-past previous next/search.
247   // Behavior is undefined if previous next/search failed.
248   bool Advance() {
249     C = PEnd;
250     return C < End;
251   }
252 
253   // Return the text matched by the previous next/search.
254   // Behavior is undefined if previous next/search failed.
255   StringRef Match() { return StringRef(P, PEnd - P); }
256 
257   // Skip zero or more whitespace.
258   void SkipWhitespace() {
259     for (; C < End && isWhitespace(*C); ++C)
260       ;
261   }
262 
263   // Return true if EOF reached.
264   bool Done() {
265     return !(C < End);
266   }
267 
268   // Beginning of expected content.
269   const char * const Begin;
270 
271   // End of expected content (1-past).
272   const char * const End;
273 
274   // Position of next char in content.
275   const char *C;
276 
277   // Previous next/search subject start.
278   const char *P;
279 
280 private:
281   // Previous next/search subject end (1-past).
282   const char *PEnd = nullptr;
283 };
284 
285 // The information necessary to create a directive.
286 struct UnattachedDirective {
287   DirectiveList *DL = nullptr;
288   bool RegexKind = false;
289   SourceLocation DirectivePos, ContentBegin;
290   std::string Text;
291   unsigned Min = 1, Max = 1;
292 };
293 
294 // Attach the specified directive to the line of code indicated by
295 // \p ExpectedLoc.
296 void attachDirective(DiagnosticsEngine &Diags, const UnattachedDirective &UD,
297                      SourceLocation ExpectedLoc,
298                      bool MatchAnyFileAndLine = false,
299                      bool MatchAnyLine = false) {
300   // Construct new directive.
301   std::unique_ptr<Directive> D = Directive::create(
302       UD.RegexKind, UD.DirectivePos, ExpectedLoc, MatchAnyFileAndLine,
303       MatchAnyLine, UD.Text, UD.Min, UD.Max);
304 
305   std::string Error;
306   if (!D->isValid(Error)) {
307     Diags.Report(UD.ContentBegin, diag::err_verify_invalid_content)
308       << (UD.RegexKind ? "regex" : "string") << Error;
309   }
310 
311   UD.DL->push_back(std::move(D));
312 }
313 
314 } // anonymous
315 
316 // Tracker for markers in the input files. A marker is a comment of the form
317 //
318 //   n = 123; // #123
319 //
320 // ... that can be referred to by a later expected-* directive:
321 //
322 //   // expected-error@#123 {{undeclared identifier 'n'}}
323 //
324 // Marker declarations must be at the start of a comment or preceded by
325 // whitespace to distinguish them from uses of markers in directives.
326 class VerifyDiagnosticConsumer::MarkerTracker {
327   DiagnosticsEngine &Diags;
328 
329   struct Marker {
330     SourceLocation DefLoc;
331     SourceLocation RedefLoc;
332     SourceLocation UseLoc;
333   };
334   llvm::StringMap<Marker> Markers;
335 
336   // Directives that couldn't be created yet because they name an unknown
337   // marker.
338   llvm::StringMap<llvm::SmallVector<UnattachedDirective, 2>> DeferredDirectives;
339 
340 public:
341   MarkerTracker(DiagnosticsEngine &Diags) : Diags(Diags) {}
342 
343   // Register a marker.
344   void addMarker(StringRef MarkerName, SourceLocation Pos) {
345     auto InsertResult = Markers.insert(
346         {MarkerName, Marker{Pos, SourceLocation(), SourceLocation()}});
347 
348     Marker &M = InsertResult.first->second;
349     if (!InsertResult.second) {
350       // Marker was redefined.
351       M.RedefLoc = Pos;
352     } else {
353       // First definition: build any deferred directives.
354       auto Deferred = DeferredDirectives.find(MarkerName);
355       if (Deferred != DeferredDirectives.end()) {
356         for (auto &UD : Deferred->second) {
357           if (M.UseLoc.isInvalid())
358             M.UseLoc = UD.DirectivePos;
359           attachDirective(Diags, UD, Pos);
360         }
361         DeferredDirectives.erase(Deferred);
362       }
363     }
364   }
365 
366   // Register a directive at the specified marker.
367   void addDirective(StringRef MarkerName, const UnattachedDirective &UD) {
368     auto MarkerIt = Markers.find(MarkerName);
369     if (MarkerIt != Markers.end()) {
370       Marker &M = MarkerIt->second;
371       if (M.UseLoc.isInvalid())
372         M.UseLoc = UD.DirectivePos;
373       return attachDirective(Diags, UD, M.DefLoc);
374     }
375     DeferredDirectives[MarkerName].push_back(UD);
376   }
377 
378   // Ensure we have no remaining deferred directives, and no
379   // multiply-defined-and-used markers.
380   void finalize() {
381     for (auto &MarkerInfo : Markers) {
382       StringRef Name = MarkerInfo.first();
383       Marker &M = MarkerInfo.second;
384       if (M.RedefLoc.isValid() && M.UseLoc.isValid()) {
385         Diags.Report(M.UseLoc, diag::err_verify_ambiguous_marker) << Name;
386         Diags.Report(M.DefLoc, diag::note_verify_ambiguous_marker) << Name;
387         Diags.Report(M.RedefLoc, diag::note_verify_ambiguous_marker) << Name;
388       }
389     }
390 
391     for (auto &DeferredPair : DeferredDirectives) {
392       Diags.Report(DeferredPair.second.front().DirectivePos,
393                    diag::err_verify_no_such_marker)
394           << DeferredPair.first();
395     }
396   }
397 };
398 
399 /// ParseDirective - Go through the comment and see if it indicates expected
400 /// diagnostics. If so, then put them in the appropriate directive list.
401 ///
402 /// Returns true if any valid directives were found.
403 static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
404                            Preprocessor *PP, SourceLocation Pos,
405                            VerifyDiagnosticConsumer::DirectiveStatus &Status,
406                            VerifyDiagnosticConsumer::MarkerTracker &Markers) {
407   DiagnosticsEngine &Diags = PP ? PP->getDiagnostics() : SM.getDiagnostics();
408 
409   // First, scan the comment looking for markers.
410   for (ParseHelper PH(S); !PH.Done();) {
411     if (!PH.Search("#", true))
412       break;
413     PH.C = PH.P;
414     if (!PH.NextMarker()) {
415       PH.Next("#");
416       PH.Advance();
417       continue;
418     }
419     PH.Advance();
420     Markers.addMarker(PH.Match(), Pos);
421   }
422 
423   // A single comment may contain multiple directives.
424   bool FoundDirective = false;
425   for (ParseHelper PH(S); !PH.Done();) {
426     // Search for the initial directive token.
427     // If one prefix, save time by searching only for its directives.
428     // Otherwise, search for any potential directive token and check it later.
429     const auto &Prefixes = Diags.getDiagnosticOptions().VerifyPrefixes;
430     if (!(Prefixes.size() == 1 ? PH.Search(*Prefixes.begin(), true, true)
431                                : PH.Search("", true, true)))
432       break;
433 
434     StringRef DToken = PH.Match();
435     PH.Advance();
436 
437     // Default directive kind.
438     UnattachedDirective D;
439     const char *KindStr = "string";
440 
441     // Parse the initial directive token in reverse so we can easily determine
442     // its exact actual prefix.  If we were to parse it from the front instead,
443     // it would be harder to determine where the prefix ends because there
444     // might be multiple matching -verify prefixes because some might prefix
445     // others.
446 
447     // Regex in initial directive token: -re
448     if (DToken.ends_with("-re")) {
449       D.RegexKind = true;
450       KindStr = "regex";
451       DToken = DToken.substr(0, DToken.size()-3);
452     }
453 
454     // Type in initial directive token: -{error|warning|note|no-diagnostics}
455     bool NoDiag = false;
456     StringRef DType;
457     if (DToken.ends_with(DType = "-error"))
458       D.DL = ED ? &ED->Errors : nullptr;
459     else if (DToken.ends_with(DType = "-warning"))
460       D.DL = ED ? &ED->Warnings : nullptr;
461     else if (DToken.ends_with(DType = "-remark"))
462       D.DL = ED ? &ED->Remarks : nullptr;
463     else if (DToken.ends_with(DType = "-note"))
464       D.DL = ED ? &ED->Notes : nullptr;
465     else if (DToken.ends_with(DType = "-no-diagnostics")) {
466       NoDiag = true;
467       if (D.RegexKind)
468         continue;
469     } else
470       continue;
471     DToken = DToken.substr(0, DToken.size()-DType.size());
472 
473     // What's left in DToken is the actual prefix.  That might not be a -verify
474     // prefix even if there is only one -verify prefix (for example, the full
475     // DToken is foo-bar-warning, but foo is the only -verify prefix).
476     if (!std::binary_search(Prefixes.begin(), Prefixes.end(), DToken))
477       continue;
478 
479     if (NoDiag) {
480       if (Status == VerifyDiagnosticConsumer::HasOtherExpectedDirectives)
481         Diags.Report(Pos, diag::err_verify_invalid_no_diags)
482           << /*IsExpectedNoDiagnostics=*/true;
483       else
484         Status = VerifyDiagnosticConsumer::HasExpectedNoDiagnostics;
485       continue;
486     }
487     if (Status == VerifyDiagnosticConsumer::HasExpectedNoDiagnostics) {
488       Diags.Report(Pos, diag::err_verify_invalid_no_diags)
489         << /*IsExpectedNoDiagnostics=*/false;
490       continue;
491     }
492     Status = VerifyDiagnosticConsumer::HasOtherExpectedDirectives;
493 
494     // If a directive has been found but we're not interested
495     // in storing the directive information, return now.
496     if (!D.DL)
497       return true;
498 
499     // Next optional token: @
500     SourceLocation ExpectedLoc;
501     StringRef Marker;
502     bool MatchAnyFileAndLine = false;
503     bool MatchAnyLine = false;
504     if (!PH.Next("@")) {
505       ExpectedLoc = Pos;
506     } else {
507       PH.Advance();
508       unsigned Line = 0;
509       bool FoundPlus = PH.Next("+");
510       if (FoundPlus || PH.Next("-")) {
511         // Relative to current line.
512         PH.Advance();
513         bool Invalid = false;
514         unsigned ExpectedLine = SM.getSpellingLineNumber(Pos, &Invalid);
515         if (!Invalid && PH.Next(Line) && (FoundPlus || Line < ExpectedLine)) {
516           if (FoundPlus) ExpectedLine += Line;
517           else ExpectedLine -= Line;
518           ExpectedLoc = SM.translateLineCol(SM.getFileID(Pos), ExpectedLine, 1);
519         }
520       } else if (PH.Next(Line)) {
521         // Absolute line number.
522         if (Line > 0)
523           ExpectedLoc = SM.translateLineCol(SM.getFileID(Pos), Line, 1);
524       } else if (PH.NextMarker()) {
525         Marker = PH.Match();
526       } else if (PP && PH.Search(":")) {
527         // Specific source file.
528         StringRef Filename(PH.C, PH.P-PH.C);
529         PH.Advance();
530 
531         if (Filename == "*") {
532           MatchAnyFileAndLine = true;
533           if (!PH.Next("*")) {
534             Diags.Report(Pos.getLocWithOffset(PH.C - PH.Begin),
535                          diag::err_verify_missing_line)
536                 << "'*'";
537             continue;
538           }
539           MatchAnyLine = true;
540           ExpectedLoc = SourceLocation();
541         } else {
542           // Lookup file via Preprocessor, like a #include.
543           OptionalFileEntryRef File =
544               PP->LookupFile(Pos, Filename, false, nullptr, nullptr, nullptr,
545                              nullptr, nullptr, nullptr, nullptr, nullptr);
546           if (!File) {
547             Diags.Report(Pos.getLocWithOffset(PH.C - PH.Begin),
548                          diag::err_verify_missing_file)
549                 << Filename << KindStr;
550             continue;
551           }
552 
553           FileID FID = SM.translateFile(*File);
554           if (FID.isInvalid())
555             FID = SM.createFileID(*File, Pos, SrcMgr::C_User);
556 
557           if (PH.Next(Line) && Line > 0)
558             ExpectedLoc = SM.translateLineCol(FID, Line, 1);
559           else if (PH.Next("*")) {
560             MatchAnyLine = true;
561             ExpectedLoc = SM.translateLineCol(FID, 1, 1);
562           }
563         }
564       } else if (PH.Next("*")) {
565         MatchAnyLine = true;
566         ExpectedLoc = SourceLocation();
567       }
568 
569       if (ExpectedLoc.isInvalid() && !MatchAnyLine && Marker.empty()) {
570         Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
571                      diag::err_verify_missing_line) << KindStr;
572         continue;
573       }
574       PH.Advance();
575     }
576 
577     // Skip optional whitespace.
578     PH.SkipWhitespace();
579 
580     // Next optional token: positive integer or a '+'.
581     if (PH.Next(D.Min)) {
582       PH.Advance();
583       // A positive integer can be followed by a '+' meaning min
584       // or more, or by a '-' meaning a range from min to max.
585       if (PH.Next("+")) {
586         D.Max = Directive::MaxCount;
587         PH.Advance();
588       } else if (PH.Next("-")) {
589         PH.Advance();
590         if (!PH.Next(D.Max) || D.Max < D.Min) {
591           Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
592                        diag::err_verify_invalid_range) << KindStr;
593           continue;
594         }
595         PH.Advance();
596       } else {
597         D.Max = D.Min;
598       }
599     } else if (PH.Next("+")) {
600       // '+' on its own means "1 or more".
601       D.Max = Directive::MaxCount;
602       PH.Advance();
603     }
604 
605     // Skip optional whitespace.
606     PH.SkipWhitespace();
607 
608     // Next token: {{
609     if (!PH.Next("{{")) {
610       Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
611                    diag::err_verify_missing_start) << KindStr;
612       continue;
613     }
614     PH.Advance();
615     const char* const ContentBegin = PH.C; // mark content begin
616     // Search for token: }}
617     if (!PH.SearchClosingBrace("{{", "}}")) {
618       Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
619                    diag::err_verify_missing_end) << KindStr;
620       continue;
621     }
622     const char* const ContentEnd = PH.P; // mark content end
623     PH.Advance();
624 
625     D.DirectivePos = Pos;
626     D.ContentBegin = Pos.getLocWithOffset(ContentBegin - PH.Begin);
627 
628     // Build directive text; convert \n to newlines.
629     StringRef NewlineStr = "\\n";
630     StringRef Content(ContentBegin, ContentEnd-ContentBegin);
631     size_t CPos = 0;
632     size_t FPos;
633     while ((FPos = Content.find(NewlineStr, CPos)) != StringRef::npos) {
634       D.Text += Content.substr(CPos, FPos-CPos);
635       D.Text += '\n';
636       CPos = FPos + NewlineStr.size();
637     }
638     if (D.Text.empty())
639       D.Text.assign(ContentBegin, ContentEnd);
640 
641     // Check that regex directives contain at least one regex.
642     if (D.RegexKind && D.Text.find("{{") == StringRef::npos) {
643       Diags.Report(D.ContentBegin, diag::err_verify_missing_regex) << D.Text;
644       return false;
645     }
646 
647     if (Marker.empty())
648       attachDirective(Diags, D, ExpectedLoc, MatchAnyFileAndLine, MatchAnyLine);
649     else
650       Markers.addDirective(Marker, D);
651     FoundDirective = true;
652   }
653 
654   return FoundDirective;
655 }
656 
657 VerifyDiagnosticConsumer::VerifyDiagnosticConsumer(DiagnosticsEngine &Diags_)
658     : Diags(Diags_), PrimaryClient(Diags.getClient()),
659       PrimaryClientOwner(Diags.takeClient()),
660       Buffer(new TextDiagnosticBuffer()), Markers(new MarkerTracker(Diags)),
661       Status(HasNoDirectives) {
662   if (Diags.hasSourceManager())
663     setSourceManager(Diags.getSourceManager());
664 }
665 
666 VerifyDiagnosticConsumer::~VerifyDiagnosticConsumer() {
667   assert(!ActiveSourceFiles && "Incomplete parsing of source files!");
668   assert(!CurrentPreprocessor && "CurrentPreprocessor should be invalid!");
669   SrcManager = nullptr;
670   CheckDiagnostics();
671   assert(!Diags.ownsClient() &&
672          "The VerifyDiagnosticConsumer takes over ownership of the client!");
673 }
674 
675 // DiagnosticConsumer interface.
676 
677 void VerifyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
678                                                const Preprocessor *PP) {
679   // Attach comment handler on first invocation.
680   if (++ActiveSourceFiles == 1) {
681     if (PP) {
682       CurrentPreprocessor = PP;
683       this->LangOpts = &LangOpts;
684       setSourceManager(PP->getSourceManager());
685       const_cast<Preprocessor *>(PP)->addCommentHandler(this);
686 #ifndef NDEBUG
687       // Debug build tracks parsed files.
688       const_cast<Preprocessor *>(PP)->addPPCallbacks(
689                       std::make_unique<VerifyFileTracker>(*this, *SrcManager));
690 #endif
691     }
692   }
693 
694   assert((!PP || CurrentPreprocessor == PP) && "Preprocessor changed!");
695   PrimaryClient->BeginSourceFile(LangOpts, PP);
696 }
697 
698 void VerifyDiagnosticConsumer::EndSourceFile() {
699   assert(ActiveSourceFiles && "No active source files!");
700   PrimaryClient->EndSourceFile();
701 
702   // Detach comment handler once last active source file completed.
703   if (--ActiveSourceFiles == 0) {
704     if (CurrentPreprocessor)
705       const_cast<Preprocessor *>(CurrentPreprocessor)->
706           removeCommentHandler(this);
707 
708     // Diagnose any used-but-not-defined markers.
709     Markers->finalize();
710 
711     // Check diagnostics once last file completed.
712     CheckDiagnostics();
713     CurrentPreprocessor = nullptr;
714     LangOpts = nullptr;
715   }
716 }
717 
718 void VerifyDiagnosticConsumer::HandleDiagnostic(
719       DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
720   if (Info.hasSourceManager()) {
721     // If this diagnostic is for a different source manager, ignore it.
722     if (SrcManager && &Info.getSourceManager() != SrcManager)
723       return;
724 
725     setSourceManager(Info.getSourceManager());
726   }
727 
728 #ifndef NDEBUG
729   // Debug build tracks unparsed files for possible
730   // unparsed expected-* directives.
731   if (SrcManager) {
732     SourceLocation Loc = Info.getLocation();
733     if (Loc.isValid()) {
734       ParsedStatus PS = IsUnparsed;
735 
736       Loc = SrcManager->getExpansionLoc(Loc);
737       FileID FID = SrcManager->getFileID(Loc);
738 
739       auto FE = SrcManager->getFileEntryRefForID(FID);
740       if (FE && CurrentPreprocessor && SrcManager->isLoadedFileID(FID)) {
741         // If the file is a modules header file it shall not be parsed
742         // for expected-* directives.
743         HeaderSearch &HS = CurrentPreprocessor->getHeaderSearchInfo();
744         if (HS.findModuleForHeader(*FE))
745           PS = IsUnparsedNoDirectives;
746       }
747 
748       UpdateParsedFileStatus(*SrcManager, FID, PS);
749     }
750   }
751 #endif
752 
753   // Send the diagnostic to the buffer, we will check it once we reach the end
754   // of the source file (or are destructed).
755   Buffer->HandleDiagnostic(DiagLevel, Info);
756 }
757 
758 /// HandleComment - Hook into the preprocessor and extract comments containing
759 ///  expected errors and warnings.
760 bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
761                                              SourceRange Comment) {
762   SourceManager &SM = PP.getSourceManager();
763 
764   // If this comment is for a different source manager, ignore it.
765   if (SrcManager && &SM != SrcManager)
766     return false;
767 
768   SourceLocation CommentBegin = Comment.getBegin();
769 
770   const char *CommentRaw = SM.getCharacterData(CommentBegin);
771   StringRef C(CommentRaw, SM.getCharacterData(Comment.getEnd()) - CommentRaw);
772 
773   if (C.empty())
774     return false;
775 
776   // Fold any "\<EOL>" sequences
777   size_t loc = C.find('\\');
778   if (loc == StringRef::npos) {
779     ParseDirective(C, &ED, SM, &PP, CommentBegin, Status, *Markers);
780     return false;
781   }
782 
783   std::string C2;
784   C2.reserve(C.size());
785 
786   for (size_t last = 0;; loc = C.find('\\', last)) {
787     if (loc == StringRef::npos || loc == C.size()) {
788       C2 += C.substr(last);
789       break;
790     }
791     C2 += C.substr(last, loc-last);
792     last = loc + 1;
793 
794     if (C[last] == '\n' || C[last] == '\r') {
795       ++last;
796 
797       // Escape \r\n  or \n\r, but not \n\n.
798       if (last < C.size())
799         if (C[last] == '\n' || C[last] == '\r')
800           if (C[last] != C[last-1])
801             ++last;
802     } else {
803       // This was just a normal backslash.
804       C2 += '\\';
805     }
806   }
807 
808   if (!C2.empty())
809     ParseDirective(C2, &ED, SM, &PP, CommentBegin, Status, *Markers);
810   return false;
811 }
812 
813 #ifndef NDEBUG
814 /// Lex the specified source file to determine whether it contains
815 /// any expected-* directives.  As a Lexer is used rather than a full-blown
816 /// Preprocessor, directives inside skipped #if blocks will still be found.
817 ///
818 /// \return true if any directives were found.
819 static bool findDirectives(SourceManager &SM, FileID FID,
820                            const LangOptions &LangOpts) {
821   // Create a raw lexer to pull all the comments out of FID.
822   if (FID.isInvalid())
823     return false;
824 
825   // Create a lexer to lex all the tokens of the main file in raw mode.
826   llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID);
827   Lexer RawLex(FID, FromFile, SM, LangOpts);
828 
829   // Return comments as tokens, this is how we find expected diagnostics.
830   RawLex.SetCommentRetentionState(true);
831 
832   Token Tok;
833   Tok.setKind(tok::comment);
834   VerifyDiagnosticConsumer::DirectiveStatus Status =
835     VerifyDiagnosticConsumer::HasNoDirectives;
836   while (Tok.isNot(tok::eof)) {
837     RawLex.LexFromRawLexer(Tok);
838     if (!Tok.is(tok::comment)) continue;
839 
840     std::string Comment = RawLex.getSpelling(Tok, SM, LangOpts);
841     if (Comment.empty()) continue;
842 
843     // We don't care about tracking markers for this phase.
844     VerifyDiagnosticConsumer::MarkerTracker Markers(SM.getDiagnostics());
845 
846     // Find first directive.
847     if (ParseDirective(Comment, nullptr, SM, nullptr, Tok.getLocation(),
848                        Status, Markers))
849       return true;
850   }
851   return false;
852 }
853 #endif // !NDEBUG
854 
855 /// Takes a list of diagnostics that have been generated but not matched
856 /// by an expected-* directive and produces a diagnostic to the user from this.
857 static unsigned PrintUnexpected(DiagnosticsEngine &Diags, SourceManager *SourceMgr,
858                                 const_diag_iterator diag_begin,
859                                 const_diag_iterator diag_end,
860                                 const char *Kind) {
861   if (diag_begin == diag_end) return 0;
862 
863   SmallString<256> Fmt;
864   llvm::raw_svector_ostream OS(Fmt);
865   for (const_diag_iterator I = diag_begin, E = diag_end; I != E; ++I) {
866     if (I->first.isInvalid() || !SourceMgr)
867       OS << "\n  (frontend)";
868     else {
869       OS << "\n ";
870       if (OptionalFileEntryRef File =
871               SourceMgr->getFileEntryRefForID(SourceMgr->getFileID(I->first)))
872         OS << " File " << File->getName();
873       OS << " Line " << SourceMgr->getPresumedLineNumber(I->first);
874     }
875     OS << ": " << I->second;
876   }
877 
878   std::string Prefix = *Diags.getDiagnosticOptions().VerifyPrefixes.begin();
879   std::string KindStr = Prefix + "-" + Kind;
880   Diags.Report(diag::err_verify_inconsistent_diags).setForceEmit()
881       << KindStr << /*Unexpected=*/true << OS.str();
882   return std::distance(diag_begin, diag_end);
883 }
884 
885 /// Takes a list of diagnostics that were expected to have been generated
886 /// but were not and produces a diagnostic to the user from this.
887 static unsigned PrintExpected(DiagnosticsEngine &Diags,
888                               SourceManager &SourceMgr,
889                               std::vector<Directive *> &DL, const char *Kind) {
890   if (DL.empty())
891     return 0;
892 
893   SmallString<256> Fmt;
894   llvm::raw_svector_ostream OS(Fmt);
895   for (const auto *D : DL) {
896     if (D->DiagnosticLoc.isInvalid() || D->MatchAnyFileAndLine)
897       OS << "\n  File *";
898     else
899       OS << "\n  File " << SourceMgr.getFilename(D->DiagnosticLoc);
900     if (D->MatchAnyLine)
901       OS << " Line *";
902     else
903       OS << " Line " << SourceMgr.getPresumedLineNumber(D->DiagnosticLoc);
904     if (D->DirectiveLoc != D->DiagnosticLoc)
905       OS << " (directive at "
906          << SourceMgr.getFilename(D->DirectiveLoc) << ':'
907          << SourceMgr.getPresumedLineNumber(D->DirectiveLoc) << ')';
908     OS << ": " << D->Text;
909   }
910 
911   std::string Prefix = *Diags.getDiagnosticOptions().VerifyPrefixes.begin();
912   std::string KindStr = Prefix + "-" + Kind;
913   Diags.Report(diag::err_verify_inconsistent_diags).setForceEmit()
914       << KindStr << /*Unexpected=*/false << OS.str();
915   return DL.size();
916 }
917 
918 /// Determine whether two source locations come from the same file.
919 static bool IsFromSameFile(SourceManager &SM, SourceLocation DirectiveLoc,
920                            SourceLocation DiagnosticLoc) {
921   while (DiagnosticLoc.isMacroID())
922     DiagnosticLoc = SM.getImmediateMacroCallerLoc(DiagnosticLoc);
923 
924   if (SM.isWrittenInSameFile(DirectiveLoc, DiagnosticLoc))
925     return true;
926 
927   const FileEntry *DiagFile = SM.getFileEntryForID(SM.getFileID(DiagnosticLoc));
928   if (!DiagFile && SM.isWrittenInMainFile(DirectiveLoc))
929     return true;
930 
931   return (DiagFile == SM.getFileEntryForID(SM.getFileID(DirectiveLoc)));
932 }
933 
934 /// CheckLists - Compare expected to seen diagnostic lists and return the
935 /// the difference between them.
936 static unsigned CheckLists(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
937                            const char *Label,
938                            DirectiveList &Left,
939                            const_diag_iterator d2_begin,
940                            const_diag_iterator d2_end,
941                            bool IgnoreUnexpected) {
942   std::vector<Directive *> LeftOnly;
943   DiagList Right(d2_begin, d2_end);
944 
945   for (auto &Owner : Left) {
946     Directive &D = *Owner;
947     unsigned LineNo1 = SourceMgr.getPresumedLineNumber(D.DiagnosticLoc);
948 
949     for (unsigned i = 0; i < D.Max; ++i) {
950       DiagList::iterator II, IE;
951       for (II = Right.begin(), IE = Right.end(); II != IE; ++II) {
952         if (!D.MatchAnyLine) {
953           unsigned LineNo2 = SourceMgr.getPresumedLineNumber(II->first);
954           if (LineNo1 != LineNo2)
955             continue;
956         }
957 
958         if (!D.DiagnosticLoc.isInvalid() && !D.MatchAnyFileAndLine &&
959             !IsFromSameFile(SourceMgr, D.DiagnosticLoc, II->first))
960           continue;
961 
962         const std::string &RightText = II->second;
963         if (D.match(RightText))
964           break;
965       }
966       if (II == IE) {
967         // Not found.
968         if (i >= D.Min) break;
969         LeftOnly.push_back(&D);
970       } else {
971         // Found. The same cannot be found twice.
972         Right.erase(II);
973       }
974     }
975   }
976   // Now all that's left in Right are those that were not matched.
977   unsigned num = PrintExpected(Diags, SourceMgr, LeftOnly, Label);
978   if (!IgnoreUnexpected)
979     num += PrintUnexpected(Diags, &SourceMgr, Right.begin(), Right.end(), Label);
980   return num;
981 }
982 
983 /// CheckResults - This compares the expected results to those that
984 /// were actually reported. It emits any discrepencies. Return "true" if there
985 /// were problems. Return "false" otherwise.
986 static unsigned CheckResults(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
987                              const TextDiagnosticBuffer &Buffer,
988                              ExpectedData &ED) {
989   // We want to capture the delta between what was expected and what was
990   // seen.
991   //
992   //   Expected \ Seen - set expected but not seen
993   //   Seen \ Expected - set seen but not expected
994   unsigned NumProblems = 0;
995 
996   const DiagnosticLevelMask DiagMask =
997     Diags.getDiagnosticOptions().getVerifyIgnoreUnexpected();
998 
999   // See if there are error mismatches.
1000   NumProblems += CheckLists(Diags, SourceMgr, "error", ED.Errors,
1001                             Buffer.err_begin(), Buffer.err_end(),
1002                             bool(DiagnosticLevelMask::Error & DiagMask));
1003 
1004   // See if there are warning mismatches.
1005   NumProblems += CheckLists(Diags, SourceMgr, "warning", ED.Warnings,
1006                             Buffer.warn_begin(), Buffer.warn_end(),
1007                             bool(DiagnosticLevelMask::Warning & DiagMask));
1008 
1009   // See if there are remark mismatches.
1010   NumProblems += CheckLists(Diags, SourceMgr, "remark", ED.Remarks,
1011                             Buffer.remark_begin(), Buffer.remark_end(),
1012                             bool(DiagnosticLevelMask::Remark & DiagMask));
1013 
1014   // See if there are note mismatches.
1015   NumProblems += CheckLists(Diags, SourceMgr, "note", ED.Notes,
1016                             Buffer.note_begin(), Buffer.note_end(),
1017                             bool(DiagnosticLevelMask::Note & DiagMask));
1018 
1019   return NumProblems;
1020 }
1021 
1022 void VerifyDiagnosticConsumer::UpdateParsedFileStatus(SourceManager &SM,
1023                                                       FileID FID,
1024                                                       ParsedStatus PS) {
1025   // Check SourceManager hasn't changed.
1026   setSourceManager(SM);
1027 
1028 #ifndef NDEBUG
1029   if (FID.isInvalid())
1030     return;
1031 
1032   OptionalFileEntryRef FE = SM.getFileEntryRefForID(FID);
1033 
1034   if (PS == IsParsed) {
1035     // Move the FileID from the unparsed set to the parsed set.
1036     UnparsedFiles.erase(FID);
1037     ParsedFiles.insert(std::make_pair(FID, FE ? &FE->getFileEntry() : nullptr));
1038   } else if (!ParsedFiles.count(FID) && !UnparsedFiles.count(FID)) {
1039     // Add the FileID to the unparsed set if we haven't seen it before.
1040 
1041     // Check for directives.
1042     bool FoundDirectives;
1043     if (PS == IsUnparsedNoDirectives)
1044       FoundDirectives = false;
1045     else
1046       FoundDirectives = !LangOpts || findDirectives(SM, FID, *LangOpts);
1047 
1048     // Add the FileID to the unparsed set.
1049     UnparsedFiles.insert(std::make_pair(FID,
1050                                       UnparsedFileStatus(FE, FoundDirectives)));
1051   }
1052 #endif
1053 }
1054 
1055 void VerifyDiagnosticConsumer::CheckDiagnostics() {
1056   // Ensure any diagnostics go to the primary client.
1057   DiagnosticConsumer *CurClient = Diags.getClient();
1058   std::unique_ptr<DiagnosticConsumer> Owner = Diags.takeClient();
1059   Diags.setClient(PrimaryClient, false);
1060 
1061 #ifndef NDEBUG
1062   // In a debug build, scan through any files that may have been missed
1063   // during parsing and issue a fatal error if directives are contained
1064   // within these files.  If a fatal error occurs, this suggests that
1065   // this file is being parsed separately from the main file, in which
1066   // case consider moving the directives to the correct place, if this
1067   // is applicable.
1068   if (!UnparsedFiles.empty()) {
1069     // Generate a cache of parsed FileEntry pointers for alias lookups.
1070     llvm::SmallPtrSet<const FileEntry *, 8> ParsedFileCache;
1071     for (const auto &I : ParsedFiles)
1072       if (const FileEntry *FE = I.second)
1073         ParsedFileCache.insert(FE);
1074 
1075     // Iterate through list of unparsed files.
1076     for (const auto &I : UnparsedFiles) {
1077       const UnparsedFileStatus &Status = I.second;
1078       OptionalFileEntryRef FE = Status.getFile();
1079 
1080       // Skip files that have been parsed via an alias.
1081       if (FE && ParsedFileCache.count(*FE))
1082         continue;
1083 
1084       // Report a fatal error if this file contained directives.
1085       if (Status.foundDirectives()) {
1086         llvm::report_fatal_error("-verify directives found after rather"
1087                                  " than during normal parsing of " +
1088                                  (FE ? FE->getName() : "(unknown)"));
1089       }
1090     }
1091 
1092     // UnparsedFiles has been processed now, so clear it.
1093     UnparsedFiles.clear();
1094   }
1095 #endif // !NDEBUG
1096 
1097   if (SrcManager) {
1098     // Produce an error if no expected-* directives could be found in the
1099     // source file(s) processed.
1100     if (Status == HasNoDirectives) {
1101       Diags.Report(diag::err_verify_no_directives).setForceEmit();
1102       ++NumErrors;
1103       Status = HasNoDirectivesReported;
1104     }
1105 
1106     // Check that the expected diagnostics occurred.
1107     NumErrors += CheckResults(Diags, *SrcManager, *Buffer, ED);
1108   } else {
1109     const DiagnosticLevelMask DiagMask =
1110         ~Diags.getDiagnosticOptions().getVerifyIgnoreUnexpected();
1111     if (bool(DiagnosticLevelMask::Error & DiagMask))
1112       NumErrors += PrintUnexpected(Diags, nullptr, Buffer->err_begin(),
1113                                    Buffer->err_end(), "error");
1114     if (bool(DiagnosticLevelMask::Warning & DiagMask))
1115       NumErrors += PrintUnexpected(Diags, nullptr, Buffer->warn_begin(),
1116                                    Buffer->warn_end(), "warn");
1117     if (bool(DiagnosticLevelMask::Remark & DiagMask))
1118       NumErrors += PrintUnexpected(Diags, nullptr, Buffer->remark_begin(),
1119                                    Buffer->remark_end(), "remark");
1120     if (bool(DiagnosticLevelMask::Note & DiagMask))
1121       NumErrors += PrintUnexpected(Diags, nullptr, Buffer->note_begin(),
1122                                    Buffer->note_end(), "note");
1123   }
1124 
1125   Diags.setClient(CurClient, Owner.release() != nullptr);
1126 
1127   // Reset the buffer, we have processed all the diagnostics in it.
1128   Buffer.reset(new TextDiagnosticBuffer());
1129   ED.Reset();
1130 }
1131 
1132 std::unique_ptr<Directive> Directive::create(bool RegexKind,
1133                                              SourceLocation DirectiveLoc,
1134                                              SourceLocation DiagnosticLoc,
1135                                              bool MatchAnyFileAndLine,
1136                                              bool MatchAnyLine, StringRef Text,
1137                                              unsigned Min, unsigned Max) {
1138   if (!RegexKind)
1139     return std::make_unique<StandardDirective>(DirectiveLoc, DiagnosticLoc,
1140                                                MatchAnyFileAndLine,
1141                                                MatchAnyLine, Text, Min, Max);
1142 
1143   // Parse the directive into a regular expression.
1144   std::string RegexStr;
1145   StringRef S = Text;
1146   while (!S.empty()) {
1147     if (S.starts_with("{{")) {
1148       S = S.drop_front(2);
1149       size_t RegexMatchLength = S.find("}}");
1150       assert(RegexMatchLength != StringRef::npos);
1151       // Append the regex, enclosed in parentheses.
1152       RegexStr += "(";
1153       RegexStr.append(S.data(), RegexMatchLength);
1154       RegexStr += ")";
1155       S = S.drop_front(RegexMatchLength + 2);
1156     } else {
1157       size_t VerbatimMatchLength = S.find("{{");
1158       if (VerbatimMatchLength == StringRef::npos)
1159         VerbatimMatchLength = S.size();
1160       // Escape and append the fixed string.
1161       RegexStr += llvm::Regex::escape(S.substr(0, VerbatimMatchLength));
1162       S = S.drop_front(VerbatimMatchLength);
1163     }
1164   }
1165 
1166   return std::make_unique<RegexDirective>(DirectiveLoc, DiagnosticLoc,
1167                                           MatchAnyFileAndLine, MatchAnyLine,
1168                                           Text, Min, Max, RegexStr);
1169 }
1170