23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
35struct DirectiveWithTokens {
40 :
Kind(
Kind), NumTokens(NumTokens) {}
63 Scanner(StringRef Input,
66 : Input(Input), Tokens(Tokens), Diags(Diags),
67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
75 LangOpts.LineComment =
true;
89 lexToken(
const char *&
First,
const char *
const End);
92 const char *
const End);
94 void skipLine(
const char *&
First,
const char *
const End);
95 void skipDirective(StringRef Name,
const char *&
First,
const char *
const End);
106 [[nodiscard]] std::optional<StringRef>
107 tryLexIdentifierOrSkipLine(
const char *&
First,
const char *
const End);
110 [[nodiscard]] StringRef lexIdentifier(
const char *&
First,
111 const char *
const End);
118 [[nodiscard]]
bool isNextIdentifierOrSkipLine(StringRef
Id,
120 const char *
const End);
128 const char *
const End);
135 [[nodiscard]] std::optional<StringRef>
136 tryLexStringLiteralOrSkipLine(
const char *&
First,
const char *
const End);
138 [[nodiscard]]
bool scanImpl(
const char *
First,
const char *
const End);
139 [[nodiscard]]
bool lexPPLine(
const char *&
First,
const char *
const End);
140 [[nodiscard]]
bool lexAt(
const char *&
First,
const char *
const End);
141 [[nodiscard]]
bool lexModule(
const char *&
First,
const char *
const End);
142 [[nodiscard]]
bool lexDefine(
const char *HashLoc,
const char *&
First,
143 const char *
const End);
144 [[nodiscard]]
bool lexPragma(
const char *&
First,
const char *
const End);
145 [[nodiscard]]
bool lex_Pragma(
const char *&
First,
const char *
const End);
146 [[nodiscard]]
bool lexEndif(
const char *&
First,
const char *
const End);
148 const char *
const End);
149 [[nodiscard]]
bool lexModuleDirectiveBody(
DirectiveKind Kind,
151 const char *
const End);
152 void lexPPDirectiveBody(
const char *&
First,
const char *
const End);
155 Tokens.append(CurDirToks);
156 DirsWithToks.emplace_back(Kind, CurDirToks.size());
158 return DirsWithToks.back();
160 void popDirective() {
161 Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);
164 return DirsWithToks.empty() ?
pp_none : DirsWithToks.back().Kind;
167 unsigned getOffsetAt(
const char *CurPtr)
const {
168 return CurPtr - Input.data();
173 bool reportError(
const char *CurPtr,
unsigned Err);
175 StringMap<char> SplitIds;
181 const char *LastTokenPtr =
nullptr;
196bool Scanner::reportError(
const char *CurPtr,
unsigned Err) {
199 assert(CurPtr >= Input.data() &&
"invalid buffer ptr");
200 Diags->Report(InputSourceLoc.getLocWithOffset(getOffsetAt(CurPtr)), Err);
210 const char *Current) {
211 assert(
First <= Current);
214 if (*Current !=
'"' ||
First == Current)
225 if (*Current ==
'u' || *Current ==
'U' || *Current ==
'L')
229 if (*Current !=
'8' ||
First == Current || *Current-- !=
'u')
235 assert(
First[0] ==
'"');
236 assert(
First[-1] ==
'R');
258 while (
Last != End &&
size_t(
Last -
First) < Terminator.size() &&
267 if (
size_t(
Last -
First) < Terminator.size())
277static unsigned isEOL(
const char *
First,
const char *
const End) {
288 const char Terminator = *
First ==
'<' ?
'>' : *
First;
303 const char *FirstAfterBackslashPastSpace =
First;
305 if (
unsigned NLSize =
isEOL(FirstAfterBackslashPastSpace, End)) {
308 First = FirstAfterBackslashPastSpace + NLSize - 1;
321 assert(Len &&
"expected newline");
327 return *(
First - (
int)EOLLen - 1) ==
'\\';
345 if (
First[-1] !=
'\\')
361 if (End -
First < 4) {
375 const char *
const Cur,
376 const char *
const End) {
377 assert(*Cur ==
'\'' &&
"expected quotation character");
385 char Prev = *(Cur - 1);
386 if (Prev ==
'L' || Prev ==
'U' || Prev ==
'u')
388 if (Prev ==
'8' && (Cur - 1 != Start) && *(Cur - 2) ==
'u')
396void Scanner::skipLine(
const char *&
First,
const char *
const End) {
398 assert(
First <= End);
406 const char *Start =
First;
411 LastTokenPtr =
First;
421 LastTokenPtr =
First;
426 if (
First[1] ==
'/') {
432 if (
First[1] !=
'*') {
433 LastTokenPtr =
First;
451void Scanner::skipDirective(StringRef Name,
const char *&
First,
452 const char *
const End) {
453 if (llvm::StringSwitch<bool>(Name)
454 .Case(
"warning",
true)
460 skipLine(
First, End);
465 assert(
First <= End);
481 if (
First[1] ==
'/') {
496 const char *
const End) {
497 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
500 if (Tok.
is(tok::eof))
503 diag::err_dep_source_scanner_missing_semi_after_at_import);
504 if (Tok.
is(tok::semi))
513 DirectiveLoc, diag::err_dep_source_scanner_unexpected_tokens_at_import);
519 const char *
const End) {
521 TheLexer.LexFromRawLexer(Tok);
522 First = Input.data() + TheLexer.getCurrentBufferOffset();
523 assert(
First <= End);
525 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.
getLength();
528 return CurDirToks.back();
532Scanner::lexIncludeFilename(
const char *&
First,
const char *
const End) {
534 TheLexer.LexIncludeFilename(Tok);
535 First = Input.data() + TheLexer.getCurrentBufferOffset();
536 assert(
First <= End);
538 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.
getLength();
541 return CurDirToks.back();
544void Scanner::lexPPDirectiveBody(
const char *&
First,
const char *
const End) {
547 if (Tok.
is(tok::eod))
555 if (LLVM_LIKELY(!NeedsCleaning))
559 Spelling.resize(Tok.
Length);
564 unsigned SpellingLength = 0;
565 const char *BufPtr = Input.begin() + Tok.
Offset;
566 const char *AfterIdent = Input.begin() + Tok.
getEnd();
567 while (BufPtr < AfterIdent) {
569 Spelling[SpellingLength++] = Char;
573 return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)
577std::optional<StringRef>
578Scanner::tryLexIdentifierOrSkipLine(
const char *&
First,
const char *
const End) {
580 if (Tok.
isNot(tok::raw_identifier)) {
581 if (!Tok.
is(tok::eod))
582 skipLine(
First, End);
586 return cleanStringIfNeeded(Tok);
589StringRef Scanner::lexIdentifier(
const char *&
First,
const char *
const End) {
590 std::optional<StringRef>
Id = tryLexIdentifierOrSkipLine(
First, End);
591 assert(
Id &&
"expected identifier token");
595bool Scanner::isNextIdentifierOrSkipLine(StringRef
Id,
const char *&
First,
596 const char *
const End) {
597 if (std::optional<StringRef> FoundId =
598 tryLexIdentifierOrSkipLine(
First, End)) {
601 skipLine(
First, End);
607 const char *
const End) {
611 skipLine(
First, End);
615std::optional<StringRef>
616Scanner::tryLexStringLiteralOrSkipLine(
const char *&
First,
617 const char *
const End) {
620 if (!Tok.
is(tok::eod))
621 skipLine(
First, End);
625 return cleanStringIfNeeded(Tok);
628bool Scanner::lexAt(
const char *&
First,
const char *
const End) {
633 assert(AtTok.
is(tok::at));
636 if (!isNextIdentifierOrSkipLine(
"import",
First, End))
641bool Scanner::lexModule(
const char *&
First,
const char *
const End) {
642 StringRef
Id = lexIdentifier(
First, End);
644 if (
Id ==
"export") {
646 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(
First, End);
652 if (
Id !=
"module" &&
Id !=
"import") {
653 skipLine(
First, End);
669 skipLine(
First, End);
674 TheLexer.seek(getOffsetAt(
First),
false);
682 return lexModuleDirectiveBody(Kind,
First, End);
685bool Scanner::lex_Pragma(
const char *&
First,
const char *
const End) {
686 if (!isNextTokenOrSkipLine(tok::l_paren,
First, End))
689 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(
First, End);
691 if (!Str || !isNextTokenOrSkipLine(tok::r_paren,
First, End))
701 const char *
Begin = Buffer.c_str();
702 Scanner PragmaScanner{StringRef(
Begin, Buffer.size()), DiscardTokens, Diags,
705 PragmaScanner.TheLexer.setParsingPreprocessorDirective(
true);
706 if (PragmaScanner.lexPragma(
Begin, Buffer.end()))
711 skipLine(
First, End);
715 assert(
Begin == Buffer.end());
720bool Scanner::lexPragma(
const char *&
First,
const char *
const End) {
721 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(
First, End);
725 StringRef
Id = *FoundId;
726 auto Kind = llvm::StringSwitch<DirectiveKind>(
Id)
733 lexPPDirectiveBody(
First, End);
739 skipLine(
First, End);
743 FoundId = tryLexIdentifierOrSkipLine(
First, End);
749 if (
Id ==
"system_header") {
750 lexPPDirectiveBody(
First, End);
755 if (
Id !=
"module") {
756 skipLine(
First, End);
761 if (!isNextIdentifierOrSkipLine(
"import",
First, End))
765 lexPPDirectiveBody(
First, End);
770bool Scanner::lexEndif(
const char *&
First,
const char *
const End) {
783 skipLine(
First, End);
791 const char *
const End) {
792 lexPPDirectiveBody(
First, End);
810bool Scanner::lexPPLine(
const char *&
First,
const char *
const End) {
811 assert(
First != End);
814 assert(
First <= End);
819 skipLine(
First, End);
820 assert(
First <= End);
824 LastTokenPtr =
First;
826 TheLexer.seek(getOffsetAt(
First),
true);
828 auto ScEx1 = make_scope_exit([&]() {
836 return lexAt(
First, End);
839 return lexModule(
First, End);
842 if (isNextIdentifierOrSkipLine(
"_Pragma",
First, End))
843 return lex_Pragma(
First, End);
849 TheLexer.setParsingPreprocessorDirective(
true);
850 auto ScEx2 = make_scope_exit(
851 [&]() { TheLexer.setParsingPreprocessorDirective(
false); });
855 if (HashTok.is(tok::hashhash)) {
859 skipLine(
First, End);
860 assert(
First <= End);
863 assert(HashTok.is(tok::hash));
866 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(
First, End);
870 StringRef
Id = *FoundId;
873 return lexPragma(
First, End);
875 auto Kind = llvm::StringSwitch<DirectiveKind>(
Id)
897 return lexEndif(
First, End);
904 lexIncludeFilename(
First, End);
911 return lexDefault(Kind,
First, End);
920bool Scanner::scanImpl(
const char *
First,
const char *
const End) {
923 if (lexPPLine(
First, End))
929 bool Error = scanImpl(Input.begin(), Input.end());
934 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
940 for (
const DirectiveWithTokens &DirWithToks : DirsWithToks) {
941 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
942 Directives.emplace_back(DirWithToks.Kind,
943 RemainingTokens.take_front(DirWithToks.NumTokens));
944 RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);
946 assert(RemainingTokens.empty());
955 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
961 llvm::raw_ostream &OS) {
963 auto needsSpaceSeparator =
966 if (Prev == Tok.
Kind)
967 return !Tok.
isOneOf(tok::l_paren, tok::r_paren, tok::l_square,
969 if (Prev == tok::raw_identifier &&
970 Tok.
isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,
971 tok::char_constant, tok::header_name))
973 if (Prev == tok::r_paren &&
974 Tok.
isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,
975 tok::char_constant, tok::unknown))
977 if (Prev == tok::comma &&
978 Tok.
isOneOf(tok::l_paren, tok::string_literal, tok::less))
985 OS <<
"<TokBeforeEOF>";
986 std::optional<tok::TokenKind> PrevTokenKind;
988 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
990 PrevTokenKind = Tok.
Kind;
Defines the Diagnostic-related interfaces.
static void skipBlockComment(const char *&First, const char *const End)
static void skipRawString(const char *&First, const char *const End)
static void skipString(const char *&First, const char *const End)
static bool isStartOfRelevantLine(char First)
static void skipWhitespace(const char *&First, const char *const End)
static bool isRawStringLiteral(const char *First, const char *Current)
static void skipOverSpaces(const char *&First, const char *const End)
static unsigned isEOL(const char *First, const char *const End)
static void skipToNewlineRaw(const char *&First, const char *const End)
static unsigned skipNewline(const char *&First, const char *End)
static void skipUTF8ByteOrderMark(const char *&First, const char *const End)
static void skipLineComment(const char *&First, const char *const End)
static bool isQuoteCppDigitSeparator(const char *const Start, const char *const Cur, const char *const End)
static bool wasLineContinuation(const char *First, unsigned EOLLen)
This is the interface for scanning header and source files to get the minimum necessary preprocessor ...
Concrete class used by the front-end to report problems and issues.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
static SizedChar getCharAndSizeNoWarn(const char *Ptr, const LangOptions &LangOpts)
getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.
Encodes a location in the source.
Token - This structure provides full information about a lexed token.
unsigned getFlags() const
Return the internal represtation of the flags.
unsigned getLength() const
tok::TokenKind getKind() const
DirectiveKind
Represents the kind of preprocessor directive or a module declaration that is tracked by the scanner ...
@ tokens_present_before_eof
Indicates that there are tokens present between the last scanned directive and eof.
@ pp_pragma_system_header
@ pp_pragma_include_alias
bool isStringLiteral(TokenKind K)
Return true if this is a C or C++ string-literal (or C++11 user-defined-string-literal) token.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
The JSON file list parser is used to communicate input to InstallAPI.
LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Returns true if this character is vertical ASCII whitespace: '\n', '\r'.
LLVM_READONLY bool isAsciiIdentifierContinue(unsigned char c)
void printDependencyDirectivesAsSource(StringRef Source, ArrayRef< dependency_directives_scan::Directive > Directives, llvm::raw_ostream &OS)
Print the previously scanned dependency directives as minimized source text.
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
LLVM_READONLY bool isWhitespace(unsigned char c)
Return true if this character is horizontal or vertical ASCII whitespace: ' ', '\t',...
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
void prepare_PragmaString(SmallVectorImpl< char > &StrVal)
Destringize a _Pragma("") string according to C11 6.10.9.1: "The string literal is destringized by de...
Diagnostic wrappers for TextAPI types for error reporting.
Represents a directive that's lexed as part of the dependency directives scanning.
DirectiveKind Kind
The kind of token.
Token lexed as part of dependency directive scanning.
bool isNot(tok::TokenKind K) const
unsigned Offset
Offset into the original source input.
bool is(tok::TokenKind K) const
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const