lib/Parser/token-parsers.h - llvm-project/flang - Git at Google

 //===-- lib/Parser/token-parsers.h ------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//

 #ifndef FORTRAN_PARSER_TOKEN_PARSERS_H_
 #define FORTRAN_PARSER_TOKEN_PARSERS_H_

 // These parsers are driven by the parsers of the Fortran grammar to consume
 // the prescanned character stream and recognize context-sensitive tokens.

 #include "basic-parsers.h"
 #include "type-parsers.h"
 #include "flang/Common/idioms.h"
 #include "flang/Parser/char-set.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/instrumented-parser.h"
 #include "flang/Parser/provenance.h"
 #include <cstddef>
 #include <cstring>
 #include <functional>
 #include <limits>
 #include <list>
 #include <optional>
 #include <string>

 namespace Fortran::parser {

 // "xyz"_ch matches one instance of the characters x, y, or z without skipping
 // any spaces before or after.  The parser returns the location of the character
 // on success.
 class AnyOfChars {
 public:
   using resultType = const char *;
   constexpr AnyOfChars(const AnyOfChars &) = default;
   constexpr AnyOfChars(SetOfChars set) : set_{set} {}
   std::optional<const char *> Parse(ParseState &state) const {
     if (std::optional<const char *> at{state.PeekAtNextChar()}) {
       if (set_.Has(**at)) {
         state.UncheckedAdvance();
         state.set_anyTokenMatched();
         return at;
       }
     }
     state.Say(MessageExpectedText{set_});
     return std::nullopt;
   }

 private:
   const SetOfChars set_;
 };

 constexpr AnyOfChars operator""_ch(const char str[], std::size_t n) {
   return AnyOfChars{SetOfChars(str, n)};
 }

 constexpr auto letter{"abcdefghijklmnopqrstuvwxyz"_ch};
 constexpr auto digit{"0123456789"_ch};

 // Skips over optional spaces.  Always succeeds.
 struct Space {
   using resultType = Success;
   constexpr Space() {}
   static std::optional<Success> Parse(ParseState &state) {
     while (std::optional<const char *> p{state.PeekAtNextChar()}) {
       if (**p != ' ') {
         break;
       }
       state.UncheckedAdvance();
     }
     return {Success{}};
   }
 };
 constexpr Space space;

 // Skips a space that in free form requires a warning if it precedes a
 // character that could begin an identifier or keyword.  Always succeeds.
 inline void MissingSpace(ParseState &state) {
   if (!state.inFixedForm()) {
     state.Nonstandard(
         LanguageFeature::OptionalFreeFormSpace, "missing space"_port_en_US);
   }
 }

 struct SpaceCheck {
   using resultType = Success;
   constexpr SpaceCheck() {}
   static std::optional<Success> Parse(ParseState &state) {
     if (std::optional<const char *> p{state.PeekAtNextChar()}) {
       char ch{**p};
       if (ch == ' ') {
         state.UncheckedAdvance();
         return space.Parse(state);
       }
       if (IsLegalInIdentifier(ch)) {
         MissingSpace(state);
       }
     }
     return {Success{}};
   }
 };
 constexpr SpaceCheck spaceCheck;

 // Matches a token string.  Spaces in the token string denote where
 // spaces may appear in the source; they can be made mandatory for
 // some free form keyword sequences.  Missing mandatory spaces in free
 // form elicit a warning; they are not necessary for recognition.
 // Spaces before and after the token are also skipped.
 //
 // Token strings appear in the grammar as C++ user-defined literals
 // like "BIND ( C )"_tok and "SYNC ALL"_sptok.  The _tok suffix is implied
 // when a string literal appears before the sequencing operator >> or
 // after the sequencing operator /.  The literal "..."_id parses a
 // token that cannot be a prefix of a longer identifier.
 template <bool MandatoryFreeFormSpace = false, bool MustBeComplete = false>
 class TokenStringMatch {
 public:
   using resultType = Success;
   constexpr TokenStringMatch(const TokenStringMatch &) = default;
   constexpr TokenStringMatch(const char *str, std::size_t n)
       : str_{str}, bytes_{n} {}
   explicit constexpr TokenStringMatch(const char *str) : str_{str} {}
   std::optional<Success> Parse(ParseState &state) const {
     space.Parse(state);
     const char *start{state.GetLocation()};
     const char *p{str_};
     std::optional<const char *> at; // initially empty
     for (std::size_t j{0}; j < bytes_ && *p != '\0'; ++j, ++p) {
       bool spaceSkipping{*p == ' '};
       if (spaceSkipping) {
         if (j + 1 == bytes_ || p[1] == ' ' || p[1] == '\0') {
           continue; // redundant; ignore
         }
       }
       if (!at) {
         at = nextCh.Parse(state);
         if (!at) {
           return std::nullopt;
         }
       }
       if (spaceSkipping) {
         if (**at == ' ') {
           at = nextCh.Parse(state);
           if (!at) {
             return std::nullopt;
           }
         } else if constexpr (MandatoryFreeFormSpace) {
           MissingSpace(state);
         }
         // 'at' remains full for next iteration
       } else if (**at == ToLowerCaseLetter(*p)) {
         at.reset();
       } else {
         state.Say(start, MessageExpectedText{str_, bytes_});
         return std::nullopt;
       }
     }
     if constexpr (MustBeComplete) {
       if (auto after{state.PeekAtNextChar()}) {
         if (IsLegalInIdentifier(**after)) {
           state.Say(start, MessageExpectedText{str_, bytes_});
           return std::nullopt;
         }
       }
     }
     state.set_anyTokenMatched();
     if (IsLegalInIdentifier(p[-1])) {
       return spaceCheck.Parse(state);
     } else {
       return space.Parse(state);
     }
   }

 private:
   const char *const str_;
   const std::size_t bytes_{std::string::npos};
 };

 constexpr TokenStringMatch<> operator""_tok(const char str[], std::size_t n) {
   return {str, n};
 }

 constexpr TokenStringMatch<true> operator""_sptok(
     const char str[], std::size_t n) {
   return {str, n};
 }

 constexpr TokenStringMatch<false, true> operator""_id(
     const char str[], std::size_t n) {
   return {str, n};
 }

 template <class PA>
 inline constexpr std::enable_if_t<std::is_class_v<PA>,
     SequenceParser<TokenStringMatch<>, PA>>
 operator>>(const char *str, const PA &p) {
   return SequenceParser<TokenStringMatch<>, PA>{TokenStringMatch<>{str}, p};
 }

 template <class PA>
 inline constexpr std::enable_if_t<std::is_class_v<PA>,
     FollowParser<PA, TokenStringMatch<>>>
 operator/(const PA &p, const char *str) {
   return FollowParser<PA, TokenStringMatch<>>{p, TokenStringMatch<>{str}};
 }

 template <class PA> inline constexpr auto parenthesized(const PA &p) {
   return "(" >> p / ")";
 }

 template <class PA> inline constexpr auto bracketed(const PA &p) {
   return "[" >> p / "]";
 }

 // Quoted character literal constants.
 struct CharLiteralChar {
   using resultType = std::pair<char, bool /* was escaped */>;
   static std::optional<resultType> Parse(ParseState &state) {
     auto at{state.GetLocation()};
     if (std::optional<const char *> cp{nextCh.Parse(state)}) {
       char ch{**cp};
       if (ch == '\n') {
         state.Say(CharBlock{at, state.GetLocation()},
             "Unclosed character constant"_err_en_US);
         return std::nullopt;
       }
       if (ch == '\\') {
         // Most escape sequences in character literals are processed later,
         // but we have to look for quotes here so that doubled quotes work.
         if (std::optional<const char *> next{state.PeekAtNextChar()}) {
           char escaped{**next};
           if (escaped == '\'' || escaped == '"' || escaped == '\\') {
             state.UncheckedAdvance();
             return std::make_pair(escaped, true);
           }
         }
       }
       return std::make_pair(ch, false);
     }
     return std::nullopt;
   }
 };

 template <char quote> struct CharLiteral {
   using resultType = std::string;
   static std::optional<std::string> Parse(ParseState &state) {
     std::string str;
     static constexpr auto nextch{attempt(CharLiteralChar{})};
     while (auto ch{nextch.Parse(state)}) {
       if (ch->second) {
         str += '\\';
       } else if (ch->first == quote) {
         static constexpr auto doubled{attempt(AnyOfChars{SetOfChars{quote}})};
         if (!doubled.Parse(state)) {
           return str;
         }
       }
       str += ch->first;
     }
     return std::nullopt;
   }
 };

 // Parse "BOZ" binary literal quoted constants.
 // As extensions, support X as an alternate hexadecimal marker, and allow
 // BOZX markers to appear as suffixes.
 struct BOZLiteral {
   using resultType = std::string;
   static std::optional<resultType> Parse(ParseState &state) {
     char base{'\0'};
     auto baseChar{[&base](char ch) -> bool {
       switch (ch) {
       case 'b':
       case 'o':
       case 'z':
         base = ch;
         return true;
       case 'x':
         base = 'z';
         return true;
       default:
         return false;
       }
     }};

     space.Parse(state);
     const char *start{state.GetLocation()};
     std::optional<const char *> at{nextCh.Parse(state)};
     if (!at) {
       return std::nullopt;
     }
     if (**at == 'x' &&
         !state.IsNonstandardOk(LanguageFeature::BOZExtensions,
             "nonstandard BOZ literal"_port_en_US)) {
       return std::nullopt;
     }
     if (baseChar(**at)) {
       at = nextCh.Parse(state);
       if (!at) {
         return std::nullopt;
       }
     }

     char quote = **at;
     if (quote != '\'' && quote != '"') {
       return std::nullopt;
     }

     std::string content;
     while (true) {
       at = nextCh.Parse(state);
       if (!at) {
         return std::nullopt;
       }
       if (**at == quote) {
         break;
       }
       if (**at == ' ') {
         continue;
       }
       if (!IsHexadecimalDigit(**at)) {
         return std::nullopt;
       }
       content += ToLowerCaseLetter(**at);
     }

     if (!base) {
       // extension: base allowed to appear as suffix, too
       if (!(at = nextCh.Parse(state)) || !baseChar(**at) ||
           !state.IsNonstandardOk(LanguageFeature::BOZExtensions,
               "nonstandard BOZ literal"_port_en_US)) {
         return std::nullopt;
       }
       spaceCheck.Parse(state);
     }

     if (content.empty()) {
       state.Say(start, "no digit in BOZ literal"_err_en_US);
       return std::nullopt;
     }
     return {std::string{base} + '"' + content + '"'};
   }
 };

 // R711 digit-string -> digit [digit]...
 // N.B. not a token -- no space is skipped
 struct DigitString {
   using resultType = CharBlock;
   static std::optional<resultType> Parse(ParseState &state) {
     if (std::optional<const char *> ch1{state.PeekAtNextChar()}) {
       if (IsDecimalDigit(**ch1)) {
         state.UncheckedAdvance();
         while (std::optional<const char *> p{state.PeekAtNextChar()}) {
           if (!IsDecimalDigit(**p)) {
             break;
           }
           state.UncheckedAdvance();
         }
         return CharBlock{*ch1, state.GetLocation()};
       }
     }
     return std::nullopt;
   }
 };
 constexpr DigitString digitString;

 struct SignedIntLiteralConstantWithoutKind {
   using resultType = CharBlock;
   static std::optional<resultType> Parse(ParseState &state) {
     resultType result{state.GetLocation()};
     static constexpr auto sign{maybe("+-"_ch / space)};
     if (sign.Parse(state)) {
       if (auto digits{digitString.Parse(state)}) {
         result.ExtendToCover(*digits);
         return result;
       }
     }
     return std::nullopt;
   }
 };

 struct DigitString64 {
   using resultType = std::uint64_t;
   static std::optional<std::uint64_t> Parse(ParseState &state) {
     std::optional<const char *> firstDigit{digit.Parse(state)};
     if (!firstDigit) {
       return std::nullopt;
     }
     std::uint64_t value = **firstDigit - '0';
     bool overflow{false};
     static constexpr auto getDigit{attempt(digit)};
     while (auto nextDigit{getDigit.Parse(state)}) {
       if (value > std::numeric_limits<std::uint64_t>::max() / 10) {
         overflow = true;
       }
       value *= 10;
       int digitValue = **nextDigit - '0';
       if (value > std::numeric_limits<std::uint64_t>::max() - digitValue) {
         overflow = true;
       }
       value += digitValue;
     }
     if (overflow) {
       state.Say(*firstDigit, "overflow in decimal literal"_err_en_US);
     }
     return {value};
   }
 };
 constexpr DigitString64 digitString64;

 // R707 signed-int-literal-constant -> [sign] int-literal-constant
 // N.B. Spaces are consumed before and after the sign, since the sign
 // and the int-literal-constant are distinct tokens.  Does not
 // handle a trailing kind parameter.
 static std::optional<std::int64_t> SignedInteger(
     const std::optional<std::uint64_t> &x, Location at, bool negate,
     ParseState &state) {
   if (!x) {
     return std::nullopt;
   }
   std::uint64_t limit{std::numeric_limits<std::int64_t>::max()};
   if (negate) {
     limit = -(limit + 1);
   }
   if (*x > limit) {
     state.Say(at, "overflow in signed decimal literal"_err_en_US);
   }
   std::int64_t value = *x;
   return std::make_optional<std::int64_t>(negate ? -value : value);
 }

 // R710 signed-digit-string -> [sign] digit-string
 // N.B. Not a complete token -- no space is skipped.
 // Used only in the exponent parts of real literal constants.
 struct SignedDigitString {
   using resultType = std::int64_t;
   static std::optional<std::int64_t> Parse(ParseState &state) {
     std::optional<const char *> sign{state.PeekAtNextChar()};
     if (!sign) {
       return std::nullopt;
     }
     bool negate{**sign == '-'};
     if (negate || **sign == '+') {
       state.UncheckedAdvance();
     }
     return SignedInteger(digitString64.Parse(state), *sign, negate, state);
   }
 };

 // Variants of the above for use in FORMAT specifications, where spaces
 // must be ignored.
 struct DigitStringIgnoreSpaces {
   using resultType = std::uint64_t;
   static std::optional<std::uint64_t> Parse(ParseState &state) {
     static constexpr auto getFirstDigit{space >> digit};
     std::optional<const char *> firstDigit{getFirstDigit.Parse(state)};
     if (!firstDigit) {
       return std::nullopt;
     }
     std::uint64_t value = **firstDigit - '0';
     bool overflow{false};
     static constexpr auto getDigit{space >> attempt(digit)};
     while (auto nextDigit{getDigit.Parse(state)}) {
       if (value > std::numeric_limits<std::uint64_t>::max() / 10) {
         overflow = true;
       }
       value *= 10;
       int digitValue = **nextDigit - '0';
       if (value > std::numeric_limits<std::uint64_t>::max() - digitValue) {
         overflow = true;
       }
       value += digitValue;
     }
     if (overflow) {
       state.Say(*firstDigit, "overflow in decimal literal"_err_en_US);
     }
     return value;
   }
 };

 struct PositiveDigitStringIgnoreSpaces {
   using resultType = std::int64_t;
   static std::optional<std::int64_t> Parse(ParseState &state) {
     Location at{state.GetLocation()};
     return SignedInteger(
         DigitStringIgnoreSpaces{}.Parse(state), at, false /*positive*/, state);
   }
 };

 struct SignedDigitStringIgnoreSpaces {
   using resultType = std::int64_t;
   static std::optional<std::int64_t> Parse(ParseState &state) {
     static constexpr auto getSign{space >> attempt("+-"_ch)};
     bool negate{false};
     if (std::optional<const char *> sign{getSign.Parse(state)}) {
       negate = **sign == '-';
     }
     Location at{state.GetLocation()};
     return SignedInteger(
         DigitStringIgnoreSpaces{}.Parse(state), at, negate, state);
   }
 };

 // Legacy feature: Hollerith literal constants
 struct HollerithLiteral {
   using resultType = std::string;
   static std::optional<std::string> Parse(ParseState &state) {
     space.Parse(state);
     const char *start{state.GetLocation()};
     std::optional<std::uint64_t> charCount{
         DigitStringIgnoreSpaces{}.Parse(state)};
     if (!charCount || *charCount < 1) {
       return std::nullopt;
     }
     static constexpr auto letterH{"h"_ch};
     std::optional<const char *> h{letterH.Parse(state)};
     if (!h) {
       return std::nullopt;
     }
     std::string content;
     for (auto j{*charCount}; j-- > 0;) {
       int chBytes{UTF_8CharacterBytes(state.GetLocation())};
       for (int bytes{chBytes}; bytes > 0; --bytes) {
         if (std::optional<const char *> at{nextCh.Parse(state)}) {
           if (chBytes == 1 && !IsPrintable(**at)) {
             state.Say(start, "Bad character in Hollerith"_err_en_US);
             return std::nullopt;
           }
           content += **at;
         } else {
           state.Say(start, "Insufficient characters in Hollerith"_err_en_US);
           return std::nullopt;
         }
       }
     }
     return content;
   }
 };

 struct ConsumedAllInputParser {
   using resultType = Success;
   constexpr ConsumedAllInputParser() {}
   static inline std::optional<Success> Parse(ParseState &state) {
     if (state.IsAtEnd()) {
       return {Success{}};
     }
     return std::nullopt;
   }
 };
 constexpr ConsumedAllInputParser consumedAllInput;

 template <char goal> struct SkipPast {
   using resultType = Success;
   constexpr SkipPast() {}
   constexpr SkipPast(const SkipPast &) {}
   static std::optional<Success> Parse(ParseState &state) {
     while (std::optional<const char *> p{state.GetNextChar()}) {
       if (**p == goal) {
         return {Success{}};
       } else if (**p == '\n') {
         break;
       }
     }
     return std::nullopt;
   }
 };

 template <char goal> struct SkipTo {
   using resultType = Success;
   constexpr SkipTo() {}
   constexpr SkipTo(const SkipTo &) {}
   static std::optional<Success> Parse(ParseState &state) {
     while (std::optional<const char *> p{state.PeekAtNextChar()}) {
       if (**p == goal) {
         return {Success{}};
       } else if (**p == '\n') {
         break;
       } else {
         state.UncheckedAdvance();
       }
     }
     return std::nullopt;
   }
 };

 template <char left, char right> struct SkipPastNested {
   using resultType = Success;
   constexpr SkipPastNested() {}
   constexpr SkipPastNested(const SkipPastNested &) {}
   static std::optional<Success> Parse(ParseState &state) {
     int nesting{1};
     while (std::optional<const char *> p{state.GetNextChar()}) {
       if (**p == right) {
         if (!--nesting) {
           return {Success{}};
         }
       } else if (**p == left) {
         ++nesting;
       } else if (**p == '\n') {
         break;
       }
     }
     return std::nullopt;
   }
 };

 // A common idiom in the Fortran grammar is an optional item (usually
 // a nonempty comma-separated list) that, if present, must follow a comma
 // and precede a doubled colon.  When the item is absent, the comma must
 // not appear, and the doubled colons are optional.
 //   [[, xyz] ::]     is  optionalBeforeColons(xyz)
 //   [[, xyz]... ::]  is  optionalBeforeColons(nonemptyList(xyz))
 template <typename PA> inline constexpr auto optionalBeforeColons(const PA &p) {
   using resultType = std::optional<typename PA::resultType>;
   return "," >> construct<resultType>(p) / "::" ||
       ("::"_tok || !","_tok) >> pure<resultType>();
 }
 template <typename PA>
 inline constexpr auto optionalListBeforeColons(const PA &p) {
   using resultType = std::list<typename PA::resultType>;
   return "," >> nonemptyList(p) / "::" ||
       ("::"_tok || !","_tok) >> pure<resultType>();
 }

 // Skip over empty lines, leading spaces, and some compiler directives (viz.,
 // the ones that specify the source form) that might appear before the
 // next statement.  Skip over empty statements (bare semicolons) when
 // not in strict standard conformance mode.  Always succeeds.
 struct SkipStuffBeforeStatement {
   using resultType = Success;
   static std::optional<Success> Parse(ParseState &state) {
     if (UserState * ustate{state.userState()}) {
       if (ParsingLog * log{ustate->log()}) {
         // Save memory: vacate the parsing log before each statement unless
         // we're logging the whole parse for debugging.
         if (!ustate->instrumentedParse()) {
           log->clear();
         }
       }
     }
     while (std::optional<const char *> at{state.PeekAtNextChar()}) {
       if (**at == '\n' || **at == ' ') {
         state.UncheckedAdvance();
       } else if (**at == '!') {
         static const char fixed[] = "!dir$ fixed\n", free[] = "!dir$ free\n";
         static constexpr std::size_t fixedBytes{sizeof fixed - 1};
         static constexpr std::size_t freeBytes{sizeof free - 1};
         std::size_t remain{state.BytesRemaining()};
         if (remain >= fixedBytes && std::memcmp(*at, fixed, fixedBytes) == 0) {
           state.set_inFixedForm(true).UncheckedAdvance(fixedBytes);
         } else if (remain >= freeBytes &&
             std::memcmp(*at, free, freeBytes) == 0) {
           state.set_inFixedForm(false).UncheckedAdvance(freeBytes);
         } else {
           break;
         }
       } else if (**at == ';' &&
           state.IsNonstandardOk(
               LanguageFeature::EmptyStatement, "empty statement"_port_en_US)) {
         state.UncheckedAdvance();
       } else {
         break;
       }
     }
     return {Success{}};
   }
 };
 constexpr SkipStuffBeforeStatement skipStuffBeforeStatement;

 // R602 underscore -> _
 constexpr auto underscore{"_"_ch};

 // Characters besides letters and digits that may appear in names.
 // N.B. Don't accept an underscore if it is immediately followed by a
 // quotation mark, so that kindParam_"character literal" is parsed properly.
 // PGI and ifort accept '$' in identifiers, even as the initial character.
 // Cray and gfortran accept '$', but not as the first character.
 // Cray accepts '@' as well.
 constexpr auto otherIdChar{underscore / !"'\""_ch ||
     extension<LanguageFeature::PunctuationInNames>(
         "nonstandard usage: punctuation in name"_port_en_US, "$@"_ch)};

 constexpr auto logicalTRUE{
     (".TRUE."_tok ||
         extension<LanguageFeature::LogicalAbbreviations>(
             "nonstandard usage: .T. spelling of .TRUE."_port_en_US,
             ".T."_tok)) >>
     pure(true)};
 constexpr auto logicalFALSE{
     (".FALSE."_tok ||
         extension<LanguageFeature::LogicalAbbreviations>(
             "nonstandard usage: .F. spelling of .FALSE."_port_en_US,
             ".F."_tok)) >>
     pure(false)};

 // deprecated: Hollerith literals
 constexpr auto rawHollerithLiteral{
     deprecated<LanguageFeature::Hollerith>(HollerithLiteral{})};

 template <typename A> constexpr decltype(auto) verbatim(A x) {
   return sourced(construct<Verbatim>(x));
 }

 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_TOKEN_PARSERS_H_
	//===-- lib/Parser/token-parsers.h ------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#ifndef FORTRAN_PARSER_TOKEN_PARSERS_H_
	#define FORTRAN_PARSER_TOKEN_PARSERS_H_

	// These parsers are driven by the parsers of the Fortran grammar to consume
	// the prescanned character stream and recognize context-sensitive tokens.

	#include "basic-parsers.h"
	#include "type-parsers.h"
	#include "flang/Common/idioms.h"
	#include "flang/Parser/char-set.h"
	#include "flang/Parser/characters.h"
	#include "flang/Parser/instrumented-parser.h"
	#include "flang/Parser/provenance.h"
	#include <cstddef>
	#include <cstring>
	#include <functional>
	#include <limits>
	#include <list>
	#include <optional>
	#include <string>

	namespace Fortran::parser {

	// "xyz"_ch matches one instance of the characters x, y, or z without skipping
	// any spaces before or after. The parser returns the location of the character
	// on success.
	class AnyOfChars {
	public:
	using resultType = const char *;
	constexpr AnyOfChars(const AnyOfChars &) = default;
	constexpr AnyOfChars(SetOfChars set) : set_{set} {}
	std::optional<const char *> Parse(ParseState &state) const {
	if (std::optional<const char *> at{state.PeekAtNextChar()}) {
	if (set_.Has(**at)) {
	state.UncheckedAdvance();
	state.set_anyTokenMatched();
	return at;
	}
	}
	state.Say(MessageExpectedText{set_});
	return std::nullopt;
	}

	private:
	const SetOfChars set_;
	};

	constexpr AnyOfChars operator""_ch(const char str[], std::size_t n) {
	return AnyOfChars{SetOfChars(str, n)};
	}

	constexpr auto letter{"abcdefghijklmnopqrstuvwxyz"_ch};
	constexpr auto digit{"0123456789"_ch};

	// Skips over optional spaces. Always succeeds.
	struct Space {
	using resultType = Success;
	constexpr Space() {}
	static std::optional<Success> Parse(ParseState &state) {
	while (std::optional<const char *> p{state.PeekAtNextChar()}) {
	if (**p != ' ') {
	break;
	}
	state.UncheckedAdvance();
	}
	return {Success{}};
	}
	};
	constexpr Space space;

	// Skips a space that in free form requires a warning if it precedes a
	// character that could begin an identifier or keyword. Always succeeds.
	inline void MissingSpace(ParseState &state) {
	if (!state.inFixedForm()) {
	state.Nonstandard(
	LanguageFeature::OptionalFreeFormSpace, "missing space"_port_en_US);
	}
	}

	struct SpaceCheck {
	using resultType = Success;
	constexpr SpaceCheck() {}
	static std::optional<Success> Parse(ParseState &state) {
	if (std::optional<const char *> p{state.PeekAtNextChar()}) {
	char ch{**p};
	if (ch == ' ') {
	state.UncheckedAdvance();
	return space.Parse(state);
	}
	if (IsLegalInIdentifier(ch)) {
	MissingSpace(state);
	}
	}
	return {Success{}};
	}
	};
	constexpr SpaceCheck spaceCheck;

	// Matches a token string. Spaces in the token string denote where
	// spaces may appear in the source; they can be made mandatory for
	// some free form keyword sequences. Missing mandatory spaces in free
	// form elicit a warning; they are not necessary for recognition.
	// Spaces before and after the token are also skipped.
	//
	// Token strings appear in the grammar as C++ user-defined literals
	// like "BIND ( C )"_tok and "SYNC ALL"_sptok. The _tok suffix is implied
	// when a string literal appears before the sequencing operator >> or
	// after the sequencing operator /. The literal "..."_id parses a
	// token that cannot be a prefix of a longer identifier.
	template <bool MandatoryFreeFormSpace = false, bool MustBeComplete = false>
	class TokenStringMatch {
	public:
	using resultType = Success;
	constexpr TokenStringMatch(const TokenStringMatch &) = default;
	constexpr TokenStringMatch(const char *str, std::size_t n)
	: str_{str}, bytes_{n} {}
	explicit constexpr TokenStringMatch(const char *str) : str_{str} {}
	std::optional<Success> Parse(ParseState &state) const {
	space.Parse(state);
	const char *start{state.GetLocation()};
	const char *p{str_};
	std::optional<const char *> at; // initially empty
	for (std::size_t j{0}; j < bytes_ && *p != '\0'; ++j, ++p) {
	bool spaceSkipping{*p == ' '};
	if (spaceSkipping) {
	if (j + 1 == bytes_ \|\| p[1] == ' ' \|\| p[1] == '\0') {
	continue; // redundant; ignore
	}
	}
	if (!at) {
	at = nextCh.Parse(state);
	if (!at) {
	return std::nullopt;
	}
	}
	if (spaceSkipping) {
	if (**at == ' ') {
	at = nextCh.Parse(state);
	if (!at) {
	return std::nullopt;
	}
	} else if constexpr (MandatoryFreeFormSpace) {
	MissingSpace(state);
	}
	// 'at' remains full for next iteration
	} else if (*at == ToLowerCaseLetter(p)) {
	at.reset();
	} else {
	state.Say(start, MessageExpectedText{str_, bytes_});
	return std::nullopt;
	}
	}
	if constexpr (MustBeComplete) {
	if (auto after{state.PeekAtNextChar()}) {
	if (IsLegalInIdentifier(**after)) {
	state.Say(start, MessageExpectedText{str_, bytes_});
	return std::nullopt;
	}
	}
	}
	state.set_anyTokenMatched();
	if (IsLegalInIdentifier(p[-1])) {
	return spaceCheck.Parse(state);
	} else {
	return space.Parse(state);
	}
	}

	private:
	const char *const str_;
	const std::size_t bytes_{std::string::npos};
	};

	constexpr TokenStringMatch<> operator""_tok(const char str[], std::size_t n) {
	return {str, n};
	}

	constexpr TokenStringMatch<true> operator""_sptok(
	const char str[], std::size_t n) {
	return {str, n};
	}

	constexpr TokenStringMatch<false, true> operator""_id(
	const char str[], std::size_t n) {
	return {str, n};
	}

	template <class PA>
	inline constexpr std::enable_if_t<std::is_class_v<PA>,
	SequenceParser<TokenStringMatch<>, PA>>
	operator>>(const char *str, const PA &p) {
	return SequenceParser<TokenStringMatch<>, PA>{TokenStringMatch<>{str}, p};
	}

	template <class PA>
	inline constexpr std::enable_if_t<std::is_class_v<PA>,
	FollowParser<PA, TokenStringMatch<>>>
	operator/(const PA &p, const char *str) {
	return FollowParser<PA, TokenStringMatch<>>{p, TokenStringMatch<>{str}};
	}

	template <class PA> inline constexpr auto parenthesized(const PA &p) {
	return "(" >> p / ")";
	}

	template <class PA> inline constexpr auto bracketed(const PA &p) {
	return "[" >> p / "]";
	}

	// Quoted character literal constants.
	struct CharLiteralChar {
	using resultType = std::pair<char, bool /* was escaped */>;
	static std::optional<resultType> Parse(ParseState &state) {
	auto at{state.GetLocation()};
	if (std::optional<const char *> cp{nextCh.Parse(state)}) {
	char ch{**cp};
	if (ch == '\n') {
	state.Say(CharBlock{at, state.GetLocation()},
	"Unclosed character constant"_err_en_US);
	return std::nullopt;
	}
	if (ch == '\\') {
	// Most escape sequences in character literals are processed later,
	// but we have to look for quotes here so that doubled quotes work.
	if (std::optional<const char *> next{state.PeekAtNextChar()}) {
	char escaped{**next};
	if (escaped == '\'' \|\| escaped == '"' \|\| escaped == '\\') {
	state.UncheckedAdvance();
	return std::make_pair(escaped, true);
	}
	}
	}
	return std::make_pair(ch, false);
	}
	return std::nullopt;
	}
	};

	template <char quote> struct CharLiteral {
	using resultType = std::string;
	static std::optional<std::string> Parse(ParseState &state) {
	std::string str;
	static constexpr auto nextch{attempt(CharLiteralChar{})};
	while (auto ch{nextch.Parse(state)}) {
	if (ch->second) {
	str += '\\';
	} else if (ch->first == quote) {
	static constexpr auto doubled{attempt(AnyOfChars{SetOfChars{quote}})};
	if (!doubled.Parse(state)) {
	return str;
	}
	}
	str += ch->first;
	}
	return std::nullopt;
	}
	};

	// Parse "BOZ" binary literal quoted constants.
	// As extensions, support X as an alternate hexadecimal marker, and allow
	// BOZX markers to appear as suffixes.
	struct BOZLiteral {
	using resultType = std::string;
	static std::optional<resultType> Parse(ParseState &state) {
	char base{'\0'};
	auto baseChar{[&base](char ch) -> bool {
	switch (ch) {
	case 'b':
	case 'o':
	case 'z':
	base = ch;
	return true;
	case 'x':
	base = 'z';
	return true;
	default:
	return false;
	}
	}};

	space.Parse(state);
	const char *start{state.GetLocation()};
	std::optional<const char *> at{nextCh.Parse(state)};
	if (!at) {
	return std::nullopt;
	}
	if (**at == 'x' &&
	!state.IsNonstandardOk(LanguageFeature::BOZExtensions,
	"nonstandard BOZ literal"_port_en_US)) {
	return std::nullopt;
	}
	if (baseChar(**at)) {
	at = nextCh.Parse(state);
	if (!at) {
	return std::nullopt;
	}
	}

	char quote = **at;
	if (quote != '\'' && quote != '"') {
	return std::nullopt;
	}

	std::string content;
	while (true) {
	at = nextCh.Parse(state);
	if (!at) {
	return std::nullopt;
	}
	if (**at == quote) {
	break;
	}
	if (**at == ' ') {
	continue;
	}
	if (!IsHexadecimalDigit(**at)) {
	return std::nullopt;
	}
	content += ToLowerCaseLetter(**at);
	}

	if (!base) {
	// extension: base allowed to appear as suffix, too
	if (!(at = nextCh.Parse(state)) \|\| !baseChar(**at) \|\|
	!state.IsNonstandardOk(LanguageFeature::BOZExtensions,
	"nonstandard BOZ literal"_port_en_US)) {
	return std::nullopt;
	}
	spaceCheck.Parse(state);
	}

	if (content.empty()) {
	state.Say(start, "no digit in BOZ literal"_err_en_US);
	return std::nullopt;
	}
	return {std::string{base} + '"' + content + '"'};
	}
	};

	// R711 digit-string -> digit [digit]...
	// N.B. not a token -- no space is skipped
	struct DigitString {
	using resultType = CharBlock;
	static std::optional<resultType> Parse(ParseState &state) {
	if (std::optional<const char *> ch1{state.PeekAtNextChar()}) {
	if (IsDecimalDigit(**ch1)) {
	state.UncheckedAdvance();
	while (std::optional<const char *> p{state.PeekAtNextChar()}) {
	if (!IsDecimalDigit(**p)) {
	break;
	}
	state.UncheckedAdvance();
	}
	return CharBlock{*ch1, state.GetLocation()};
	}
	}
	return std::nullopt;
	}
	};
	constexpr DigitString digitString;

	struct SignedIntLiteralConstantWithoutKind {
	using resultType = CharBlock;
	static std::optional<resultType> Parse(ParseState &state) {
	resultType result{state.GetLocation()};
	static constexpr auto sign{maybe("+-"_ch / space)};
	if (sign.Parse(state)) {
	if (auto digits{digitString.Parse(state)}) {
	result.ExtendToCover(*digits);
	return result;
	}
	}
	return std::nullopt;
	}
	};

	struct DigitString64 {
	using resultType = std::uint64_t;
	static std::optional<std::uint64_t> Parse(ParseState &state) {
	std::optional<const char *> firstDigit{digit.Parse(state)};
	if (!firstDigit) {
	return std::nullopt;
	}
	std::uint64_t value = **firstDigit - '0';
	bool overflow{false};
	static constexpr auto getDigit{attempt(digit)};
	while (auto nextDigit{getDigit.Parse(state)}) {
	if (value > std::numeric_limits<std::uint64_t>::max() / 10) {
	overflow = true;
	}
	value *= 10;
	int digitValue = **nextDigit - '0';
	if (value > std::numeric_limits<std::uint64_t>::max() - digitValue) {
	overflow = true;
	}
	value += digitValue;
	}
	if (overflow) {
	state.Say(*firstDigit, "overflow in decimal literal"_err_en_US);
	}
	return {value};
	}
	};
	constexpr DigitString64 digitString64;

	// R707 signed-int-literal-constant -> [sign] int-literal-constant
	// N.B. Spaces are consumed before and after the sign, since the sign
	// and the int-literal-constant are distinct tokens. Does not
	// handle a trailing kind parameter.
	static std::optional<std::int64_t> SignedInteger(
	const std::optional<std::uint64_t> &x, Location at, bool negate,
	ParseState &state) {
	if (!x) {
	return std::nullopt;
	}
	std::uint64_t limit{std::numeric_limits<std::int64_t>::max()};
	if (negate) {
	limit = -(limit + 1);
	}
	if (*x > limit) {
	state.Say(at, "overflow in signed decimal literal"_err_en_US);
	}
	std::int64_t value = *x;
	return std::make_optional<std::int64_t>(negate ? -value : value);
	}

	// R710 signed-digit-string -> [sign] digit-string
	// N.B. Not a complete token -- no space is skipped.
	// Used only in the exponent parts of real literal constants.
	struct SignedDigitString {
	using resultType = std::int64_t;
	static std::optional<std::int64_t> Parse(ParseState &state) {
	std::optional<const char *> sign{state.PeekAtNextChar()};
	if (!sign) {
	return std::nullopt;
	}
	bool negate{**sign == '-'};
	if (negate \|\| **sign == '+') {
	state.UncheckedAdvance();
	}
	return SignedInteger(digitString64.Parse(state), *sign, negate, state);
	}
	};

	// Variants of the above for use in FORMAT specifications, where spaces
	// must be ignored.
	struct DigitStringIgnoreSpaces {
	using resultType = std::uint64_t;
	static std::optional<std::uint64_t> Parse(ParseState &state) {
	static constexpr auto getFirstDigit{space >> digit};
	std::optional<const char *> firstDigit{getFirstDigit.Parse(state)};
	if (!firstDigit) {
	return std::nullopt;
	}
	std::uint64_t value = **firstDigit - '0';
	bool overflow{false};
	static constexpr auto getDigit{space >> attempt(digit)};
	while (auto nextDigit{getDigit.Parse(state)}) {
	if (value > std::numeric_limits<std::uint64_t>::max() / 10) {
	overflow = true;
	}
	value *= 10;
	int digitValue = **nextDigit - '0';
	if (value > std::numeric_limits<std::uint64_t>::max() - digitValue) {
	overflow = true;
	}
	value += digitValue;
	}
	if (overflow) {
	state.Say(*firstDigit, "overflow in decimal literal"_err_en_US);
	}
	return value;
	}
	};

	struct PositiveDigitStringIgnoreSpaces {
	using resultType = std::int64_t;
	static std::optional<std::int64_t> Parse(ParseState &state) {
	Location at{state.GetLocation()};
	return SignedInteger(
	DigitStringIgnoreSpaces{}.Parse(state), at, false /positive/, state);
	}
	};

	struct SignedDigitStringIgnoreSpaces {
	using resultType = std::int64_t;
	static std::optional<std::int64_t> Parse(ParseState &state) {
	static constexpr auto getSign{space >> attempt("+-"_ch)};
	bool negate{false};
	if (std::optional<const char *> sign{getSign.Parse(state)}) {
	negate = **sign == '-';
	}
	Location at{state.GetLocation()};
	return SignedInteger(
	DigitStringIgnoreSpaces{}.Parse(state), at, negate, state);
	}
	};

	// Legacy feature: Hollerith literal constants
	struct HollerithLiteral {
	using resultType = std::string;
	static std::optional<std::string> Parse(ParseState &state) {
	space.Parse(state);
	const char *start{state.GetLocation()};
	std::optional<std::uint64_t> charCount{
	DigitStringIgnoreSpaces{}.Parse(state)};
	if (!charCount \|\| *charCount < 1) {
	return std::nullopt;
	}
	static constexpr auto letterH{"h"_ch};
	std::optional<const char *> h{letterH.Parse(state)};
	if (!h) {
	return std::nullopt;
	}
	std::string content;
	for (auto j{*charCount}; j-- > 0;) {
	int chBytes{UTF_8CharacterBytes(state.GetLocation())};
	for (int bytes{chBytes}; bytes > 0; --bytes) {
	if (std::optional<const char *> at{nextCh.Parse(state)}) {
	if (chBytes == 1 && !IsPrintable(**at)) {
	state.Say(start, "Bad character in Hollerith"_err_en_US);
	return std::nullopt;
	}
	content += **at;
	} else {
	state.Say(start, "Insufficient characters in Hollerith"_err_en_US);
	return std::nullopt;
	}
	}
	}
	return content;
	}
	};

	struct ConsumedAllInputParser {
	using resultType = Success;
	constexpr ConsumedAllInputParser() {}
	static inline std::optional<Success> Parse(ParseState &state) {
	if (state.IsAtEnd()) {
	return {Success{}};
	}
	return std::nullopt;
	}
	};
	constexpr ConsumedAllInputParser consumedAllInput;

	template <char goal> struct SkipPast {
	using resultType = Success;
	constexpr SkipPast() {}
	constexpr SkipPast(const SkipPast &) {}
	static std::optional<Success> Parse(ParseState &state) {
	while (std::optional<const char *> p{state.GetNextChar()}) {
	if (**p == goal) {
	return {Success{}};
	} else if (**p == '\n') {
	break;
	}
	}
	return std::nullopt;
	}
	};

	template <char goal> struct SkipTo {
	using resultType = Success;
	constexpr SkipTo() {}
	constexpr SkipTo(const SkipTo &) {}
	static std::optional<Success> Parse(ParseState &state) {
	while (std::optional<const char *> p{state.PeekAtNextChar()}) {
	if (**p == goal) {
	return {Success{}};
	} else if (**p == '\n') {
	break;
	} else {
	state.UncheckedAdvance();
	}
	}
	return std::nullopt;
	}
	};

	template <char left, char right> struct SkipPastNested {
	using resultType = Success;
	constexpr SkipPastNested() {}
	constexpr SkipPastNested(const SkipPastNested &) {}
	static std::optional<Success> Parse(ParseState &state) {
	int nesting{1};
	while (std::optional<const char *> p{state.GetNextChar()}) {
	if (**p == right) {
	if (!--nesting) {
	return {Success{}};
	}
	} else if (**p == left) {
	++nesting;
	} else if (**p == '\n') {
	break;
	}
	}
	return std::nullopt;
	}
	};

	// A common idiom in the Fortran grammar is an optional item (usually
	// a nonempty comma-separated list) that, if present, must follow a comma
	// and precede a doubled colon. When the item is absent, the comma must
	// not appear, and the doubled colons are optional.
	// [[, xyz] ::] is optionalBeforeColons(xyz)
	// [[, xyz]... ::] is optionalBeforeColons(nonemptyList(xyz))
	template <typename PA> inline constexpr auto optionalBeforeColons(const PA &p) {
	using resultType = std::optional<typename PA::resultType>;
	return "," >> construct<resultType>(p) / "::" \|\|
	("::"_tok \|\| !","_tok) >> pure<resultType>();
	}
	template <typename PA>
	inline constexpr auto optionalListBeforeColons(const PA &p) {
	using resultType = std::list<typename PA::resultType>;
	return "," >> nonemptyList(p) / "::" \|\|
	("::"_tok \|\| !","_tok) >> pure<resultType>();
	}

	// Skip over empty lines, leading spaces, and some compiler directives (viz.,
	// the ones that specify the source form) that might appear before the
	// next statement. Skip over empty statements (bare semicolons) when
	// not in strict standard conformance mode. Always succeeds.
	struct SkipStuffBeforeStatement {
	using resultType = Success;
	static std::optional<Success> Parse(ParseState &state) {
	if (UserState * ustate{state.userState()}) {
	if (ParsingLog * log{ustate->log()}) {
	// Save memory: vacate the parsing log before each statement unless
	// we're logging the whole parse for debugging.
	if (!ustate->instrumentedParse()) {
	log->clear();
	}
	}
	}
	while (std::optional<const char *> at{state.PeekAtNextChar()}) {
	if (at == '\n' \|\| at == ' ') {
	state.UncheckedAdvance();
	} else if (**at == '!') {
	static const char fixed[] = "!dir$ fixed\n", free[] = "!dir$ free\n";
	static constexpr std::size_t fixedBytes{sizeof fixed - 1};
	static constexpr std::size_t freeBytes{sizeof free - 1};
	std::size_t remain{state.BytesRemaining()};
	if (remain >= fixedBytes && std::memcmp(*at, fixed, fixedBytes) == 0) {
	state.set_inFixedForm(true).UncheckedAdvance(fixedBytes);
	} else if (remain >= freeBytes &&
	std::memcmp(*at, free, freeBytes) == 0) {
	state.set_inFixedForm(false).UncheckedAdvance(freeBytes);
	} else {
	break;
	}
	} else if (**at == ';' &&
	state.IsNonstandardOk(
	LanguageFeature::EmptyStatement, "empty statement"_port_en_US)) {
	state.UncheckedAdvance();
	} else {
	break;
	}
	}
	return {Success{}};
	}
	};
	constexpr SkipStuffBeforeStatement skipStuffBeforeStatement;

	// R602 underscore -> _
	constexpr auto underscore{"_"_ch};

	// Characters besides letters and digits that may appear in names.
	// N.B. Don't accept an underscore if it is immediately followed by a
	// quotation mark, so that kindParam_"character literal" is parsed properly.
	// PGI and ifort accept '$' in identifiers, even as the initial character.
	// Cray and gfortran accept '$', but not as the first character.
	// Cray accepts '@' as well.
	constexpr auto otherIdChar{underscore / !"'\""_ch \|\|
	extension<LanguageFeature::PunctuationInNames>(
	"nonstandard usage: punctuation in name"_port_en_US, "$@"_ch)};

	constexpr auto logicalTRUE{
	(".TRUE."_tok \|\|
	extension<LanguageFeature::LogicalAbbreviations>(
	"nonstandard usage: .T. spelling of .TRUE."_port_en_US,
	".T."_tok)) >>
	pure(true)};
	constexpr auto logicalFALSE{
	(".FALSE."_tok \|\|
	extension<LanguageFeature::LogicalAbbreviations>(
	"nonstandard usage: .F. spelling of .FALSE."_port_en_US,
	".F."_tok)) >>
	pure(false)};

	// deprecated: Hollerith literals
	constexpr auto rawHollerithLiteral{
	deprecated<LanguageFeature::Hollerith>(HollerithLiteral{})};

	template <typename A> constexpr decltype(auto) verbatim(A x) {
	return sourced(construct<Verbatim>(x));
	}

	} // namespace Fortran::parser
	#endif // FORTRAN_PARSER_TOKEN_PARSERS_H_