| //===--- BreakableToken.h - Format C++ code ---------------------*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| /// |
| /// \file |
| /// Declares BreakableToken, BreakableStringLiteral, BreakableComment, |
| /// BreakableBlockComment and BreakableLineCommentSection classes, that contain |
| /// token type-specific logic to break long lines in tokens and reflow content |
| /// between tokens. |
| /// |
| //===----------------------------------------------------------------------===// |
| |
| #ifndef LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H |
| #define LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H |
| |
| #include "Encoding.h" |
| #include "TokenAnnotator.h" |
| #include "WhitespaceManager.h" |
| #include "llvm/ADT/StringSet.h" |
| #include "llvm/Support/Regex.h" |
| #include <utility> |
| |
| namespace clang { |
| namespace format { |
| |
| /// Checks if \p Token switches formatting, like /* clang-format off */. |
| /// \p Token must be a comment. |
| bool switchesFormatting(const FormatToken &Token); |
| |
| struct FormatStyle; |
| |
| /// Base class for tokens / ranges of tokens that can allow breaking |
| /// within the tokens - for example, to avoid whitespace beyond the column |
| /// limit, or to reflow text. |
| /// |
| /// Generally, a breakable token consists of logical lines, addressed by a line |
| /// index. For example, in a sequence of line comments, each line comment is its |
| /// own logical line; similarly, for a block comment, each line in the block |
| /// comment is on its own logical line. |
| /// |
| /// There are two methods to compute the layout of the token: |
| /// - getRangeLength measures the number of columns needed for a range of text |
| /// within a logical line, and |
| /// - getContentStartColumn returns the start column at which we want the |
| /// content of a logical line to start (potentially after introducing a line |
| /// break). |
| /// |
| /// The mechanism to adapt the layout of the breakable token is organised |
| /// around the concept of a \c Split, which is a whitespace range that signifies |
| /// a position of the content of a token where a reformatting might be done. |
| /// |
| /// Operating with splits is divided into two operations: |
| /// - getSplit, for finding a split starting at a position, |
| /// - insertBreak, for executing the split using a whitespace manager. |
| /// |
| /// There is a pair of operations that are used to compress a long whitespace |
| /// range with a single space if that will bring the line length under the |
| /// column limit: |
| /// - getLineLengthAfterCompression, for calculating the size in columns of the |
| /// line after a whitespace range has been compressed, and |
| /// - compressWhitespace, for executing the whitespace compression using a |
| /// whitespace manager; note that the compressed whitespace may be in the |
| /// middle of the original line and of the reformatted line. |
| /// |
| /// For tokens where the whitespace before each line needs to be also |
| /// reformatted, for example for tokens supporting reflow, there are analogous |
| /// operations that might be executed before the main line breaking occurs: |
| /// - getReflowSplit, for finding a split such that the content preceding it |
| /// needs to be specially reflown, |
| /// - reflow, for executing the split using a whitespace manager, |
| /// - introducesBreakBefore, for checking if reformatting the beginning |
| /// of the content introduces a line break before it, |
| /// - adaptStartOfLine, for executing the reflow using a whitespace |
| /// manager. |
| /// |
| /// For tokens that require the whitespace after the last line to be |
| /// reformatted, for example in multiline jsdoc comments that require the |
| /// trailing '*/' to be on a line of itself, there are analogous operations |
| /// that might be executed after the last line has been reformatted: |
| /// - getSplitAfterLastLine, for finding a split after the last line that needs |
| /// to be reflown, |
| /// - replaceWhitespaceAfterLastLine, for executing the reflow using a |
| /// whitespace manager. |
| /// |
| class BreakableToken { |
| public: |
| /// Contains starting character index and length of split. |
| typedef std::pair<StringRef::size_type, unsigned> Split; |
| |
| virtual ~BreakableToken() {} |
| |
| /// Returns the number of lines in this token in the original code. |
| virtual unsigned getLineCount() const = 0; |
| |
| /// Returns the number of columns required to format the text in the |
| /// byte range [\p Offset, \p Offset \c + \p Length). |
| /// |
| /// \p Offset is the byte offset from the start of the content of the line |
| /// at \p LineIndex. |
| /// |
| /// \p StartColumn is the column at which the text starts in the formatted |
| /// file, needed to compute tab stops correctly. |
| virtual unsigned getRangeLength(unsigned LineIndex, unsigned Offset, |
| StringRef::size_type Length, |
| unsigned StartColumn) const = 0; |
| |
| /// Returns the number of columns required to format the text following |
| /// the byte \p Offset in the line \p LineIndex, including potentially |
| /// unbreakable sequences of tokens following after the end of the token. |
| /// |
| /// \p Offset is the byte offset from the start of the content of the line |
| /// at \p LineIndex. |
| /// |
| /// \p StartColumn is the column at which the text starts in the formatted |
| /// file, needed to compute tab stops correctly. |
| /// |
| /// For breakable tokens that never use extra space at the end of a line, this |
| /// is equivalent to getRangeLength with a Length of StringRef::npos. |
| virtual unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, |
| unsigned StartColumn) const { |
| return getRangeLength(LineIndex, Offset, StringRef::npos, StartColumn); |
| } |
| |
| /// Returns the column at which content in line \p LineIndex starts, |
| /// assuming no reflow. |
| /// |
| /// If \p Break is true, returns the column at which the line should start |
| /// after the line break. |
| /// If \p Break is false, returns the column at which the line itself will |
| /// start. |
| virtual unsigned getContentStartColumn(unsigned LineIndex, |
| bool Break) const = 0; |
| |
| /// Returns additional content indent required for the second line after the |
| /// content at line \p LineIndex is broken. |
| /// |
| // (Next lines do not start with `///` since otherwise -Wdocumentation picks |
| // up the example annotations and generates warnings for them) |
| // For example, Javadoc @param annotations require and indent of 4 spaces and |
| // in this example getContentIndex(1) returns 4. |
| // /** |
| // * @param loooooooooooooong line |
| // * continuation |
| // */ |
| virtual unsigned getContentIndent(unsigned LineIndex) const { return 0; } |
| |
| /// Returns a range (offset, length) at which to break the line at |
| /// \p LineIndex, if previously broken at \p TailOffset. If possible, do not |
| /// violate \p ColumnLimit, assuming the text starting at \p TailOffset in |
| /// the token is formatted starting at ContentStartColumn in the reformatted |
| /// file. |
| virtual Split getSplit(unsigned LineIndex, unsigned TailOffset, |
| unsigned ColumnLimit, unsigned ContentStartColumn, |
| llvm::Regex &CommentPragmasRegex) const = 0; |
| |
| /// Emits the previously retrieved \p Split via \p Whitespaces. |
| virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, |
| unsigned ContentIndent, |
| WhitespaceManager &Whitespaces) const = 0; |
| |
| /// Returns the number of columns needed to format |
| /// \p RemainingTokenColumns, assuming that Split is within the range measured |
| /// by \p RemainingTokenColumns, and that the whitespace in Split is reduced |
| /// to a single space. |
| unsigned getLengthAfterCompression(unsigned RemainingTokenColumns, |
| Split Split) const; |
| |
| /// Replaces the whitespace range described by \p Split with a single |
| /// space. |
| virtual void compressWhitespace(unsigned LineIndex, unsigned TailOffset, |
| Split Split, |
| WhitespaceManager &Whitespaces) const = 0; |
| |
| /// Returns whether the token supports reflowing text. |
| virtual bool supportsReflow() const { return false; } |
| |
| /// Returns a whitespace range (offset, length) of the content at \p |
| /// LineIndex such that the content of that line is reflown to the end of the |
| /// previous one. |
| /// |
| /// Returning (StringRef::npos, 0) indicates reflowing is not possible. |
| /// |
| /// The range will include any whitespace preceding the specified line's |
| /// content. |
| /// |
| /// If the split is not contained within one token, for example when reflowing |
| /// line comments, returns (0, <length>). |
| virtual Split getReflowSplit(unsigned LineIndex, |
| llvm::Regex &CommentPragmasRegex) const { |
| return Split(StringRef::npos, 0); |
| } |
| |
| /// Reflows the current line into the end of the previous one. |
| virtual void reflow(unsigned LineIndex, |
| WhitespaceManager &Whitespaces) const {} |
| |
| /// Returns whether there will be a line break at the start of the |
| /// token. |
| virtual bool introducesBreakBeforeToken() const { return false; } |
| |
| /// Replaces the whitespace between \p LineIndex-1 and \p LineIndex. |
| virtual void adaptStartOfLine(unsigned LineIndex, |
| WhitespaceManager &Whitespaces) const {} |
| |
| /// Returns a whitespace range (offset, length) of the content at |
| /// the last line that needs to be reformatted after the last line has been |
| /// reformatted. |
| /// |
| /// A result having offset == StringRef::npos means that no reformat is |
| /// necessary. |
| virtual Split getSplitAfterLastLine(unsigned TailOffset) const { |
| return Split(StringRef::npos, 0); |
| } |
| |
| /// Replaces the whitespace from \p SplitAfterLastLine on the last line |
| /// after the last line has been formatted by performing a reformatting. |
| void replaceWhitespaceAfterLastLine(unsigned TailOffset, |
| Split SplitAfterLastLine, |
| WhitespaceManager &Whitespaces) const { |
| insertBreak(getLineCount() - 1, TailOffset, SplitAfterLastLine, |
| /*ContentIndent=*/0, Whitespaces); |
| } |
| |
| /// Updates the next token of \p State to the next token after this |
| /// one. This can be used when this token manages a set of underlying tokens |
| /// as a unit and is responsible for the formatting of the them. |
| virtual void updateNextToken(LineState &State) const {} |
| |
| protected: |
| BreakableToken(const FormatToken &Tok, bool InPPDirective, |
| encoding::Encoding Encoding, const FormatStyle &Style) |
| : Tok(Tok), InPPDirective(InPPDirective), Encoding(Encoding), |
| Style(Style) {} |
| |
| const FormatToken &Tok; |
| const bool InPPDirective; |
| const encoding::Encoding Encoding; |
| const FormatStyle &Style; |
| }; |
| |
| class BreakableStringLiteral : public BreakableToken { |
| public: |
| /// Creates a breakable token for a single line string literal. |
| /// |
| /// \p StartColumn specifies the column in which the token will start |
| /// after formatting. |
| BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn, |
| StringRef Prefix, StringRef Postfix, |
| unsigned UnbreakableTailLength, bool InPPDirective, |
| encoding::Encoding Encoding, const FormatStyle &Style); |
| |
| Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, |
| unsigned ContentStartColumn, |
| llvm::Regex &CommentPragmasRegex) const override; |
| void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, |
| unsigned ContentIndent, |
| WhitespaceManager &Whitespaces) const override; |
| void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split, |
| WhitespaceManager &Whitespaces) const override {} |
| unsigned getLineCount() const override; |
| unsigned getRangeLength(unsigned LineIndex, unsigned Offset, |
| StringRef::size_type Length, |
| unsigned StartColumn) const override; |
| unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, |
| unsigned StartColumn) const override; |
| unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; |
| |
| protected: |
| // The column in which the token starts. |
| unsigned StartColumn; |
| // The prefix a line needs after a break in the token. |
| StringRef Prefix; |
| // The postfix a line needs before introducing a break. |
| StringRef Postfix; |
| // The token text excluding the prefix and postfix. |
| StringRef Line; |
| // Length of the sequence of tokens after this string literal that cannot |
| // contain line breaks. |
| unsigned UnbreakableTailLength; |
| }; |
| |
| class BreakableComment : public BreakableToken { |
| protected: |
| /// Creates a breakable token for a comment. |
| /// |
| /// \p StartColumn specifies the column in which the comment will start after |
| /// formatting. |
| BreakableComment(const FormatToken &Token, unsigned StartColumn, |
| bool InPPDirective, encoding::Encoding Encoding, |
| const FormatStyle &Style); |
| |
| public: |
| bool supportsReflow() const override { return true; } |
| unsigned getLineCount() const override; |
| Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, |
| unsigned ContentStartColumn, |
| llvm::Regex &CommentPragmasRegex) const override; |
| void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split, |
| WhitespaceManager &Whitespaces) const override; |
| |
| protected: |
| // Returns the token containing the line at LineIndex. |
| const FormatToken &tokenAt(unsigned LineIndex) const; |
| |
| // Checks if the content of line LineIndex may be reflown with the previous |
| // line. |
| virtual bool mayReflow(unsigned LineIndex, |
| llvm::Regex &CommentPragmasRegex) const = 0; |
| |
| // Contains the original text of the lines of the block comment. |
| // |
| // In case of a block comments, excludes the leading /* in the first line and |
| // trailing */ in the last line. In case of line comments, excludes the |
| // leading // and spaces. |
| SmallVector<StringRef, 16> Lines; |
| |
| // Contains the text of the lines excluding all leading and trailing |
| // whitespace between the lines. Note that the decoration (if present) is also |
| // not considered part of the text. |
| SmallVector<StringRef, 16> Content; |
| |
| // Tokens[i] contains a reference to the token containing Lines[i] if the |
| // whitespace range before that token is managed by this block. |
| // Otherwise, Tokens[i] is a null pointer. |
| SmallVector<FormatToken *, 16> Tokens; |
| |
| // ContentColumn[i] is the target column at which Content[i] should be. |
| // Note that this excludes a leading "* " or "*" in case of block comments |
| // where all lines have a "*" prefix, or the leading "// " or "//" in case of |
| // line comments. |
| // |
| // In block comments, the first line's target column is always positive. The |
| // remaining lines' target columns are relative to the first line to allow |
| // correct indentation of comments in \c WhitespaceManager. Thus they can be |
| // negative as well (in case the first line needs to be unindented more than |
| // there's actual whitespace in another line). |
| SmallVector<int, 16> ContentColumn; |
| |
| // The intended start column of the first line of text from this section. |
| unsigned StartColumn; |
| |
| // The prefix to use in front a line that has been reflown up. |
| // For example, when reflowing the second line after the first here: |
| // // comment 1 |
| // // comment 2 |
| // we expect: |
| // // comment 1 comment 2 |
| // and not: |
| // // comment 1comment 2 |
| StringRef ReflowPrefix = " "; |
| }; |
| |
| class BreakableBlockComment : public BreakableComment { |
| public: |
| BreakableBlockComment(const FormatToken &Token, unsigned StartColumn, |
| unsigned OriginalStartColumn, bool FirstInLine, |
| bool InPPDirective, encoding::Encoding Encoding, |
| const FormatStyle &Style, bool UseCRLF); |
| |
| Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, |
| unsigned ContentStartColumn, |
| llvm::Regex &CommentPragmasRegex) const override; |
| unsigned getRangeLength(unsigned LineIndex, unsigned Offset, |
| StringRef::size_type Length, |
| unsigned StartColumn) const override; |
| unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, |
| unsigned StartColumn) const override; |
| unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; |
| unsigned getContentIndent(unsigned LineIndex) const override; |
| void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, |
| unsigned ContentIndent, |
| WhitespaceManager &Whitespaces) const override; |
| Split getReflowSplit(unsigned LineIndex, |
| llvm::Regex &CommentPragmasRegex) const override; |
| void reflow(unsigned LineIndex, |
| WhitespaceManager &Whitespaces) const override; |
| bool introducesBreakBeforeToken() const override; |
| void adaptStartOfLine(unsigned LineIndex, |
| WhitespaceManager &Whitespaces) const override; |
| Split getSplitAfterLastLine(unsigned TailOffset) const override; |
| |
| bool mayReflow(unsigned LineIndex, |
| llvm::Regex &CommentPragmasRegex) const override; |
| |
| // Contains Javadoc annotations that require additional indent when continued |
| // on multiple lines. |
| static const llvm::StringSet<> ContentIndentingJavadocAnnotations; |
| |
| private: |
| // Rearranges the whitespace between Lines[LineIndex-1] and Lines[LineIndex]. |
| // |
| // Updates Content[LineIndex-1] and Content[LineIndex] by stripping off |
| // leading and trailing whitespace. |
| // |
| // Sets ContentColumn to the intended column in which the text at |
| // Lines[LineIndex] starts (note that the decoration, if present, is not |
| // considered part of the text). |
| void adjustWhitespace(unsigned LineIndex, int IndentDelta); |
| |
| // The column at which the text of a broken line should start. |
| // Note that an optional decoration would go before that column. |
| // IndentAtLineBreak is a uniform position for all lines in a block comment, |
| // regardless of their relative position. |
| // FIXME: Revisit the decision to do this; the main reason was to support |
| // patterns like |
| // /**************//** |
| // * Comment |
| // We could also support such patterns by special casing the first line |
| // instead. |
| unsigned IndentAtLineBreak; |
| |
| // This is to distinguish between the case when the last line was empty and |
| // the case when it started with a decoration ("*" or "* "). |
| bool LastLineNeedsDecoration; |
| |
| // Either "* " if all lines begin with a "*", or empty. |
| StringRef Decoration; |
| |
| // If this block comment has decorations, this is the column of the start of |
| // the decorations. |
| unsigned DecorationColumn; |
| |
| // If true, make sure that the opening '/**' and the closing '*/' ends on a |
| // line of itself. Styles like jsdoc require this for multiline comments. |
| bool DelimitersOnNewline; |
| |
| // Length of the sequence of tokens after this string literal that cannot |
| // contain line breaks. |
| unsigned UnbreakableTailLength; |
| }; |
| |
| class BreakableLineCommentSection : public BreakableComment { |
| public: |
| BreakableLineCommentSection(const FormatToken &Token, unsigned StartColumn, |
| unsigned OriginalStartColumn, bool FirstInLine, |
| bool InPPDirective, encoding::Encoding Encoding, |
| const FormatStyle &Style); |
| |
| unsigned getRangeLength(unsigned LineIndex, unsigned Offset, |
| StringRef::size_type Length, |
| unsigned StartColumn) const override; |
| unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; |
| void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, |
| unsigned ContentIndent, |
| WhitespaceManager &Whitespaces) const override; |
| Split getReflowSplit(unsigned LineIndex, |
| llvm::Regex &CommentPragmasRegex) const override; |
| void reflow(unsigned LineIndex, |
| WhitespaceManager &Whitespaces) const override; |
| void adaptStartOfLine(unsigned LineIndex, |
| WhitespaceManager &Whitespaces) const override; |
| void updateNextToken(LineState &State) const override; |
| bool mayReflow(unsigned LineIndex, |
| llvm::Regex &CommentPragmasRegex) const override; |
| |
| private: |
| // OriginalPrefix[i] contains the original prefix of line i, including |
| // trailing whitespace before the start of the content. The indentation |
| // preceding the prefix is not included. |
| // For example, if the line is: |
| // // content |
| // then the original prefix is "// ". |
| SmallVector<StringRef, 16> OriginalPrefix; |
| |
| // Prefix[i] contains the intended leading "//" with trailing spaces to |
| // account for the indentation of content within the comment at line i after |
| // formatting. It can be different than the original prefix when the original |
| // line starts like this: |
| // //content |
| // Then the original prefix is "//", but the prefix is "// ". |
| SmallVector<StringRef, 16> Prefix; |
| |
| SmallVector<unsigned, 16> OriginalContentColumn; |
| |
| /// The token to which the last line of this breakable token belongs |
| /// to; nullptr if that token is the initial token. |
| /// |
| /// The distinction is because if the token of the last line of this breakable |
| /// token is distinct from the initial token, this breakable token owns the |
| /// whitespace before the token of the last line, and the whitespace manager |
| /// must be able to modify it. |
| FormatToken *LastLineTok = nullptr; |
| }; |
| } // namespace format |
| } // namespace clang |
| |
| #endif |