| //===----------------------------------------------------------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| include "llvm/Option/LibraryOptions.td" | |
| def VectorizeOptions : OptionsStruct; | |
| // LoopIdiomVectorize.cpp | |
| defm : BoolField<"disable-loop-idiom-vectorize-all", "false", | |
| "Disable Loop Idiom Vectorize Pass">; | |
| defm : BoolField<"disable-loop-idiom-vectorize-bytecmp", "false", | |
| "Proceed with Loop Idiom Vectorize Pass, but do not convert byte-compare loop(s)">; | |
| defm : BoolField<"disable-loop-idiom-vectorize-find-first-byte", "false", | |
| "Do not convert find-first-byte loop(s)">; | |
| defm : OptionalValueField<"loop-idiom-vectorize-bytecmp-vf", "unsigned", | |
| "The vectorization factor for byte-compare patterns">; | |
| defm : OptionalEnumField<"loop-idiom-vectorize-style", "LoopIdiomVectorizeStyle", "masked,predicated", | |
| ["LoopIdiomVectorizeStyle::Masked", "LoopIdiomVectorizeStyle::Predicated"], | |
| "The vectorization style for loop idiom transform">; | |
| defm : BoolField<"loop-idiom-vectorize-verify", "false", | |
| "Verify loops generated Loop Idiom Vectorize Pass">; | |
| // LoopVectorizationLegality.cpp | |
| defm : BoolField<"enable-histogram-loop-vectorization", "false", | |
| "Enables autovectorization of some loops containing histograms">; | |
| defm : BoolField<"enable-if-conversion", "true", | |
| "Enable if-conversion during vectorization">; | |
| defm : BoolField<"hints-allow-reordering", "true", | |
| "Allow enabling loop hints to reorder FP operations during vectorization">; | |
| defm : BoolField<"lv-strided-pointer-ivs", "false", | |
| "Enable recognition of non-constant strided pointer induction variables">; | |
| defm : EnumField<"scalable-vectorization", "LoopVectorizeHints::ScalableForceKind", "LoopVectorizeHints::SK_Unspecified", "off,preferred,on,always", | |
| ["LoopVectorizeHints::SK_FixedWidthOnly", "LoopVectorizeHints::SK_PreferScalable", "LoopVectorizeHints::SK_PreferScalable", "LoopVectorizeHints::SK_AlwaysScalable"], | |
| "Control whether the compiler can use scalable vectors to vectorize a loop">; | |
| // LoopVectorizationPlanner.cpp | |
| defm : BoolField<"force-target-supports-gather-scatter-ops", "false", | |
| "Assume the target supports gather/scatter operations (used for testing)">; | |
| // Note: This currently only applies to `llvm.masked.load` and | |
| // `llvm.masked.store`. TODO: Extend this to cover other operations as needed. | |
| defm : BoolField<"force-target-supports-masked-memory-ops", "false", | |
| "Assume the target supports masked memory operations (used for testing)">; | |
| defm : BoolField<"force-target-supports-scalable-vectors", "false", | |
| "Pretend that scalable vectors are supported, even if the target does not support them. This flag should only be used for testing">; | |
| defm : BoolField<"prefer-inloop-reductions", "false", | |
| "Prefer in-loop vector reductions, overriding the targets preference">; | |
| defm : ValueField<"scalable-epilogue-vf-cost-scale-factor", "float", "2.0", | |
| "Scale the cost of scalable epilogue VFs by this factor">; | |
| defm : OptionalBoolField<"vectorizer-consider-reg-pressure", | |
| "Discard VFs if their register pressure is too high">; | |
| defm : OptionalBoolField<"vectorizer-maximize-bandwidth", | |
| "Maximize bandwidth when selecting vectorization factor which will be determined by the smallest type in loop">; | |
| defm : BoolField<"vectorizer-maximize-bandwidth-for-vector-calls", "true", | |
| "Try wider VFs if they enable the use of vector variants">; | |
| // LoopVectorize.cpp | |
| defm : BoolField<"enable-early-exit-vectorization", "true", | |
| "Enable vectorization of early exit loops with uncountable exits">; | |
| defm : BoolField<"enable-early-exit-vectorization-with-side-effects", "false", | |
| "Enable vectorization of early exit loops with uncountable exits and side effects">; | |
| defm : BoolField<"enable-epilogue-vectorization", "true", | |
| "Enable vectorization of epilogue loops">; | |
| defm : BoolField<"enable-ind-var-reg-heur", "true", | |
| "Count the induction variable only once when interleaving">; | |
| defm : OptionalBoolField<"enable-interleaved-mem-accesses", | |
| "Enable vectorization on interleaved memory accesses in a loop">; | |
| // Runtime interleave loops for load/store throughput. | |
| defm : BoolField<"enable-loadstore-runtime-interleave", "true", | |
| "Enable runtime interleaving until load/store ports are saturated">; | |
| // An interleave-group may need masking if it resides in a block that needs | |
| // predication, or in order to mask away gaps. | |
| defm : OptionalBoolField<"enable-masked-interleaved-mem-accesses", | |
| "Enable vectorization on masked interleaved memory accesses in a loop">; | |
| defm : BoolField<"enable-vplan-native-path", "false", | |
| "Enable VPlan-native vectorization path with support for outer loop vectorization">; | |
| defm : OptionalEnumField<"epilogue-tail-folding-policy", "TailFoldingPolicyTy", "dont-fold-tail,prefer-fold-tail", | |
| ["TailFoldingPolicyTy::None", "TailFoldingPolicyTy::PreferFoldTail"], | |
| "Epilogue-tail-folding preferences over creating an epilogue loop">; | |
| defm : OptionalValueField<"epilogue-vectorization-minimum-VF", "unsigned", | |
| "Only loops with vectorization factor equal to or larger than the specified value are considered for epilogue vectorization">; | |
| defm : OptionalBoolField<"force-ordered-reductions", | |
| "Enable the vectorisation of loops with in-order (strict) FP reductions">; | |
| defm : BoolField<"force-partial-aliasing-vectorization", "false", | |
| "Replace pointer diff checks with alias masks">; | |
| defm : OptionalEnumField<"force-tail-folding-style", "TailFoldingStyle", "none,data,data-without-lane-mask,data-and-control,data-with-evl", | |
| ["TailFoldingStyle::None", "TailFoldingStyle::Data", "TailFoldingStyle::DataWithoutLaneMask", "TailFoldingStyle::DataAndControlFlow", "TailFoldingStyle::DataWithEVL"], | |
| "Force the tail folding style">; | |
| defm : OptionalValueField<"force-target-instruction-cost", "unsigned", | |
| "A flag that overrides the target's expected cost for an instruction to a single constant value. Mostly useful for getting consistent testing">; | |
| defm : OptionalValueField<"force-target-max-scalar-interleave", "unsigned", | |
| "A flag that overrides the target's max interleave factor for scalar loops">; | |
| defm : OptionalValueField<"force-target-max-vector-interleave", "unsigned", | |
| "A flag that overrides the target's max interleave factor for vectorized loops">; | |
| defm : OptionalValueField<"force-target-num-scalar-regs", "unsigned", | |
| "A flag that overrides the target's number of scalar registers">; | |
| defm : OptionalValueField<"force-target-num-vector-regs", "unsigned", | |
| "A flag that overrides the target's number of vector registers">; | |
| defm : OptionalBoolField<"force-widen-divrem-via-masked-intrinsic", | |
| "Override cost based masked intrinsic widening for div/rem instructions">; | |
| defm : BoolField<"interleave-loops", "true", | |
| "Enable loop interleaving in Loop vectorization passes">; | |
| defm : BoolField<"loop-vectorize-with-block-frequency", "true", | |
| "Enable the use of the block frequency analysis to access PGO heuristics minimizing code growth in cold regions and being more aggressive in hot regions">; | |
| defm : ValueField<"low-trip-count-loop-body-size-limit", "unsigned", "20", | |
| "Minimum number of instructions to vectorize loops with trip counts below tail folding threshold">; | |
| defm : ValueField<"max-nested-scalar-reduction-interleave", "unsigned", "2", | |
| "The maximum interleave count to use when interleaving a scalar reduction in a nested loop">; | |
| defm : ValueField<"pragma-vectorize-scev-check-threshold", "unsigned", "128", | |
| "The maximum number of SCEV checks allowed with a vectorize(enable) pragma">; | |
| defm : BoolField<"prefer-predicated-reduction-select", "false", | |
| "Prefer predicating a reduction operation over an after loop select">; | |
| defm : ValueField<"small-loop-cost", "unsigned", "20", | |
| "The cost of a loop that is considered 'small' by the interleaver">; | |
| defm : OptionalEnumField<"tail-folding-policy", "TailFoldingPolicyTy", "dont-fold-tail,prefer-fold-tail,must-fold-tail", | |
| ["TailFoldingPolicyTy::None", "TailFoldingPolicyTy::PreferFoldTail", "TailFoldingPolicyTy::MustFoldTail"], | |
| "Tail-folding preferences over creating an epilogue loop">; | |
| defm : BoolField<"vectorize-loops", "true", | |
| "Run the Loop vectorization passes">; | |
| // The number of stores in a loop that are allowed to need predication. | |
| defm : ValueField<"vectorize-num-stores-pred", "unsigned", "1", | |
| "Max number of stores to be predicated behind an if">; | |
| // TODO: Move size-based thresholds out of legality checking, make cost based | |
| // decisions instead of hard thresholds. | |
| defm : ValueField<"vectorize-scev-check-threshold", "unsigned", "16", | |
| "The maximum number of SCEV checks allowed">; | |
| // Loops with a known constant trip count below this number are vectorized only | |
| // if no scalar iteration overheads are incurred. | |
| defm : ValueField<"vectorizer-min-trip-count", "unsigned", "16", | |
| "Loops with a constant trip count that is smaller than this value are vectorized only if no scalar iteration overheads are incurred">; | |
| // This flag enables the stress testing of the VPlan H-CFG construction in the | |
| // VPlan-native vectorization path. It must be used in conjuction with | |
| // -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the | |
| // verification of the H-CFGs built. | |
| defm : BoolField<"vplan-build-outerloop-stress-test", "false", | |
| "Build VPlan for every supported loop nest in the function and bail out right after the build (stress test the VPlan H-CFG construction in the VPlan-native vectorization path)">; | |
| let Condition = AssertsOnly in { | |
| defm : ListField<"vplan-print-after", "StringRef", | |
| "Print VPlans after specified VPlan transformations (regexp)">; | |
| defm : BoolField<"vplan-print-after-all", "false", | |
| "Print VPlans after all VPlan transformations">; | |
| defm : ListField<"vplan-print-before", "StringRef", | |
| "Print VPlans before specified VPlan transformations (regexp)">; | |
| defm : BoolField<"vplan-print-before-all", "false", | |
| "Print VPlans before all VPlan transformations">; | |
| defm : BoolField<"vplan-print-vector-region-scope", "false", | |
| "Limit VPlan printing to vector loop region in `-vplan-print-after*` if the plan has one">; | |
| } | |
| defm : BoolField<"vplan-verify-each", "VPlanVerifyEachDefault", | |
| "Verify VPlans after VPlan transforms">; | |
| // SandboxVectorizer/Passes/BundleVec.cpp | |
| let Condition = AssertsOnly in | |
| defm : BoolField<"sbvec-always-verify", "false", | |
| "Helps find bugs by verifying the IR whenever we emit new instructions (*very* expensive)">; | |
| defm : ValueField<"sbvec-stop-at", "unsigned long", "SBVecStopDisabled", | |
| "Vectorize if the invocation count is < than this. 0 disables vectorization">; | |
| defm : ValueField<"sbvec-stop-bndl", "unsigned long", "SBVecStopDisabled", | |
| "Vectorize up to this many bundles">; | |
| // SandboxVectorizer/Passes/SeedCollection.cpp | |
| defm : BoolField<"sbvec-allow-non-pow2", "false", | |
| "Allow non-power-of-2 vectorization">; | |
| defm : ValueField<"sbvec-collect-seeds", "StringRef", "\"stores\"", | |
| "Collect these seeds. Use empty for none or a comma-separated list of 'stores' and 'loads'">; | |
| defm : ValueField<"sbvec-vec-reg-bits", "unsigned", "0", | |
| "Override the vector register size in bits, which is otherwise found by querying TTI">; | |
| // SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp | |
| defm : ValueField<"sbvec-cost-threshold", "int", "0", | |
| "Vectorization cost threshold">; | |
| // SandboxVectorizer/SandboxVectorizer.cpp | |
| // This option is useful for bisection debugging. For example you may use it to | |
| // figure out which filename is the one causing a miscompile. You can specify a | |
| // regex for the filename like: "/[a-m][^/]*" which will enable any file name | |
| // starting with 'a' to 'm' and disable the rest. If the miscompile goes away, | |
| // then we try "/[n-z][^/]*" for the other half of the range, from 'n' to 'z'. | |
| // If we can reproduce the miscompile then we can keep looking in [n-r] and | |
| // [s-z] and so on, in a binary-search fashion. | |
| // | |
| // Please note that we are using [^/]* and not .* to make sure that we are | |
| // matching the actual filename and not some other directory in the path. | |
| defm : ValueField<"sbvec-allow-files", "StringRef", "\".*\"", | |
| "Run the vectorizer only on file paths that match any in the list of comma-separated regex's">; | |
| defm : ValueField<"sbvec-passes", "StringRef", "\"*\"", | |
| "Comma-separated list of vectorizer passes. If not set we run the predefined pipeline">; | |
| defm : BoolField<"sbvec-print-pass-pipeline", "false", | |
| "Prints the pass pipeline and returns">; | |
| // SandboxVectorizer/SeedCollector.cpp | |
| defm : ValueField<"sbvec-seed-bundle-size-limit", "unsigned", "32", | |
| "Limit the size of the seed bundle to cap compilation time">; | |
| defm : ValueField<"sbvec-seed-groups-limit", "unsigned", "256", | |
| "Limit the number of collected seeds groups in a BB to cap compilation time">; | |
| // SandboxVectorizer/VecUtils.cpp | |
| defm : ValueField<"sbvec-max-users-to-consider", "unsigned", "16", | |
| "Limit the number of a seed's users that getNextUserBundles() will examine as candidates for a matching bundle, to cap compilation time">; | |
| // SLPVectorizer.cpp | |
| // Enables vectorization of copyable elements. | |
| defm : BoolField<"slp-copyable-elements", "true", | |
| "Try to replace values with the idempotent instructions for better vectorization">; | |
| defm : ValueField<"slp-cost-loop-trip-count", "unsigned", "2", | |
| "Loop trip count, considered by the cost model during modeling (0=loops are ignored and considered flat code)">; | |
| defm : BoolField<"slp-disable-tree-reorder", "false", | |
| "Disable tree reordering even if it is profitable. Used for testing only">; | |
| defm : BoolField<"slp-enable-masked-stores", "true", | |
| "Enable vectorization of non-consecutive stores as a single masked store, when the target supports masked stores">; | |
| defm : BoolField<"slp-enable-strided-stores", "false", | |
| "Enable SLP trees to be built from strided store chains">; | |
| defm : BoolField<"slp-force-strided-loads", "false", | |
| "Generate strided loads even if they are not profitable. Used for testing only">; | |
| defm : BoolField<"slp-inst-count-check", "true", | |
| "Reject vectorization if vector instruction count exceeds scalar instruction count">; | |
| // The maximum depth that the look-ahead score heuristic will explore. The | |
| // higher this value, the higher the compilation time overhead. | |
| defm : ValueField<"slp-max-look-ahead-depth", "int", "2", | |
| "The maximum look-ahead depth for operand reordering scores">; | |
| defm : OptionalValueField<"slp-max-reg-size", "int", | |
| "Attempt to vectorize for this register size in bits">; | |
| // The maximum depth that the look-ahead score heuristic will explore when it | |
| // probing among candidates for vectorization tree roots. The higher this value, | |
| // the higher the compilation time overhead but unlike similar limit for | |
| // operands ordering this is less frequently used, hence impact of higher value | |
| // is less noticeable. | |
| defm : ValueField<"slp-max-root-look-ahead-depth", "int", "2", | |
| "The maximum look-ahead depth for searching best rooting option">; | |
| // Maximum number of runtime alias checks (one per pair of base objects) that | |
| // may guard a single versioned region. | |
| defm : ValueField<"slp-max-runtime-alias-checks", "unsigned", "8", | |
| "The maximum number of runtime alias checks generated to guard a single SLP-vectorized region">; | |
| defm : ValueField<"slp-max-stride", "unsigned", "8", | |
| "The maximum stride, considered to be profitable">; | |
| defm : OptionalValueField<"slp-max-vf", "unsigned", | |
| "Maximum SLP vectorization factor (0=unlimited)">; | |
| defm : OptionalValueField<"slp-min-reg-size", "int", | |
| "Attempt to vectorize for this register size in bits">; | |
| defm : ValueField<"slp-min-strided-loads", "unsigned", "2", | |
| "The minimum number of loads, which should be considered strided, if the stride is > 1 or is runtime value">; | |
| defm : ValueField<"slp-min-strided-stores", "unsigned", "2", | |
| "The minimum number of stores, which should be considered strided, if the stride is > 1 or is runtime value">; | |
| defm : ValueField<"slp-min-tree-size", "unsigned", "3", | |
| "Only vectorize small trees if they are fully vectorizable">; | |
| defm : BoolField<"slp-non-vectorizables-as-reductions", "false", | |
| "Use non-vectorizable instructions as potential reduction roots">; | |
| // Refine the loop-aware cost scaling of gather/buildvector tree entries by | |
| // using the per-lane execution scale of the operand that feeds each lane, | |
| // instead of a single whole-entry scale. This matches the LICM hoisting | |
| // performed by optimizeGatherSequence() at codegen time: lanes whose operands | |
| // are loop-invariant in an inner loop contribute the outer loop's execution | |
| // scale rather than the inner loop's, which avoids over-costing buildvectors | |
| // that bridge values from outer loop nests into an inner loop. | |
| defm : BoolField<"slp-per-lane-gather-scale", "true", | |
| "Use per-lane execution scale for gather/buildvector tree entries to model LICM-hoistable buildvector sequences">; | |
| defm : ValueField<"slp-phi-list-budget", "unsigned", "1024", | |
| "Maximum size of the phi list to be vectorized">; | |
| defm : ValueField<"slp-phi-vectorization-budget", "unsigned", "1024", | |
| "Do not vectorize a bundle of PHI nodes if the product of the bundle size and the number of incoming values exceeds this value, to limit the compile time spent on wide PHIs">; | |
| defm : BoolField<"slp-postprocess-stores-operands", "false", | |
| "Force vectorization of non-vectorizable stores operands">; | |
| // Gather operands of associative single-use binary chains into one node. | |
| defm : BoolField<"slp-reassociate-ops", "true", | |
| "Gather operands of associative binary chains into one node">; | |
| // The family-realigned seed already groups the vectorizable columns; the | |
| // VLOperands polish on top is quadratic in the column count, so past this many | |
| // columns keep the seed instead. | |
| defm : ValueField<"slp-reassociate-reorder-limit", "unsigned", "32", | |
| "Max flattened operand columns for which associative-chain reordering runs the full operand reorder">; | |
| defm : ValueField<"slp-recursion-max-depth", "unsigned", "12", | |
| "Limit the recursion depth when building a vectorizable tree">; | |
| defm : ValueField<"slp-reduction-budget", "unsigned", "65536", | |
| "Limit the number of scalars in the rejected trees of the same reduction">; | |
| defm : BoolField<"slp-revec", "false", | |
| "Enable vectorization for wider vector utilization">; | |
| // The runtime checks and the guard branch execute on both the vector and the | |
| // scalar fallback path, so they add overhead to the scalar code. | |
| defm : ValueField<"slp-runtime-alias-checks-max-scalar-cost-percent", "unsigned", "25", | |
| "Maximum SLP runtime alias check cost, as a percentage of the guarded scalar region cost, before versioning is rejected to avoid pessimizing the scalar fallback path">; | |
| // Limits the size of scheduling regions in a block. It avoid long compile times | |
| // for _very_ large blocks where vector instructions are spread over a wide | |
| // range. This limit is way higher than needed by real-world functions. | |
| defm : ValueField<"slp-schedule-budget", "int", "100000", | |
| "Limit the size of the SLP scheduling region per block">; | |
| defm : BoolField<"slp-split-alternate-instructions", "true", | |
| "Improve the code quality by splitting alternate instructions">; | |
| defm : OptionalValueField<"slp-threshold", "int", | |
| "Only vectorize if you gain more than this number">; | |
| // The scalar loop left by the loop vectorizer runs only a few iterations or the | |
| // ranges its runtime checks found overlapping, where the checks executed on | |
| // each iteration are unlikely to pay off, so they get a tighter bound. | |
| defm : ValueField<"slp-vec-loop-rt-checks-cost-percent", "unsigned", "10", | |
| "Maximum SLP runtime alias check cost, as a percentage of the guarded scalar region cost, for blocks of loops already vectorized by the loop vectorizer">; | |
| defm : BoolField<"slp-vectorize-hor", "true", | |
| "Attempt to vectorize horizontal reductions">; | |
| defm : BoolField<"slp-vectorize-hor-store", "false", | |
| "Attempt to vectorize horizontal reductions feeding into a store">; | |
| defm : BoolField<"slp-vectorize-loop-acc-rdx", "true", | |
| "Emit loop-carried horizontal reductions as a vector accumulator in the loop plus a single reduction after it">; | |
| defm : BoolField<"slp-vectorize-non-power-of-2", "false", | |
| "Try to vectorize with non-power-of-2 number of elements">; | |
| defm : BoolField<"slp-vectorize-once-used", "true", | |
| "Use instructions with the single user as standalone vectorization seeds">; | |
| defm : BoolField<"slp-vectorize-poor-throughput", "true", | |
| "Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as standalone vectorization seeds">; | |
| // Enable versioning of a basic block with runtime alias checks. | |
| defm : BoolField<"slp-vectorize-with-runtime-alias-checks", "true", | |
| "Allow SLP to version a block with runtime alias checks to vectorize trees blocked by may-alias memory dependencies">; | |
| defm : BoolField<"vectorize-slp", "true", | |
| "Run the SLP vectorization passes">; | |
| defm : BoolField<"view-slp-tree", "false", | |
| "Display the SLP trees with Graphviz">; | |
| // VectorCombine.cpp | |
| defm : BoolField<"disable-binop-extract-shuffle", "false", | |
| "Disable binop extract to shuffle transforms">; | |
| defm : BoolField<"disable-vector-combine", "false", | |
| "Disable all vector combine transforms">; | |
| defm : ValueField<"vector-combine-max-scan-instrs", "unsigned", "30", | |
| "Max number of instructions to scan for vector combining">; | |
| // VPlan.cpp | |
| defm : BoolField<"vplan-print-in-dot-format", "false", | |
| "Use dot format instead of plain text when dumping VPlans">; | |
| // VPlanTransforms.cpp | |
| // TODO: Remove this once the partial reduction intrinsics are no worse than | |
| // normal vector operations. | |
| defm : BoolField<"use-partial-reductions-by-default", "false", | |
| "Use partial reduction intrinsics for all supported unordered reductions">; |