Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 1 | //==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines the builtins for RISC-V V-extension. See: |
| 10 | // |
| 11 | // https://github.com/riscv/rvv-intrinsic-doc |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | //===----------------------------------------------------------------------===// |
| 16 | // Instruction definitions |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | // Each record of the class RVVBuiltin defines a collection of builtins (i.e. |
| 19 | // "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1", |
| 20 | // "vadd_vv_i32m2", etc). |
| 21 | // |
| 22 | // The elements of this collection are defined by an instantiation process the |
| 23 | // range of which is specified by the cross product of the LMUL attribute and |
| 24 | // every element in the attribute TypeRange. By default builtins have LMUL = [1, |
| 25 | // 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we |
| 26 | // use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL. |
| 27 | // |
| 28 | // LMUL represents the fact that the types of values used by that builtin are |
| 29 | // values generated by instructions that are executed under that LMUL. However, |
| 30 | // this does not mean the builtin is necessarily lowered into an instruction |
| 31 | // that executes under the specified LMUL. An example where this happens are |
| 32 | // loads and stores of masks. A mask like `vbool8_t` can be generated, for |
| 33 | // instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two |
| 34 | // `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will |
| 35 | // be performed under LMUL=1 because mask registers are not grouped. |
| 36 | // |
| 37 | // TypeRange is a non-empty sequence of basic types: |
| 38 | // |
| 39 | // c: int8_t (i8) |
| 40 | // s: int16_t (i16) |
| 41 | // i: int32_t (i32) |
| 42 | // l: int64_t (i64) |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 43 | // x: float16_t (half) |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 44 | // f: float32_t (float) |
| 45 | // d: float64_t (double) |
| 46 | // |
| 47 | // This way, given an LMUL, a record with a TypeRange "sil" will cause the |
| 48 | // definition of 3 builtins. Each type "t" in the TypeRange (in this example |
| 49 | // they are int16_t, int32_t, int64_t) is used as a parameter that drives the |
| 50 | // definition of that particular builtin (for the given LMUL). |
| 51 | // |
| 52 | // During the instantiation, types can be transformed or modified using type |
| 53 | // transformers. Given a type "t" the following primitive type transformers can |
| 54 | // be applied to it to yield another type. |
| 55 | // |
| 56 | // e: type of "t" as is (identity) |
| 57 | // v: computes a vector type whose element type is "t" for the current LMUL |
| 58 | // w: computes a vector type identical to what 'v' computes except for the |
| 59 | // element type which is twice as wide as the element type of 'v' |
| 60 | // q: computes a vector type identical to what 'v' computes except for the |
| 61 | // element type which is four times as wide as the element type of 'v' |
| 62 | // o: computes a vector type identical to what 'v' computes except for the |
| 63 | // element type which is eight times as wide as the element type of 'v' |
| 64 | // m: computes a vector type identical to what 'v' computes except for the |
| 65 | // element type which is bool |
| 66 | // 0: void type, ignores "t" |
| 67 | // z: size_t, ignores "t" |
| 68 | // t: ptrdiff_t, ignores "t" |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 69 | // u: unsigned long, ignores "t" |
| 70 | // l: long, ignores "t" |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 71 | // |
| 72 | // So for instance if t is "i", i.e. int, then "e" will yield int again. "v" |
| 73 | // will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t. |
| 74 | // Accordingly "w" would yield __rvv_int64m2_t. |
| 75 | // |
| 76 | // A type transformer can be prefixed by other non-primitive type transformers. |
| 77 | // |
| 78 | // P: constructs a pointer to the current type |
| 79 | // C: adds const to the type |
| 80 | // K: requires the integer type to be a constant expression |
| 81 | // U: given an integer type or vector type, computes its unsigned variant |
| 82 | // I: given a vector type, compute the vector type with integer type |
| 83 | // elements of the same width |
| 84 | // F: given a vector type, compute the vector type with floating-point type |
| 85 | // elements of the same width |
| 86 | // S: given a vector type, computes its equivalent one for LMUL=1. This is a |
| 87 | // no-op if the vector was already LMUL=1 |
Zakk Chen | 88c2d4c | 2021-03-17 20:25:32 -0700 | [diff] [blame] | 88 | // (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a |
| 89 | // vector type (SEW and LMUL) and EEW (8/16/32/64), computes its |
| 90 | // equivalent integer vector type with EEW and corresponding ELMUL (elmul = |
| 91 | // (eew/sew) * lmul). For example, vector type is __rvv_float16m4 |
| 92 | // (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector |
| 93 | // type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new |
| 94 | // builtins if its equivalent type has illegal lmul. |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 95 | // (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another |
| 96 | // vector type which only changed SEW as given value. Ignore to define a new |
| 97 | // builtin if its equivalent type has illegal lmul or the SEW does not changed. |
| 98 | // (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW |
| 99 | // and LMUL), and computes another vector type which only changed LMUL as |
| 100 | // given value. The new LMUL should be smaller than the old one. Ignore to |
| 101 | // define a new builtin if its equivalent type has illegal lmul. |
| 102 | // (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW |
| 103 | // and LMUL), and computes another vector type which only changed LMUL as |
| 104 | // given value. The new LMUL should be larger than the old one. Ignore to |
| 105 | // define a new builtin if its equivalent type has illegal lmul. |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 106 | // |
| 107 | // Following with the example above, if t is "i", then "Ue" will yield unsigned |
| 108 | // int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would |
| 109 | // yield __rvv_float64m2_t, etc. |
| 110 | // |
| 111 | // Each builtin is then defined by applying each type in TypeRange against the |
| 112 | // sequence of type transformers described in Suffix and Prototype. |
| 113 | // |
| 114 | // The name of the builtin is defined by the Name attribute (which defaults to |
| 115 | // the name of the class) appended (separated with an underscore) the Suffix |
| 116 | // attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il", |
| 117 | // the builtin generated will be __builtin_rvv_foo_i32m1 and |
| 118 | // __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one |
| 119 | // type transformer (say "vv") each of the types is separated with an |
| 120 | // underscore as in "__builtin_rvv_foo_i32m1_i32m1". |
| 121 | // |
| 122 | // The C/C++ prototype of the builtin is defined by the Prototype attribute. |
| 123 | // Prototype is a non-empty sequence of type transformers, the first of which |
| 124 | // is the return type of the builtin and the rest are the parameters of the |
| 125 | // builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si" |
| 126 | // a first builtin will have type |
| 127 | // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin |
| 128 | // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again |
| 129 | // under LMUL=1). |
| 130 | // |
| 131 | // There are a number of attributes that are used to constraint the number and |
| 132 | // shape of the builtins generated. Refer to the comments below for them. |
| 133 | class RVVBuiltin<string suffix, string prototype, string type_range, |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 134 | string mangled_suffix = ""> { |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 135 | // Base name that will be prepended in __builtin_rvv_ and appended the |
| 136 | // computed Suffix. |
| 137 | string Name = NAME; |
| 138 | |
| 139 | // If not empty, each instantiated builtin will have this appended after an |
| 140 | // underscore (_). It is instantiated like Prototype. |
| 141 | string Suffix = suffix; |
| 142 | |
| 143 | // If empty, default MangledName is sub string of `Name` which end of first |
| 144 | // '_'. For example, the default mangled name is `vadd` for Name `vadd_vv`. |
| 145 | // It's used for describe some special naming cases. |
| 146 | string MangledName = ""; |
| 147 | |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 148 | // If not empty, each MangledName will have this appended after an |
| 149 | // underscore (_). It is instantiated like Prototype. |
| 150 | string MangledSuffix = mangled_suffix; |
| 151 | |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 152 | // The different variants of the builtin, parameterised with a type. |
| 153 | string TypeRange = type_range; |
| 154 | |
| 155 | // We use each type described in TypeRange and LMUL with prototype to |
| 156 | // instantiate a specific element of the set of builtins being defined. |
| 157 | // Prototype attribute defines the C/C++ prototype of the builtin. It is a |
| 158 | // non-empty sequence of type transformers, the first of which is the return |
| 159 | // type of the builtin and the rest are the parameters of the builtin, in |
| 160 | // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a |
| 161 | // first builtin will have type |
| 162 | // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin |
| 163 | // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t). |
| 164 | string Prototype = prototype; |
| 165 | |
| 166 | // This builtin has a masked form. |
| 167 | bit HasMask = true; |
| 168 | |
| 169 | // If HasMask, this flag states that this builtin has a maskedoff operand. It |
| 170 | // is always the first operand in builtin and IR intrinsic. |
| 171 | bit HasMaskedOffOperand = true; |
| 172 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 173 | // This builtin has a granted vector length parameter. |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 174 | bit HasVL = true; |
| 175 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 176 | // There are several cases for specifying tail policy. |
| 177 | // 1. Add tail policy argument to masked intrinsics. It may have the maskedoff |
| 178 | // argument or not. |
| 179 | // * Have the maskedoff argument: (HasPolicy = true, HasMaskedOffOperand = true) |
| 180 | // Ex: vadd_vv_i8m1_mt(mask, maskedoff, op1, op2, vl, ta); |
| 181 | // * Do not have the maskedoff argument: (HasPolicy = true, HasMaskedOffOperand = false) |
| 182 | // Ex: vmacc_vv_i8m1_mt(mask, vd, vs1, vs2, vl, ta); |
| 183 | // 2. Add dest argument for no mask intrinsics. (TODO) |
| 184 | // Ex: vmv_v_x_i8m1_t(dest, src, vl); |
| 185 | // 3. Always tail agnostic. (HasPolicy = false) |
| 186 | // Ex: vmseq_vv_i8m1_b8_m(mask, maskedoff, op1, op2, vl); |
| 187 | // The tail policy argument is located at the last position. |
| 188 | bit HasPolicy = true; |
| 189 | |
Zakk Chen | 821547c | 2021-03-22 07:51:52 -0700 | [diff] [blame] | 190 | // This builtin supports non-masked function overloading api. |
| 191 | // All masked operations support overloading api. |
| 192 | bit HasNoMaskedOverloaded = true; |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 193 | |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 194 | // This builtin is valid for the given Log2LMULs. |
| 195 | list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3]; |
| 196 | |
Zakk Chen | 95c0125 | 2021-02-25 00:15:14 -0800 | [diff] [blame] | 197 | // Manual code in clang codegen riscv_vector_builtin_cg.inc |
| 198 | code ManualCodegen = [{}]; |
| 199 | code ManualCodegenMask = [{}]; |
| 200 | |
| 201 | // When emit the automatic clang codegen, it describes what types we have to use |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 202 | // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise, |
| 203 | // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd |
| 204 | // parameter of the unmasked version. k can't be the mask operand's position. |
| 205 | list<int> IntrinsicTypes = []; |
| 206 | |
| 207 | // If these names are not empty, this is the ID of the LLVM intrinsic |
| 208 | // we want to lower to. |
| 209 | string IRName = NAME; |
| 210 | |
| 211 | // If HasMask, this is the ID of the LLVM intrinsic we want to lower to. |
| 212 | string IRNameMask = NAME #"_mask"; |
Zakk Chen | 95c0125 | 2021-02-25 00:15:14 -0800 | [diff] [blame] | 213 | |
| 214 | // If non empty, this is the code emitted in the header, otherwise |
| 215 | // an automatic definition in header is emitted. |
| 216 | string HeaderCode = ""; |
| 217 | |
Craig Topper | 6c6abb1 | 2021-10-21 16:02:42 -0700 | [diff] [blame] | 218 | // Sub extension of vector spec. Currently only support Zvlsseg. |
ShihPo Hung | 11072a0 | 2021-04-21 01:48:02 -0700 | [diff] [blame] | 219 | string RequiredExtension = ""; |
| 220 | |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 221 | // Number of fields for Zvlsseg. |
| 222 | int NF = 1; |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 223 | } |
| 224 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 225 | class RVVHeader |
| 226 | { |
| 227 | code HeaderCode; |
| 228 | } |
| 229 | |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 230 | //===----------------------------------------------------------------------===// |
| 231 | // Basic classes with automatic codegen. |
| 232 | //===----------------------------------------------------------------------===// |
| 233 | |
Zakk Chen | 5f7739b | 2021-04-08 08:21:06 -0700 | [diff] [blame] | 234 | class RVVOutBuiltin<string suffix, string prototype, string type_range> |
| 235 | : RVVBuiltin<suffix, prototype, type_range> { |
| 236 | let IntrinsicTypes = [-1]; |
| 237 | } |
| 238 | |
| 239 | class RVVOp0Builtin<string suffix, string prototype, string type_range> |
| 240 | : RVVBuiltin<suffix, prototype, type_range> { |
| 241 | let IntrinsicTypes = [0]; |
| 242 | } |
| 243 | |
Zakk Chen | ccc624b | 2021-04-06 03:26:44 -0700 | [diff] [blame] | 244 | class RVVOutOp1Builtin<string suffix, string prototype, string type_range> |
| 245 | : RVVBuiltin<suffix, prototype, type_range> { |
| 246 | let IntrinsicTypes = [-1, 1]; |
| 247 | } |
| 248 | |
| 249 | class RVVOutOp0Op1Builtin<string suffix, string prototype, string type_range> |
| 250 | : RVVBuiltin<suffix, prototype, type_range> { |
| 251 | let IntrinsicTypes = [-1, 0, 1]; |
| 252 | } |
| 253 | |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 254 | multiclass RVVBuiltinSet<string intrinsic_name, string type_range, |
| 255 | list<list<string>> suffixes_prototypes, |
| 256 | list<int> intrinsic_types> { |
| 257 | let IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask", |
| 258 | IntrinsicTypes = intrinsic_types in { |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 259 | foreach s_p = suffixes_prototypes in { |
| 260 | let Name = NAME # "_" # s_p[0] in { |
| 261 | defvar suffix = s_p[1]; |
| 262 | defvar prototype = s_p[2]; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 263 | def : RVVBuiltin<suffix, prototype, type_range>; |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 264 | } |
| 265 | } |
| 266 | } |
| 267 | } |
| 268 | |
Zakk Chen | 0a18ea0 | 2021-03-29 09:38:55 -0700 | [diff] [blame] | 269 | // IntrinsicTypes is output, op0, op1 [-1, 0, 1] |
| 270 | multiclass RVVOutOp0Op1BuiltinSet<string intrinsic_name, string type_range, |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 271 | list<list<string>> suffixes_prototypes> |
| 272 | : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, |
Zakk Chen | 0a18ea0 | 2021-03-29 09:38:55 -0700 | [diff] [blame] | 273 | [-1, 0, 1]>; |
Zakk Chen | 0a18ea0 | 2021-03-29 09:38:55 -0700 | [diff] [blame] | 274 | |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 275 | multiclass RVVOutBuiltinSet<string intrinsic_name, string type_range, |
| 276 | list<list<string>> suffixes_prototypes> |
| 277 | : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1]>; |
| 278 | |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 279 | multiclass RVVOp0BuiltinSet<string intrinsic_name, string type_range, |
| 280 | list<list<string>> suffixes_prototypes> |
| 281 | : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0]>; |
| 282 | |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 283 | // IntrinsicTypes is output, op1 [-1, 1] |
| 284 | multiclass RVVOutOp1BuiltinSet<string intrinsic_name, string type_range, |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 285 | list<list<string>> suffixes_prototypes> |
| 286 | : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1]>; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 287 | |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 288 | multiclass RVVOp0Op1BuiltinSet<string intrinsic_name, string type_range, |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 289 | list<list<string>> suffixes_prototypes> |
| 290 | : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0, 1]>; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 291 | |
| 292 | multiclass RVVOutOp1Op2BuiltinSet<string intrinsic_name, string type_range, |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 293 | list<list<string>> suffixes_prototypes> |
| 294 | : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1, 2]>; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 295 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 296 | multiclass RVVSignedBinBuiltinSet |
| 297 | : RVVOutOp1BuiltinSet<NAME, "csil", |
| 298 | [["vv", "v", "vvv"], |
| 299 | ["vx", "v", "vve"]]>; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 300 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 301 | multiclass RVVUnsignedBinBuiltinSet |
| 302 | : RVVOutOp1BuiltinSet<NAME, "csil", |
| 303 | [["vv", "Uv", "UvUvUv"], |
| 304 | ["vx", "Uv", "UvUvUe"]]>; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 305 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 306 | multiclass RVVIntBinBuiltinSet |
| 307 | : RVVSignedBinBuiltinSet, |
| 308 | RVVUnsignedBinBuiltinSet; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 309 | |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 310 | multiclass RVVSlideOneBuiltinSet |
| 311 | : RVVOutOp1BuiltinSet<NAME, "csil", |
| 312 | [["vx", "v", "vve"], |
| 313 | ["vx", "Uv", "UvUve"]]>; |
| 314 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 315 | multiclass RVVSignedShiftBuiltinSet |
| 316 | : RVVOutOp1BuiltinSet<NAME, "csil", |
| 317 | [["vv", "v", "vvUv"], |
| 318 | ["vx", "v", "vvz"]]>; |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 319 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 320 | multiclass RVVUnsignedShiftBuiltinSet |
| 321 | : RVVOutOp1BuiltinSet<NAME, "csil", |
| 322 | [["vv", "Uv", "UvUvUv"], |
| 323 | ["vx", "Uv", "UvUvz"]]>; |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 324 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 325 | multiclass RVVShiftBuiltinSet |
| 326 | : RVVSignedShiftBuiltinSet, |
| 327 | RVVUnsignedShiftBuiltinSet; |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 328 | |
| 329 | let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 330 | multiclass RVVSignedNShiftBuiltinSet |
| 331 | : RVVOutOp0Op1BuiltinSet<NAME, "csil", |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 332 | [["wv", "v", "vwUv"], |
| 333 | ["wx", "v", "vwz"]]>; |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 334 | multiclass RVVUnsignedNShiftBuiltinSet |
| 335 | : RVVOutOp0Op1BuiltinSet<NAME, "csil", |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 336 | [["wv", "Uv", "UvUwUv"], |
| 337 | ["wx", "Uv", "UvUwz"]]>; |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 338 | } |
| 339 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 340 | multiclass RVVCarryinBuiltinSet |
| 341 | : RVVOutOp1BuiltinSet<NAME, "csil", |
| 342 | [["vvm", "v", "vvvm"], |
| 343 | ["vxm", "v", "vvem"], |
| 344 | ["vvm", "Uv", "UvUvUvm"], |
| 345 | ["vxm", "Uv", "UvUvUem"]]>; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 346 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 347 | multiclass RVVCarryOutInBuiltinSet<string intrinsic_name> |
| 348 | : RVVOp0Op1BuiltinSet<intrinsic_name, "csil", |
| 349 | [["vvm", "vm", "mvvm"], |
| 350 | ["vxm", "vm", "mvem"], |
| 351 | ["vvm", "Uvm", "mUvUvm"], |
| 352 | ["vxm", "Uvm", "mUvUem"]]>; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 353 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 354 | multiclass RVVSignedMaskOutBuiltinSet |
| 355 | : RVVOp0Op1BuiltinSet<NAME, "csil", |
| 356 | [["vv", "vm", "mvv"], |
| 357 | ["vx", "vm", "mve"]]>; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 358 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 359 | multiclass RVVUnsignedMaskOutBuiltinSet |
| 360 | : RVVOp0Op1BuiltinSet<NAME, "csil", |
| 361 | [["vv", "Uvm", "mUvUv"], |
| 362 | ["vx", "Uvm", "mUvUe"]]>; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 363 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 364 | multiclass RVVIntMaskOutBuiltinSet |
| 365 | : RVVSignedMaskOutBuiltinSet, |
| 366 | RVVUnsignedMaskOutBuiltinSet; |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 367 | |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 368 | class RVVIntExt<string intrinsic_name, string suffix, string prototype, |
| 369 | string type_range> |
| 370 | : RVVBuiltin<suffix, prototype, type_range> { |
| 371 | let IRName = intrinsic_name; |
| 372 | let IRNameMask = intrinsic_name # "_mask"; |
| 373 | let MangledName = NAME; |
| 374 | let IntrinsicTypes = [-1, 0]; |
| 375 | } |
| 376 | |
| 377 | let HasMaskedOffOperand = false in { |
| 378 | multiclass RVVIntTerBuiltinSet { |
| 379 | defm "" : RVVOutOp1BuiltinSet<NAME, "csil", |
| 380 | [["vv", "v", "vvvv"], |
| 381 | ["vx", "v", "vvev"], |
| 382 | ["vv", "Uv", "UvUvUvUv"], |
| 383 | ["vx", "Uv", "UvUvUeUv"]]>; |
| 384 | } |
| 385 | multiclass RVVFloatingTerBuiltinSet { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 386 | defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 387 | [["vv", "v", "vvvv"], |
| 388 | ["vf", "v", "vvev"]]>; |
| 389 | } |
| 390 | } |
| 391 | |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 392 | let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in { |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 393 | multiclass RVVFloatingWidenTerBuiltinSet { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 394 | defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf", |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 395 | [["vv", "w", "wwvv"], |
| 396 | ["vf", "w", "wwev"]]>; |
| 397 | } |
| 398 | } |
| 399 | |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 400 | multiclass RVVFloatingBinBuiltinSet |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 401 | : RVVOutOp1BuiltinSet<NAME, "xfd", |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 402 | [["vv", "v", "vvv"], |
| 403 | ["vf", "v", "vve"]]>; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 404 | |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 405 | multiclass RVVFloatingBinVFBuiltinSet |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 406 | : RVVOutOp1BuiltinSet<NAME, "xfd", |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 407 | [["vf", "v", "vve"]]>; |
| 408 | |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 409 | multiclass RVVFloatingMaskOutBuiltinSet |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 410 | : RVVOp0Op1BuiltinSet<NAME, "xfd", |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 411 | [["vv", "vm", "mvv"], |
| 412 | ["vf", "vm", "mve"]]>; |
| 413 | |
| 414 | multiclass RVVFloatingMaskOutVFBuiltinSet |
| 415 | : RVVOp0Op1BuiltinSet<NAME, "fd", |
| 416 | [["vf", "vm", "mve"]]>; |
Zakk Chen | f720c22 | 2021-03-30 08:55:46 -0700 | [diff] [blame] | 417 | |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 418 | class RVVMaskBinBuiltin : RVVOutBuiltin<"m", "mmm", "c"> { |
| 419 | let Name = NAME # "_mm"; |
| 420 | let HasMask = false; |
| 421 | } |
| 422 | |
| 423 | class RVVMaskUnaryBuiltin : RVVOutBuiltin<"m", "mm", "c"> { |
| 424 | let Name = NAME # "_m"; |
| 425 | } |
| 426 | |
| 427 | class RVVMaskNullaryBuiltin : RVVOutBuiltin<"m", "m", "c"> { |
| 428 | let Name = NAME # "_m"; |
| 429 | let HasMask = false; |
| 430 | let HasNoMaskedOverloaded = false; |
| 431 | } |
| 432 | |
| 433 | class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> { |
| 434 | let Name = NAME # "_m"; |
| 435 | let HasMaskedOffOperand = false; |
| 436 | } |
| 437 | |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 438 | let HasMaskedOffOperand = false in { |
| 439 | multiclass RVVSlideBuiltinSet { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 440 | defm "" : RVVOutBuiltinSet<NAME, "csilxfd", |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 441 | [["vx","v", "vvvz"]]>; |
| 442 | defm "" : RVVOutBuiltinSet<NAME, "csil", |
| 443 | [["vx","Uv", "UvUvUvz"]]>; |
| 444 | } |
| 445 | } |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 446 | |
Zakk Chen | 5f7739b | 2021-04-08 08:21:06 -0700 | [diff] [blame] | 447 | class RVVFloatingUnaryBuiltin<string builtin_suffix, string ir_suffix, |
| 448 | string prototype> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 449 | : RVVOutBuiltin<ir_suffix, prototype, "xfd"> { |
Zakk Chen | 5f7739b | 2021-04-08 08:21:06 -0700 | [diff] [blame] | 450 | let Name = NAME # "_" # builtin_suffix; |
| 451 | } |
| 452 | |
| 453 | class RVVFloatingUnaryVVBuiltin : RVVFloatingUnaryBuiltin<"v", "v", "vv">; |
| 454 | |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 455 | class RVVConvBuiltin<string suffix, string prototype, string type_range, |
| 456 | string mangled_name> |
| 457 | : RVVBuiltin<suffix, prototype, type_range> { |
| 458 | let IntrinsicTypes = [-1, 0]; |
| 459 | let MangledName = mangled_name; |
| 460 | } |
| 461 | |
| 462 | class RVVConvToSignedBuiltin<string mangled_name> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 463 | : RVVConvBuiltin<"Iv", "Ivv", "xfd", mangled_name>; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 464 | |
| 465 | class RVVConvToUnsignedBuiltin<string mangled_name> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 466 | : RVVConvBuiltin<"Uv", "Uvv", "xfd", mangled_name>; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 467 | |
| 468 | class RVVConvToWidenSignedBuiltin<string mangled_name> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 469 | : RVVConvBuiltin<"Iw", "Iwv", "xf", mangled_name>; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 470 | |
| 471 | class RVVConvToWidenUnsignedBuiltin<string mangled_name> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 472 | : RVVConvBuiltin<"Uw", "Uwv", "xf", mangled_name>; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 473 | |
| 474 | class RVVConvToNarrowingSignedBuiltin<string mangled_name> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 475 | : RVVConvBuiltin<"Iv", "IvFw", "csi", mangled_name>; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 476 | |
| 477 | class RVVConvToNarrowingUnsignedBuiltin<string mangled_name> |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 478 | : RVVConvBuiltin<"Uv", "UvFw", "csi", mangled_name>; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 479 | |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 480 | let HasMaskedOffOperand = false in { |
| 481 | multiclass RVVSignedReductionBuiltin { |
| 482 | defm "" : RVVOutOp1BuiltinSet<NAME, "csil", |
| 483 | [["vs", "vSv", "SvSvvSv"]]>; |
| 484 | } |
| 485 | multiclass RVVUnsignedReductionBuiltin { |
| 486 | defm "" : RVVOutOp1BuiltinSet<NAME, "csil", |
| 487 | [["vs", "UvUSv", "USvUSvUvUSv"]]>; |
| 488 | } |
| 489 | multiclass RVVFloatingReductionBuiltin { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 490 | defm "" : RVVOutOp1BuiltinSet<NAME, "xfd", |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 491 | [["vs", "vSv", "SvSvvSv"]]>; |
| 492 | } |
| 493 | multiclass RVVFloatingWidenReductionBuiltin { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 494 | defm "" : RVVOutOp1BuiltinSet<NAME, "xf", |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 495 | [["vs", "vSw", "SwSwvSw"]]>; |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | multiclass RVVIntReductionBuiltinSet |
| 500 | : RVVSignedReductionBuiltin, |
| 501 | RVVUnsignedReductionBuiltin; |
| 502 | |
Zakk Chen | ccc624b | 2021-04-06 03:26:44 -0700 | [diff] [blame] | 503 | // For widen operation which has different mangling name. |
| 504 | multiclass RVVWidenBuiltinSet<string intrinsic_name, string type_range, |
| 505 | list<list<string>> suffixes_prototypes> { |
| 506 | let Log2LMUL = [-3, -2, -1, 0, 1, 2], |
| 507 | IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask" in { |
| 508 | foreach s_p = suffixes_prototypes in { |
| 509 | let Name = NAME # "_" # s_p[0], |
| 510 | MangledName = NAME # "_" # s_p[0] in { |
| 511 | defvar suffix = s_p[1]; |
| 512 | defvar prototype = s_p[2]; |
| 513 | def : RVVOutOp0Op1Builtin<suffix, prototype, type_range>; |
| 514 | } |
| 515 | } |
| 516 | } |
| 517 | } |
| 518 | |
| 519 | // For widen operation with widen operand which has different mangling name. |
| 520 | multiclass RVVWidenWOp0BuiltinSet<string intrinsic_name, string type_range, |
| 521 | list<list<string>> suffixes_prototypes> { |
| 522 | let Log2LMUL = [-3, -2, -1, 0, 1, 2], |
| 523 | IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask" in { |
| 524 | foreach s_p = suffixes_prototypes in { |
| 525 | let Name = NAME # "_" # s_p[0], |
| 526 | MangledName = NAME # "_" # s_p[0] in { |
| 527 | defvar suffix = s_p[1]; |
| 528 | defvar prototype = s_p[2]; |
| 529 | def : RVVOutOp1Builtin<suffix, prototype, type_range>; |
| 530 | } |
| 531 | } |
| 532 | } |
| 533 | } |
| 534 | |
| 535 | multiclass RVVSignedWidenBinBuiltinSet |
| 536 | : RVVWidenBuiltinSet<NAME, "csi", |
| 537 | [["vv", "w", "wvv"], |
| 538 | ["vx", "w", "wve"]]>; |
| 539 | |
| 540 | multiclass RVVSignedWidenOp0BinBuiltinSet |
| 541 | : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", |
| 542 | [["wv", "w", "wwv"], |
| 543 | ["wx", "w", "wwe"]]>; |
| 544 | |
| 545 | multiclass RVVUnsignedWidenBinBuiltinSet |
| 546 | : RVVWidenBuiltinSet<NAME, "csi", |
| 547 | [["vv", "Uw", "UwUvUv"], |
| 548 | ["vx", "Uw", "UwUvUe"]]>; |
| 549 | |
| 550 | multiclass RVVUnsignedWidenOp0BinBuiltinSet |
| 551 | : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi", |
| 552 | [["wv", "Uw", "UwUwUv"], |
| 553 | ["wx", "Uw", "UwUwUe"]]>; |
| 554 | |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 555 | multiclass RVVFloatingWidenBinBuiltinSet |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 556 | : RVVWidenBuiltinSet<NAME, "xf", |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 557 | [["vv", "w", "wvv"], |
| 558 | ["vf", "w", "wve"]]>; |
| 559 | |
| 560 | multiclass RVVFloatingWidenOp0BinBuiltinSet |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 561 | : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf", |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 562 | [["wv", "w", "wwv"], |
| 563 | ["wf", "w", "wwe"]]>; |
| 564 | |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 565 | defvar TypeList = ["c","s","i","l","x","f","d"]; |
Zakk Chen | 88c2d4c | 2021-03-17 20:25:32 -0700 | [diff] [blame] | 566 | defvar EEWList = [["8", "(Log2EEW:3)"], |
| 567 | ["16", "(Log2EEW:4)"], |
| 568 | ["32", "(Log2EEW:5)"], |
| 569 | ["64", "(Log2EEW:6)"]]; |
| 570 | |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 571 | class IsFloat<string type> { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 572 | bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d")); |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 573 | } |
| 574 | |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 575 | let HasNoMaskedOverloaded = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 576 | HasPolicy = false, |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 577 | ManualCodegen = [{ |
| 578 | IntrinsicTypes = {ResultType, Ops[1]->getType()}; |
| 579 | Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); |
| 580 | }], |
| 581 | ManualCodegenMask= [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 582 | // Move mask to right before vl. |
| 583 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 584 | IntrinsicTypes = {ResultType, Ops[3]->getType()}; |
| 585 | Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); |
| 586 | }] in { |
| 587 | class RVVVLEMaskBuiltin : RVVBuiltin<"m", "mPCUe", "c"> { |
Hsiangkai Wang | 80a6456 | 2021-10-05 14:20:36 +0800 | [diff] [blame] | 588 | let Name = "vlm_v"; |
| 589 | let IRName = "vlm"; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 590 | let HasMask = false; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 591 | } |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 592 | } |
| 593 | |
| 594 | let HasNoMaskedOverloaded = false, |
| 595 | ManualCodegen = [{ |
| 596 | IntrinsicTypes = {ResultType, Ops[1]->getType()}; |
| 597 | Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); |
| 598 | }], |
| 599 | ManualCodegenMask= [{ |
| 600 | // Move mask to right before vl. |
| 601 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
| 602 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
| 603 | IntrinsicTypes = {ResultType, Ops[3]->getType()}; |
| 604 | Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); |
| 605 | }] in { |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 606 | multiclass RVVVLEBuiltin<list<string> types> { |
| 607 | let Name = NAME # "_v", |
| 608 | IRName = "vle", |
| 609 | IRNameMask ="vle_mask" in { |
| 610 | foreach type = types in { |
| 611 | def : RVVBuiltin<"v", "vPCe", type>; |
| 612 | if !not(IsFloat<type>.val) then { |
| 613 | def : RVVBuiltin<"Uv", "UvPCUe", type>; |
Zakk Chen | 88c2d4c | 2021-03-17 20:25:32 -0700 | [diff] [blame] | 614 | } |
| 615 | } |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 616 | } |
Zakk Chen | 88c2d4c | 2021-03-17 20:25:32 -0700 | [diff] [blame] | 617 | } |
| 618 | } |
| 619 | |
Hsiangkai Wang | 471ae42 | 2021-04-10 07:02:08 +0800 | [diff] [blame] | 620 | multiclass RVVVLEFFBuiltin<list<string> types> { |
| 621 | let Name = NAME # "_v", |
| 622 | IRName = "vleff", |
| 623 | IRNameMask = "vleff_mask", |
| 624 | HasNoMaskedOverloaded = false, |
| 625 | ManualCodegen = [{ |
| 626 | { |
| 627 | IntrinsicTypes = {ResultType, Ops[2]->getType()}; |
| 628 | Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); |
| 629 | Value *NewVL = Ops[1]; |
| 630 | Ops.erase(Ops.begin() + 1); |
| 631 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 632 | llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); |
| 633 | llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); |
| 634 | // Store new_vl. |
| 635 | clang::CharUnits Align = |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 636 | CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); |
Hsiangkai Wang | 471ae42 | 2021-04-10 07:02:08 +0800 | [diff] [blame] | 637 | Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {1}), |
| 638 | Address(NewVL, Align)); |
| 639 | return V; |
| 640 | } |
| 641 | }], |
| 642 | ManualCodegenMask = [{ |
| 643 | { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 644 | // Move mask to right before vl. |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 645 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 646 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | 471ae42 | 2021-04-10 07:02:08 +0800 | [diff] [blame] | 647 | IntrinsicTypes = {ResultType, Ops[4]->getType()}; |
| 648 | Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); |
| 649 | Value *NewVL = Ops[2]; |
| 650 | Ops.erase(Ops.begin() + 2); |
| 651 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 652 | llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); |
| 653 | llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); |
| 654 | // Store new_vl. |
| 655 | clang::CharUnits Align = |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 656 | CGM.getNaturalPointeeTypeAlignment(E->getArg(3)->getType()); |
Hsiangkai Wang | 471ae42 | 2021-04-10 07:02:08 +0800 | [diff] [blame] | 657 | Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {1}), |
| 658 | Address(NewVL, Align)); |
| 659 | return V; |
| 660 | } |
| 661 | }] in { |
| 662 | foreach type = types in { |
| 663 | def : RVVBuiltin<"v", "vPCePz", type>; |
| 664 | // Skip floating types for unsigned versions. |
| 665 | if !not(IsFloat<type>.val) then { |
| 666 | def : RVVBuiltin<"Uv", "UvPCUePz", type>; |
| 667 | } |
| 668 | } |
| 669 | } |
| 670 | } |
| 671 | |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 672 | multiclass RVVVLSEBuiltin<list<string> types> { |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 673 | let Name = NAME # "_v", |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 674 | IRName = "vlse", |
| 675 | IRNameMask ="vlse_mask", |
| 676 | HasNoMaskedOverloaded = false, |
| 677 | ManualCodegen = [{ |
| 678 | IntrinsicTypes = {ResultType, Ops[2]->getType()}; |
| 679 | Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); |
| 680 | }], |
| 681 | ManualCodegenMask= [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 682 | // Move mask to right before vl. |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 683 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 684 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 685 | IntrinsicTypes = {ResultType, Ops[4]->getType()}; |
| 686 | Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); |
| 687 | }] in { |
| 688 | foreach type = types in { |
| 689 | def : RVVBuiltin<"v", "vPCet", type>; |
| 690 | if !not(IsFloat<type>.val) then { |
| 691 | def : RVVBuiltin<"Uv", "UvPCUet", type>; |
| 692 | } |
| 693 | } |
| 694 | } |
| 695 | } |
| 696 | |
| 697 | multiclass RVVIndexedLoad<string op> { |
| 698 | let ManualCodegen = [{ |
| 699 | IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType()}; |
| 700 | Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); |
| 701 | }], |
| 702 | ManualCodegenMask = [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 703 | // Move mask to right before vl. |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 704 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 705 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 706 | IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops[4]->getType()}; |
| 707 | Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); |
| 708 | }] in { |
| 709 | foreach type = TypeList in { |
| 710 | foreach eew_list = EEWList in { |
| 711 | defvar eew = eew_list[0]; |
| 712 | defvar eew_type = eew_list[1]; |
| 713 | let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in { |
| 714 | def: RVVBuiltin<"v", "vPCe" # eew_type # "Uv", type>; |
| 715 | if !not(IsFloat<type>.val) then { |
| 716 | def: RVVBuiltin<"Uv", "UvPCUe" # eew_type # "Uv", type>; |
| 717 | } |
| 718 | } |
| 719 | } |
| 720 | } |
| 721 | } |
| 722 | } |
| 723 | |
| 724 | let HasMaskedOffOperand = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 725 | HasPolicy = false, |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 726 | ManualCodegen = [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 727 | // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) |
| 728 | std::swap(Ops[0], Ops[1]); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 729 | Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); |
| 730 | IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; |
| 731 | }], |
| 732 | ManualCodegenMask= [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 733 | // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) |
| 734 | std::swap(Ops[0], Ops[2]); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 735 | Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); |
| 736 | IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; |
| 737 | }] in { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 738 | class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { |
Hsiangkai Wang | 80a6456 | 2021-10-05 14:20:36 +0800 | [diff] [blame] | 739 | let Name = "vsm_v"; |
| 740 | let IRName = "vsm"; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 741 | let HasMask = false; |
| 742 | } |
| 743 | multiclass RVVVSEBuiltin<list<string> types> { |
| 744 | let Name = NAME # "_v", |
| 745 | IRName = "vse", |
| 746 | IRNameMask = "vse_mask" in { |
| 747 | foreach type = types in { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 748 | def : RVVBuiltin<"v", "0Pev", type>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 749 | if !not(IsFloat<type>.val) then { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 750 | def : RVVBuiltin<"Uv", "0PUeUv", type>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 751 | } |
| 752 | } |
| 753 | } |
| 754 | } |
| 755 | } |
| 756 | |
| 757 | multiclass RVVVSSEBuiltin<list<string> types> { |
| 758 | let Name = NAME # "_v", |
| 759 | IRName = "vsse", |
| 760 | IRNameMask = "vsse_mask", |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 761 | HasMaskedOffOperand = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 762 | HasPolicy = false, |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 763 | ManualCodegen = [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 764 | // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) |
| 765 | std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 766 | Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 767 | IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 768 | }], |
| 769 | ManualCodegenMask= [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 770 | // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) |
| 771 | std::swap(Ops[0], Ops[3]); |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 772 | Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 773 | IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 774 | }] in { |
| 775 | foreach type = types in { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 776 | def : RVVBuiltin<"v", "0Petv", type>; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 777 | if !not(IsFloat<type>.val) then { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 778 | def : RVVBuiltin<"Uv", "0PUetUv", type>; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 779 | } |
| 780 | } |
| 781 | } |
| 782 | } |
Zakk Chen | 95c0125 | 2021-02-25 00:15:14 -0800 | [diff] [blame] | 783 | |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 784 | multiclass RVVIndexedStore<string op> { |
| 785 | let HasMaskedOffOperand = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 786 | HasPolicy = false, |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 787 | ManualCodegen = [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 788 | // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) |
| 789 | std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 790 | Ops[1] = Builder.CreateBitCast(Ops[1],Ops[0]->getType()->getPointerTo()); |
| 791 | IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; |
| 792 | }], |
| 793 | ManualCodegenMask= [{ |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 794 | // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) |
| 795 | std::swap(Ops[0], Ops[3]); |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 796 | Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); |
| 797 | IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; |
| 798 | }] in { |
| 799 | foreach type = TypeList in { |
| 800 | foreach eew_list = EEWList in { |
| 801 | defvar eew = eew_list[0]; |
| 802 | defvar eew_type = eew_list[1]; |
| 803 | let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 804 | def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 805 | if !not(IsFloat<type>.val) then { |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 806 | def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 807 | } |
| 808 | } |
| 809 | } |
| 810 | } |
| 811 | } |
| 812 | } |
| 813 | |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 814 | defvar NFList = [2, 3, 4, 5, 6, 7, 8]; |
| 815 | |
| 816 | class PVString<int nf, bit signed> { |
| 817 | string S = |
| 818 | !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), |
| 819 | !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), |
| 820 | !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), |
| 821 | !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), |
| 822 | !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), |
| 823 | !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), |
| 824 | !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); |
| 825 | } |
| 826 | |
| 827 | multiclass RVVUnitStridedSegLoad<string op> { |
| 828 | foreach type = TypeList in { |
| 829 | defvar eew = !cond(!eq(type, "c") : "8", |
| 830 | !eq(type, "s") : "16", |
| 831 | !eq(type, "i") : "32", |
| 832 | !eq(type, "l") : "64", |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 833 | !eq(type, "x") : "16", |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 834 | !eq(type, "f") : "32", |
| 835 | !eq(type, "d") : "64"); |
| 836 | foreach nf = NFList in { |
| 837 | let Name = op # nf # "e" # eew # "_v", |
| 838 | IRName = op # nf, |
| 839 | IRNameMask = op # nf # "_mask", |
| 840 | NF = nf, |
| 841 | HasNoMaskedOverloaded = false, |
| 842 | ManualCodegen = [{ |
| 843 | { |
| 844 | // builtin: (val0 address, val1 address, ..., ptr, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 845 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 846 | Ops[NF + 1]->getType()}; |
| 847 | // intrinsic: (ptr, vl) |
| 848 | llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1]}; |
| 849 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 850 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 851 | clang::CharUnits Align = |
| 852 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 853 | llvm::Value *V; |
| 854 | for (unsigned I = 0; I < NF; ++I) { |
| 855 | V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 856 | Address(Ops[I], Align)); |
| 857 | } |
| 858 | return V; |
| 859 | } |
| 860 | }], |
| 861 | ManualCodegenMask = [{ |
| 862 | { |
| 863 | // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl) |
| 864 | // intrinsic: (maskedoff0, ..., ptr, mask, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 865 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 866 | Ops[2 * NF + 2]->getType()}; |
| 867 | SmallVector<llvm::Value*, 12> Operands; |
| 868 | for (unsigned I = 0; I < NF; ++I) |
| 869 | Operands.push_back(Ops[NF + I + 1]); |
| 870 | Operands.push_back(Ops[2 * NF + 1]); |
| 871 | Operands.push_back(Ops[NF]); |
| 872 | Operands.push_back(Ops[2 * NF + 2]); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 873 | Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
| 874 | assert(Operands.size() == NF + 4); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 875 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 876 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 877 | clang::CharUnits Align = |
| 878 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 879 | llvm::Value *V; |
| 880 | for (unsigned I = 0; I < NF; ++I) { |
| 881 | V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 882 | Address(Ops[I], Align)); |
| 883 | } |
| 884 | return V; |
| 885 | } |
| 886 | }] in { |
| 887 | defvar PV = PVString<nf, /*signed=*/true>.S; |
| 888 | defvar PUV = PVString<nf, /*signed=*/false>.S; |
| 889 | def : RVVBuiltin<"v", "0" # PV # "PCe", type>; |
| 890 | if !not(IsFloat<type>.val) then { |
| 891 | def : RVVBuiltin<"Uv", "0" # PUV # "PCUe", type>; |
| 892 | } |
| 893 | } |
| 894 | } |
| 895 | } |
| 896 | } |
| 897 | |
| 898 | multiclass RVVUnitStridedSegLoadFF<string op> { |
| 899 | foreach type = TypeList in { |
| 900 | defvar eew = !cond(!eq(type, "c") : "8", |
| 901 | !eq(type, "s") : "16", |
| 902 | !eq(type, "i") : "32", |
| 903 | !eq(type, "l") : "64", |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 904 | !eq(type, "x") : "16", |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 905 | !eq(type, "f") : "32", |
| 906 | !eq(type, "d") : "64"); |
| 907 | foreach nf = NFList in { |
| 908 | let Name = op # nf # "e" # eew # "ff_v", |
| 909 | IRName = op # nf # "ff", |
| 910 | IRNameMask = op # nf # "ff_mask", |
| 911 | NF = nf, |
| 912 | HasNoMaskedOverloaded = false, |
| 913 | ManualCodegen = [{ |
| 914 | { |
| 915 | // builtin: (val0 address, val1 address, ..., ptr, new_vl, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 916 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 917 | Ops[NF + 2]->getType()}; |
| 918 | // intrinsic: (ptr, vl) |
| 919 | llvm::Value *Operands[] = {Ops[NF], Ops[NF + 2]}; |
| 920 | Value *NewVL = Ops[NF + 1]; |
| 921 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 922 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 923 | clang::CharUnits Align = |
| 924 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 925 | for (unsigned I = 0; I < NF; ++I) { |
| 926 | Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 927 | Address(Ops[I], Align)); |
| 928 | } |
| 929 | // Store new_vl. |
| 930 | return Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {NF}), |
| 931 | Address(NewVL, Align)); |
| 932 | } |
| 933 | }], |
| 934 | ManualCodegenMask = [{ |
| 935 | { |
| 936 | // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl) |
| 937 | // intrinsic: (maskedoff0, ..., ptr, mask, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 938 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 939 | Ops[2 * NF + 3]->getType()}; |
| 940 | SmallVector<llvm::Value*, 12> Operands; |
| 941 | for (unsigned I = 0; I < NF; ++I) |
| 942 | Operands.push_back(Ops[NF + I + 1]); |
| 943 | Operands.push_back(Ops[2 * NF + 1]); |
| 944 | Operands.push_back(Ops[NF]); |
| 945 | Operands.push_back(Ops[2 * NF + 3]); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 946 | Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 947 | Value *NewVL = Ops[2 * NF + 2]; |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 948 | assert(Operands.size() == NF + 4); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 949 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 950 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 951 | clang::CharUnits Align = |
| 952 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 953 | for (unsigned I = 0; I < NF; ++I) { |
| 954 | Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 955 | Address(Ops[I], Align)); |
| 956 | } |
| 957 | // Store new_vl. |
| 958 | return Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {NF}), |
| 959 | Address(NewVL, Align)); |
| 960 | } |
| 961 | }] in { |
| 962 | defvar PV = PVString<nf, /*signed=*/true>.S; |
| 963 | defvar PUV = PVString<nf, /*signed=*/false>.S; |
| 964 | def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>; |
| 965 | if !not(IsFloat<type>.val) then { |
| 966 | def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "Pz", type>; |
| 967 | } |
| 968 | } |
| 969 | } |
| 970 | } |
| 971 | } |
| 972 | |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 973 | multiclass RVVStridedSegLoad<string op> { |
| 974 | foreach type = TypeList in { |
| 975 | defvar eew = !cond(!eq(type, "c") : "8", |
| 976 | !eq(type, "s") : "16", |
| 977 | !eq(type, "i") : "32", |
| 978 | !eq(type, "l") : "64", |
| 979 | !eq(type, "x") : "16", |
| 980 | !eq(type, "f") : "32", |
| 981 | !eq(type, "d") : "64"); |
| 982 | foreach nf = NFList in { |
| 983 | let Name = op # nf # "e" # eew # "_v", |
| 984 | IRName = op # nf, |
| 985 | IRNameMask = op # nf # "_mask", |
| 986 | NF = nf, |
| 987 | HasNoMaskedOverloaded = false, |
| 988 | ManualCodegen = [{ |
| 989 | { |
| 990 | // builtin: (val0 address, val1 address, ..., ptr, stride, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 991 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 992 | Ops[NF + 2]->getType()}; |
| 993 | // intrinsic: (ptr, stride, vl) |
| 994 | llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1], Ops[NF + 2]}; |
| 995 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 996 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 997 | clang::CharUnits Align = |
| 998 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 999 | llvm::Value *V; |
| 1000 | for (unsigned I = 0; I < NF; ++I) { |
| 1001 | V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 1002 | Address(Ops[I], Align)); |
| 1003 | } |
| 1004 | return V; |
| 1005 | } |
| 1006 | }], |
| 1007 | ManualCodegenMask = [{ |
| 1008 | { |
| 1009 | // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl) |
| 1010 | // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 1011 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 1012 | Ops[2 * NF + 3]->getType()}; |
| 1013 | SmallVector<llvm::Value*, 12> Operands; |
| 1014 | for (unsigned I = 0; I < NF; ++I) |
| 1015 | Operands.push_back(Ops[NF + I + 1]); |
| 1016 | Operands.push_back(Ops[2 * NF + 1]); |
| 1017 | Operands.push_back(Ops[2 * NF + 2]); |
| 1018 | Operands.push_back(Ops[NF]); |
| 1019 | Operands.push_back(Ops[2 * NF + 3]); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1020 | Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
| 1021 | assert(Operands.size() == NF + 5); |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 1022 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 1023 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 1024 | clang::CharUnits Align = |
| 1025 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 1026 | llvm::Value *V; |
| 1027 | for (unsigned I = 0; I < NF; ++I) { |
| 1028 | V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 1029 | Address(Ops[I], Align)); |
| 1030 | } |
| 1031 | return V; |
| 1032 | } |
| 1033 | }] in { |
| 1034 | defvar PV = PVString<nf, /*signed=*/true>.S; |
| 1035 | defvar PUV = PVString<nf, /*signed=*/false>.S; |
| 1036 | def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>; |
| 1037 | if !not(IsFloat<type>.val) then { |
| 1038 | def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "t", type>; |
| 1039 | } |
| 1040 | } |
| 1041 | } |
| 1042 | } |
| 1043 | } |
| 1044 | |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1045 | multiclass RVVIndexedSegLoad<string op> { |
| 1046 | foreach type = TypeList in { |
| 1047 | foreach eew_info = EEWList in { |
| 1048 | defvar eew = eew_info[0]; |
| 1049 | defvar eew_type = eew_info[1]; |
| 1050 | foreach nf = NFList in { |
| 1051 | let Name = op # nf # "ei" # eew # "_v", |
| 1052 | IRName = op # nf, |
| 1053 | IRNameMask = op # nf # "_mask", |
| 1054 | NF = nf, |
| 1055 | ManualCodegen = [{ |
| 1056 | { |
| 1057 | // builtin: (val0 address, val1 address, ..., ptr, index, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 1058 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1059 | Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; |
| 1060 | // intrinsic: (ptr, index, vl) |
| 1061 | llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1], Ops[NF + 2]}; |
| 1062 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 1063 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 1064 | clang::CharUnits Align = |
| 1065 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1066 | llvm::Value *V; |
| 1067 | for (unsigned I = 0; I < NF; ++I) { |
| 1068 | V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 1069 | Address(Ops[I], Align)); |
| 1070 | } |
| 1071 | return V; |
| 1072 | } |
| 1073 | }], |
| 1074 | ManualCodegenMask = [{ |
| 1075 | { |
| 1076 | // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl) |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 1077 | IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()), |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1078 | Ops[2 * NF + 2]->getType(), Ops[2 * NF + 3]->getType()}; |
| 1079 | // intrinsic: (maskedoff0, ..., ptr, index, mask, vl) |
| 1080 | SmallVector<llvm::Value*, 12> Operands; |
| 1081 | for (unsigned I = 0; I < NF; ++I) |
| 1082 | Operands.push_back(Ops[NF + I + 1]); |
| 1083 | Operands.push_back(Ops[2 * NF + 1]); |
| 1084 | Operands.push_back(Ops[2 * NF + 2]); |
| 1085 | Operands.push_back(Ops[NF]); |
| 1086 | Operands.push_back(Ops[2 * NF + 3]); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1087 | Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
| 1088 | assert(Operands.size() == NF + 5); |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1089 | llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); |
| 1090 | llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); |
Craig Topper | 6171f84 | 2021-08-12 08:52:20 -0700 | [diff] [blame] | 1091 | clang::CharUnits Align = |
| 1092 | CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1093 | llvm::Value *V; |
| 1094 | for (unsigned I = 0; I < NF; ++I) { |
| 1095 | V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}), |
| 1096 | Address(Ops[I], Align)); |
| 1097 | } |
| 1098 | return V; |
| 1099 | } |
| 1100 | }] in { |
| 1101 | defvar PV = PVString<nf, /*signed=*/true>.S; |
| 1102 | defvar PUV = PVString<nf, /*signed=*/false>.S; |
| 1103 | def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>; |
| 1104 | if !not(IsFloat<type>.val) then { |
| 1105 | def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # eew_type # "Uv", type>; |
| 1106 | } |
| 1107 | } |
| 1108 | } |
| 1109 | } |
| 1110 | } |
| 1111 | } |
| 1112 | |
Hsiangkai Wang | d1a401b | 2021-06-08 13:09:07 +0800 | [diff] [blame] | 1113 | class VString<int nf, bit signed> { |
| 1114 | string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), |
| 1115 | !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), |
| 1116 | !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), |
| 1117 | !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), |
| 1118 | !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), |
| 1119 | !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), |
| 1120 | !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); |
| 1121 | } |
| 1122 | |
| 1123 | multiclass RVVUnitStridedSegStore<string op> { |
| 1124 | foreach type = TypeList in { |
| 1125 | defvar eew = !cond(!eq(type, "c") : "8", |
| 1126 | !eq(type, "s") : "16", |
| 1127 | !eq(type, "i") : "32", |
| 1128 | !eq(type, "l") : "64", |
| 1129 | !eq(type, "x") : "16", |
| 1130 | !eq(type, "f") : "32", |
| 1131 | !eq(type, "d") : "64"); |
| 1132 | foreach nf = NFList in { |
| 1133 | let Name = op # nf # "e" # eew # "_v", |
| 1134 | IRName = op # nf, |
| 1135 | IRNameMask = op # nf # "_mask", |
| 1136 | NF = nf, |
| 1137 | HasMaskedOffOperand = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1138 | HasPolicy = false, |
Hsiangkai Wang | d1a401b | 2021-06-08 13:09:07 +0800 | [diff] [blame] | 1139 | ManualCodegen = [{ |
| 1140 | { |
| 1141 | // Builtin: (ptr, val0, val1, ..., vl) |
| 1142 | // Intrinsic: (val0, val1, ..., ptr, vl) |
| 1143 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
| 1144 | IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; |
| 1145 | assert(Ops.size() == NF + 2); |
| 1146 | } |
| 1147 | }], |
| 1148 | ManualCodegenMask = [{ |
| 1149 | { |
| 1150 | // Builtin: (mask, ptr, val0, val1, ..., vl) |
| 1151 | // Intrinsic: (val0, val1, ..., ptr, mask, vl) |
| 1152 | std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); |
| 1153 | std::swap(Ops[NF], Ops[NF + 1]); |
| 1154 | IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()}; |
| 1155 | assert(Ops.size() == NF + 3); |
| 1156 | } |
| 1157 | }] in { |
| 1158 | defvar V = VString<nf, /*signed=*/true>.S; |
| 1159 | defvar UV = VString<nf, /*signed=*/false>.S; |
| 1160 | def : RVVBuiltin<"v", "0Pe" # V, type>; |
| 1161 | if !not(IsFloat<type>.val) then { |
| 1162 | def : RVVBuiltin<"Uv", "0PUe" # UV, type>; |
| 1163 | } |
| 1164 | } |
| 1165 | } |
| 1166 | } |
| 1167 | } |
| 1168 | |
Hsiangkai Wang | 915e6dc | 2021-06-08 13:29:51 +0800 | [diff] [blame] | 1169 | multiclass RVVStridedSegStore<string op> { |
| 1170 | foreach type = TypeList in { |
| 1171 | defvar eew = !cond(!eq(type, "c") : "8", |
| 1172 | !eq(type, "s") : "16", |
| 1173 | !eq(type, "i") : "32", |
| 1174 | !eq(type, "l") : "64", |
| 1175 | !eq(type, "x") : "16", |
| 1176 | !eq(type, "f") : "32", |
| 1177 | !eq(type, "d") : "64"); |
| 1178 | foreach nf = NFList in { |
| 1179 | let Name = op # nf # "e" # eew # "_v", |
| 1180 | IRName = op # nf, |
| 1181 | IRNameMask = op # nf # "_mask", |
| 1182 | NF = nf, |
| 1183 | HasMaskedOffOperand = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1184 | HasPolicy = false, |
Hsiangkai Wang | 915e6dc | 2021-06-08 13:29:51 +0800 | [diff] [blame] | 1185 | ManualCodegen = [{ |
| 1186 | { |
| 1187 | // Builtin: (ptr, stride, val0, val1, ..., vl). |
| 1188 | // Intrinsic: (val0, val1, ..., ptr, stride, vl) |
| 1189 | std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); |
| 1190 | IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; |
| 1191 | assert(Ops.size() == NF + 3); |
| 1192 | } |
| 1193 | }], |
| 1194 | ManualCodegenMask = [{ |
| 1195 | { |
| 1196 | // Builtin: (mask, ptr, stride, val0, val1, ..., vl). |
| 1197 | // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) |
| 1198 | std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); |
| 1199 | std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); |
| 1200 | IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; |
| 1201 | assert(Ops.size() == NF + 4); |
| 1202 | } |
| 1203 | }] in { |
| 1204 | defvar V = VString<nf, /*signed=*/true>.S; |
| 1205 | defvar UV = VString<nf, /*signed=*/false>.S; |
| 1206 | def : RVVBuiltin<"v", "0Pet" # V, type>; |
| 1207 | if !not(IsFloat<type>.val) then { |
| 1208 | def : RVVBuiltin<"Uv", "0PUet" # UV, type>; |
| 1209 | } |
| 1210 | } |
| 1211 | } |
| 1212 | } |
| 1213 | } |
| 1214 | |
Hsiangkai Wang | 698f288 | 2021-06-08 14:29:40 +0800 | [diff] [blame] | 1215 | multiclass RVVIndexedSegStore<string op> { |
| 1216 | foreach type = TypeList in { |
| 1217 | foreach eew_info = EEWList in { |
| 1218 | defvar eew = eew_info[0]; |
| 1219 | defvar eew_type = eew_info[1]; |
| 1220 | foreach nf = NFList in { |
| 1221 | let Name = op # nf # "ei" # eew # "_v", |
| 1222 | IRName = op # nf, |
| 1223 | IRNameMask = op # nf # "_mask", |
| 1224 | NF = nf, |
| 1225 | HasMaskedOffOperand = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1226 | HasPolicy = false, |
Hsiangkai Wang | 698f288 | 2021-06-08 14:29:40 +0800 | [diff] [blame] | 1227 | ManualCodegen = [{ |
| 1228 | { |
| 1229 | // Builtin: (ptr, index, val0, val1, ..., vl) |
| 1230 | // Intrinsic: (val0, val1, ..., ptr, index, vl) |
| 1231 | std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); |
| 1232 | IntrinsicTypes = {Ops[0]->getType(), |
| 1233 | Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; |
| 1234 | assert(Ops.size() == NF + 3); |
| 1235 | } |
| 1236 | }], |
| 1237 | ManualCodegenMask = [{ |
| 1238 | { |
| 1239 | // Builtin: (mask, ptr, index, val0, val1, ..., vl) |
| 1240 | // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) |
| 1241 | std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); |
| 1242 | std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); |
| 1243 | IntrinsicTypes = {Ops[0]->getType(), |
| 1244 | Ops[NF + 1]->getType(), Ops[NF + 3]->getType()}; |
| 1245 | assert(Ops.size() == NF + 4); |
| 1246 | } |
| 1247 | }] in { |
| 1248 | defvar V = VString<nf, /*signed=*/true>.S; |
| 1249 | defvar UV = VString<nf, /*signed=*/false>.S; |
| 1250 | def : RVVBuiltin<"v", "0Pe" # eew_type # "Uv" # V, type>; |
| 1251 | if !not(IsFloat<type>.val) then { |
| 1252 | def : RVVBuiltin<"Uv", "0PUe" # eew_type # "Uv" # UV, type>; |
| 1253 | } |
| 1254 | } |
| 1255 | } |
| 1256 | } |
| 1257 | } |
| 1258 | } |
| 1259 | |
Hsiangkai Wang | 14cc1cb2 | 2021-04-20 10:53:19 +0800 | [diff] [blame] | 1260 | multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { |
| 1261 | let Name = NAME, |
| 1262 | IRName = IR, |
| 1263 | IRNameMask = IR # "_mask", |
| 1264 | ManualCodegen = [{ |
| 1265 | { |
| 1266 | // op1, vl |
| 1267 | IntrinsicTypes = {ResultType, |
| 1268 | cast<llvm::VectorType>(ResultType)->getElementType(), |
| 1269 | Ops[1]->getType()}; |
| 1270 | Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[1])); |
| 1271 | break; |
| 1272 | } |
| 1273 | }], |
| 1274 | ManualCodegenMask = [{ |
| 1275 | { |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 1276 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1277 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | 14cc1cb2 | 2021-04-20 10:53:19 +0800 | [diff] [blame] | 1278 | // maskedoff, op1, mask, vl |
| 1279 | IntrinsicTypes = {ResultType, |
| 1280 | cast<llvm::VectorType>(ResultType)->getElementType(), |
| 1281 | Ops[3]->getType()}; |
| 1282 | Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[1])); |
| 1283 | break; |
| 1284 | } |
| 1285 | }] in { |
| 1286 | def : RVVBuiltin<"v", "vv", type_range>; |
| 1287 | } |
| 1288 | } |
| 1289 | |
Hsiangkai Wang | 43cd588 | 2021-04-20 11:01:22 +0800 | [diff] [blame] | 1290 | multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { |
| 1291 | let Name = NAME, |
| 1292 | IRName = IR, |
| 1293 | IRNameMask = IR # "_mask", |
| 1294 | ManualCodegen = [{ |
| 1295 | { |
| 1296 | // op1, vl |
| 1297 | IntrinsicTypes = {ResultType, |
| 1298 | cast<llvm::VectorType>(ResultType)->getElementType(), |
| 1299 | Ops[1]->getType()}; |
| 1300 | Ops.insert(Ops.begin() + 1, |
| 1301 | llvm::Constant::getAllOnesValue(IntrinsicTypes[1])); |
| 1302 | break; |
| 1303 | } |
| 1304 | }], |
| 1305 | ManualCodegenMask = [{ |
| 1306 | { |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 1307 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1308 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | 43cd588 | 2021-04-20 11:01:22 +0800 | [diff] [blame] | 1309 | // maskedoff, op1, mask, vl |
| 1310 | IntrinsicTypes = {ResultType, |
| 1311 | cast<llvm::VectorType>(ResultType)->getElementType(), |
| 1312 | Ops[3]->getType()}; |
| 1313 | Ops.insert(Ops.begin() + 2, |
| 1314 | llvm::Constant::getAllOnesValue(IntrinsicTypes[1])); |
| 1315 | break; |
| 1316 | } |
| 1317 | }] in { |
| 1318 | def : RVVBuiltin<"v", "vv", type_range>; |
| 1319 | def : RVVBuiltin<"Uv", "UvUv", type_range>; |
| 1320 | } |
| 1321 | } |
| 1322 | |
Hsiangkai Wang | 4b24341 | 2021-04-20 11:12:53 +0800 | [diff] [blame] | 1323 | multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { |
| 1324 | let Name = NAME, |
| 1325 | IRName = IR, |
| 1326 | HasMask = false, |
| 1327 | ManualCodegen = [{ |
| 1328 | { |
| 1329 | // op1, vl |
| 1330 | IntrinsicTypes = {ResultType, |
| 1331 | Ops[1]->getType()}; |
| 1332 | Ops.insert(Ops.begin() + 1, Ops[0]); |
| 1333 | break; |
| 1334 | } |
| 1335 | }] in { |
| 1336 | def : RVVBuiltin<"m", "mm", type_range>; |
| 1337 | } |
| 1338 | } |
| 1339 | |
Hsiangkai Wang | bfb3fca | 2021-04-20 13:17:40 +0800 | [diff] [blame] | 1340 | multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { |
| 1341 | let Name = NAME, |
| 1342 | IRName = IR, |
| 1343 | IRNameMask = IR # "_mask", |
| 1344 | ManualCodegen = [{ |
| 1345 | { |
| 1346 | // op1, vl |
| 1347 | IntrinsicTypes = {ResultType, |
| 1348 | Ops[0]->getType(), Ops[1]->getType()}; |
| 1349 | Ops.insert(Ops.begin() + 1, Ops[0]); |
| 1350 | break; |
| 1351 | } |
| 1352 | }], |
| 1353 | ManualCodegenMask = [{ |
| 1354 | { |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 1355 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1356 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | bfb3fca | 2021-04-20 13:17:40 +0800 | [diff] [blame] | 1357 | // maskedoff, op1, mask, vl |
| 1358 | IntrinsicTypes = {ResultType, |
| 1359 | Ops[1]->getType(), |
| 1360 | Ops[3]->getType()}; |
| 1361 | Ops.insert(Ops.begin() + 2, Ops[1]); |
| 1362 | break; |
| 1363 | } |
| 1364 | }] in { |
| 1365 | def : RVVBuiltin<"v", "vv", type_range>; |
| 1366 | } |
| 1367 | } |
| 1368 | |
Hsiangkai Wang | bd32c2d | 2021-04-20 15:31:13 +0800 | [diff] [blame] | 1369 | multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, |
| 1370 | list<list<string>> suffixes_prototypes> { |
| 1371 | let Name = NAME, |
| 1372 | MangledName = MName, |
| 1373 | IRName = IR, |
| 1374 | IRNameMask = IR # "_mask", |
| 1375 | ManualCodegen = [{ |
| 1376 | { |
| 1377 | // op1, vl |
| 1378 | IntrinsicTypes = {ResultType, |
| 1379 | Ops[0]->getType(), |
| 1380 | cast<llvm::VectorType>(Ops[0]->getType())->getElementType(), |
| 1381 | Ops[1]->getType()}; |
| 1382 | Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2])); |
| 1383 | break; |
| 1384 | } |
| 1385 | }], |
| 1386 | ManualCodegenMask = [{ |
| 1387 | { |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 1388 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1389 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | bd32c2d | 2021-04-20 15:31:13 +0800 | [diff] [blame] | 1390 | // maskedoff, op1, mask, vl |
| 1391 | IntrinsicTypes = {ResultType, |
| 1392 | Ops[1]->getType(), |
| 1393 | cast<llvm::VectorType>(Ops[1]->getType())->getElementType(), |
| 1394 | Ops[3]->getType()}; |
| 1395 | Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2])); |
| 1396 | break; |
| 1397 | } |
| 1398 | }] in { |
| 1399 | foreach s_p = suffixes_prototypes in { |
| 1400 | def : RVVBuiltin<s_p[0], s_p[1], type_range>; |
| 1401 | } |
| 1402 | } |
| 1403 | } |
| 1404 | |
| 1405 | multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, |
| 1406 | list<list<string>> suffixes_prototypes> { |
| 1407 | let Name = NAME, |
| 1408 | MangledName = MName, |
| 1409 | IRName = IR, |
| 1410 | IRNameMask = IR # "_mask", |
| 1411 | ManualCodegen = [{ |
| 1412 | { |
| 1413 | // op1, vl |
| 1414 | IntrinsicTypes = {ResultType, |
| 1415 | Ops[0]->getType(), |
| 1416 | Ops[1]->getType(), |
| 1417 | Ops[1]->getType()}; |
| 1418 | Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2])); |
| 1419 | break; |
| 1420 | } |
| 1421 | }], |
| 1422 | ManualCodegenMask = [{ |
| 1423 | { |
Hsiangkai Wang | 7ccd31c | 2021-10-13 14:50:45 +0800 | [diff] [blame] | 1424 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1425 | Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED)); |
Hsiangkai Wang | bd32c2d | 2021-04-20 15:31:13 +0800 | [diff] [blame] | 1426 | // maskedoff, op1, mask, vl |
| 1427 | IntrinsicTypes = {ResultType, |
| 1428 | Ops[1]->getType(), |
| 1429 | Ops[3]->getType(), |
| 1430 | Ops[3]->getType()}; |
| 1431 | Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2])); |
| 1432 | break; |
| 1433 | } |
| 1434 | }] in { |
| 1435 | foreach s_p = suffixes_prototypes in { |
| 1436 | def : RVVBuiltin<s_p[0], s_p[1], type_range>; |
| 1437 | } |
| 1438 | } |
| 1439 | } |
| 1440 | |
Zakk Chen | 95c0125 | 2021-02-25 00:15:14 -0800 | [diff] [blame] | 1441 | // 6. Configuration-Setting Instructions |
| 1442 | // 6.1. vsetvli/vsetvl instructions |
| 1443 | let HasVL = false, |
| 1444 | HasMask = false, |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1445 | HasPolicy = false, |
Zakk Chen | 95c0125 | 2021-02-25 00:15:14 -0800 | [diff] [blame] | 1446 | Log2LMUL = [0], |
| 1447 | ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type |
| 1448 | { |
| 1449 | // vsetvl is a macro because for it require constant integers in SEW and LMUL. |
| 1450 | let HeaderCode = |
| 1451 | [{ |
| 1452 | #define vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) |
| 1453 | #define vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) |
| 1454 | #define vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) |
| 1455 | #define vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) |
| 1456 | #define vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) |
| 1457 | #define vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) |
| 1458 | #define vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) |
| 1459 | |
| 1460 | #define vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) |
| 1461 | #define vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) |
| 1462 | #define vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) |
| 1463 | #define vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) |
| 1464 | #define vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) |
| 1465 | #define vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) |
| 1466 | |
| 1467 | #define vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) |
| 1468 | #define vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) |
| 1469 | #define vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) |
| 1470 | #define vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) |
| 1471 | #define vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) |
| 1472 | |
| 1473 | #define vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) |
| 1474 | #define vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) |
| 1475 | #define vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) |
| 1476 | #define vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) |
| 1477 | |
| 1478 | }] in |
| 1479 | def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; |
| 1480 | |
| 1481 | let HeaderCode = |
| 1482 | [{ |
| 1483 | #define vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) |
| 1484 | #define vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) |
| 1485 | #define vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) |
| 1486 | #define vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) |
| 1487 | #define vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) |
| 1488 | #define vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) |
| 1489 | #define vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) |
| 1490 | |
| 1491 | #define vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) |
| 1492 | #define vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) |
| 1493 | #define vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) |
| 1494 | #define vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) |
| 1495 | #define vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) |
| 1496 | #define vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) |
| 1497 | |
| 1498 | #define vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) |
| 1499 | #define vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) |
| 1500 | #define vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) |
| 1501 | #define vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) |
| 1502 | #define vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) |
| 1503 | |
| 1504 | #define vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) |
| 1505 | #define vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) |
| 1506 | #define vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) |
| 1507 | #define vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) |
| 1508 | |
| 1509 | }] in |
| 1510 | def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; |
| 1511 | } |
| 1512 | |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 1513 | // 7. Vector Loads and Stores |
| 1514 | // 7.4. Vector Unit-Stride Instructions |
Hsiangkai Wang | 80a6456 | 2021-10-05 14:20:36 +0800 | [diff] [blame] | 1515 | def vlm: RVVVLEMaskBuiltin; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 1516 | defm vle8: RVVVLEBuiltin<["c"]>; |
Hsiangkai Wang | 89ce6449 | 2021-07-20 11:36:48 +0800 | [diff] [blame] | 1517 | defm vle16: RVVVLEBuiltin<["s","x"]>; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 1518 | defm vle32: RVVVLEBuiltin<["i","f"]>; |
| 1519 | defm vle64: RVVVLEBuiltin<["l","d"]>; |
| 1520 | |
Hsiangkai Wang | 80a6456 | 2021-10-05 14:20:36 +0800 | [diff] [blame] | 1521 | def vsm : RVVVSEMaskBuiltin; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 1522 | defm vse8 : RVVVSEBuiltin<["c"]>; |
Hsiangkai Wang | 89ce6449 | 2021-07-20 11:36:48 +0800 | [diff] [blame] | 1523 | defm vse16: RVVVSEBuiltin<["s","x"]>; |
Zakk Chen | be947ad | 2021-03-17 07:56:55 -0700 | [diff] [blame] | 1524 | defm vse32: RVVVSEBuiltin<["i","f"]>; |
| 1525 | defm vse64: RVVVSEBuiltin<["l","d"]>; |
| 1526 | |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 1527 | // 7.5. Vector Strided Instructions |
| 1528 | defm vlse8: RVVVLSEBuiltin<["c"]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1529 | defm vlse16: RVVVLSEBuiltin<["s","x"]>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 1530 | defm vlse32: RVVVLSEBuiltin<["i","f"]>; |
| 1531 | defm vlse64: RVVVLSEBuiltin<["l","d"]>; |
| 1532 | |
| 1533 | defm vsse8 : RVVVSSEBuiltin<["c"]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1534 | defm vsse16: RVVVSSEBuiltin<["s","x"]>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 1535 | defm vsse32: RVVVSSEBuiltin<["i","f"]>; |
| 1536 | defm vsse64: RVVVSSEBuiltin<["l","d"]>; |
| 1537 | |
Zakk Chen | 88c2d4c | 2021-03-17 20:25:32 -0700 | [diff] [blame] | 1538 | // 7.6. Vector Indexed Instructions |
| 1539 | defm : RVVIndexedLoad<"vluxei">; |
| 1540 | defm : RVVIndexedLoad<"vloxei">; |
| 1541 | |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 1542 | defm : RVVIndexedStore<"vsuxei">; |
| 1543 | defm : RVVIndexedStore<"vsoxei">; |
| 1544 | |
| 1545 | // 7.7. Unit-stride Fault-Only-First Loads |
| 1546 | defm vle8ff: RVVVLEFFBuiltin<["c"]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1547 | defm vle16ff: RVVVLEFFBuiltin<["s","x"]>; |
Zakk Chen | e5a8219 | 2021-04-11 07:25:06 -0700 | [diff] [blame] | 1548 | defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; |
| 1549 | defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; |
| 1550 | |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 1551 | // 7.8 Vector Load/Store Segment Instructions |
| 1552 | let RequiredExtension = "Zvlsseg" in { |
| 1553 | defm : RVVUnitStridedSegLoad<"vlseg">; |
| 1554 | defm : RVVUnitStridedSegLoadFF<"vlseg">; |
Hsiangkai Wang | a9de8f7 | 2021-06-07 17:54:00 +0800 | [diff] [blame] | 1555 | defm : RVVStridedSegLoad<"vlsseg">; |
Hsiangkai Wang | 1c55033 | 2021-06-07 21:53:37 +0800 | [diff] [blame] | 1556 | defm : RVVIndexedSegLoad<"vluxseg">; |
| 1557 | defm : RVVIndexedSegLoad<"vloxseg">; |
Hsiangkai Wang | d1a401b | 2021-06-08 13:09:07 +0800 | [diff] [blame] | 1558 | defm : RVVUnitStridedSegStore<"vsseg">; |
Hsiangkai Wang | 915e6dc | 2021-06-08 13:29:51 +0800 | [diff] [blame] | 1559 | defm : RVVStridedSegStore<"vssseg">; |
Hsiangkai Wang | 698f288 | 2021-06-08 14:29:40 +0800 | [diff] [blame] | 1560 | defm : RVVIndexedSegStore<"vsuxseg">; |
| 1561 | defm : RVVIndexedSegStore<"vsoxseg">; |
Hsiangkai Wang | 593bf9b | 2021-05-25 16:13:34 +0800 | [diff] [blame] | 1562 | } |
| 1563 | |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 1564 | // 12. Vector Integer Arithmetic Instructions |
| 1565 | // 12.1. Vector Single-Width Integer Add and Subtract |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1566 | defm vadd : RVVIntBinBuiltinSet; |
| 1567 | defm vsub : RVVIntBinBuiltinSet; |
| 1568 | defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", |
| 1569 | [["vx", "v", "vve"], |
| 1570 | ["vx", "Uv", "UvUvUe"]]>; |
Hsiangkai Wang | 14cc1cb2 | 2021-04-20 10:53:19 +0800 | [diff] [blame] | 1571 | defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1572 | |
| 1573 | // 12.2. Vector Widening Integer Add/Subtract |
Zakk Chen | ccc624b | 2021-04-06 03:26:44 -0700 | [diff] [blame] | 1574 | // Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW |
| 1575 | defm vwaddu : RVVUnsignedWidenBinBuiltinSet; |
| 1576 | defm vwsubu : RVVUnsignedWidenBinBuiltinSet; |
| 1577 | // Widening signed integer add/subtract, 2*SEW = SEW +/- SEW |
| 1578 | defm vwadd : RVVSignedWidenBinBuiltinSet; |
| 1579 | defm vwsub : RVVSignedWidenBinBuiltinSet; |
| 1580 | // Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW |
| 1581 | defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; |
| 1582 | defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; |
| 1583 | // Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW |
| 1584 | defm vwadd : RVVSignedWidenOp0BinBuiltinSet; |
| 1585 | defm vwsub : RVVSignedWidenOp0BinBuiltinSet; |
Hsiangkai Wang | bd32c2d | 2021-04-20 15:31:13 +0800 | [diff] [blame] | 1586 | defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", |
| 1587 | [["Uw", "UwUv"]]>; |
| 1588 | defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", |
| 1589 | [["w", "wv"]]>; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1590 | |
| 1591 | // 12.3. Vector Integer Extension |
Zakk Chen | f720c22 | 2021-03-30 08:55:46 -0700 | [diff] [blame] | 1592 | let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 1593 | def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; |
| 1594 | def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; |
Zakk Chen | f720c22 | 2021-03-30 08:55:46 -0700 | [diff] [blame] | 1595 | } |
| 1596 | let Log2LMUL = [-3, -2, -1, 0, 1] in { |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 1597 | def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; |
| 1598 | def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; |
Zakk Chen | f720c22 | 2021-03-30 08:55:46 -0700 | [diff] [blame] | 1599 | } |
| 1600 | let Log2LMUL = [-3, -2, -1, 0] in { |
Craig Topper | 5f6b3d1 | 2021-04-07 17:33:20 -0700 | [diff] [blame] | 1601 | def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; |
| 1602 | def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; |
Zakk Chen | f720c22 | 2021-03-30 08:55:46 -0700 | [diff] [blame] | 1603 | } |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1604 | |
| 1605 | // 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1606 | let HasMask = false, HasPolicy = false in { |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 1607 | defm vadc : RVVCarryinBuiltinSet; |
| 1608 | defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; |
| 1609 | defm vmadc : RVVIntMaskOutBuiltinSet; |
| 1610 | defm vsbc : RVVCarryinBuiltinSet; |
| 1611 | defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; |
| 1612 | defm vmsbc : RVVIntMaskOutBuiltinSet; |
| 1613 | } |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1614 | |
| 1615 | // 12.5. Vector Bitwise Logical Instructions |
| 1616 | defm vand : RVVIntBinBuiltinSet; |
| 1617 | defm vxor : RVVIntBinBuiltinSet; |
| 1618 | defm vor : RVVIntBinBuiltinSet; |
Hsiangkai Wang | 43cd588 | 2021-04-20 11:01:22 +0800 | [diff] [blame] | 1619 | defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1620 | |
| 1621 | // 12.6. Vector Single-Width Bit Shift Instructions |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 1622 | defm vsll : RVVShiftBuiltinSet; |
| 1623 | defm vsrl : RVVUnsignedShiftBuiltinSet; |
| 1624 | defm vsra : RVVSignedShiftBuiltinSet; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1625 | |
| 1626 | // 12.7. Vector Narrowing Integer Right Shift Instructions |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 1627 | defm vnsrl : RVVUnsignedNShiftBuiltinSet; |
| 1628 | defm vnsra : RVVSignedNShiftBuiltinSet; |
Hsiangkai Wang | bd32c2d | 2021-04-20 15:31:13 +0800 | [diff] [blame] | 1629 | defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", |
| 1630 | [["v", "vw"], |
| 1631 | ["Uv", "UvUw"]]>; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1632 | |
| 1633 | // 12.8. Vector Integer Comparison Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1634 | let HasPolicy = false in { |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 1635 | defm vmseq : RVVIntMaskOutBuiltinSet; |
| 1636 | defm vmsne : RVVIntMaskOutBuiltinSet; |
| 1637 | defm vmsltu : RVVUnsignedMaskOutBuiltinSet; |
| 1638 | defm vmslt : RVVSignedMaskOutBuiltinSet; |
| 1639 | defm vmsleu : RVVUnsignedMaskOutBuiltinSet; |
| 1640 | defm vmsle : RVVSignedMaskOutBuiltinSet; |
Hsiangkai Wang | 645c5f2 | 2021-04-20 14:50:42 +0800 | [diff] [blame] | 1641 | defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; |
| 1642 | defm vmsgt : RVVSignedMaskOutBuiltinSet; |
| 1643 | defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; |
| 1644 | defm vmsge : RVVSignedMaskOutBuiltinSet; |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1645 | } |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1646 | |
| 1647 | // 12.9. Vector Integer Min/Max Instructions |
| 1648 | defm vminu : RVVUnsignedBinBuiltinSet; |
| 1649 | defm vmin : RVVSignedBinBuiltinSet; |
| 1650 | defm vmaxu : RVVUnsignedBinBuiltinSet; |
| 1651 | defm vmax : RVVSignedBinBuiltinSet; |
| 1652 | |
| 1653 | // 12.10. Vector Single-Width Integer Multiply Instructions |
| 1654 | defm vmul : RVVIntBinBuiltinSet; |
| 1655 | defm vmulh : RVVSignedBinBuiltinSet; |
| 1656 | defm vmulhu : RVVUnsignedBinBuiltinSet; |
| 1657 | defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", |
| 1658 | [["vv", "v", "vvUv"], |
| 1659 | ["vx", "v", "vvUe"]]>; |
| 1660 | |
| 1661 | // 12.11. Vector Integer Divide Instructions |
| 1662 | defm vdivu : RVVUnsignedBinBuiltinSet; |
| 1663 | defm vdiv : RVVSignedBinBuiltinSet; |
| 1664 | defm vremu : RVVUnsignedBinBuiltinSet; |
| 1665 | defm vrem : RVVSignedBinBuiltinSet; |
| 1666 | |
| 1667 | // 12.12. Vector Widening Integer Multiply Instructions |
Zakk Chen | 0a18ea0 | 2021-03-29 09:38:55 -0700 | [diff] [blame] | 1668 | let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { |
| 1669 | defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", |
| 1670 | [["vv", "w", "wvv"], |
| 1671 | ["vx", "w", "wve"]]>; |
| 1672 | defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", |
| 1673 | [["vv", "Uw", "UwUvUv"], |
| 1674 | ["vx", "Uw", "UwUvUe"]]>; |
| 1675 | defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", |
| 1676 | [["vv", "w", "wvUv"], |
| 1677 | ["vx", "w", "wvUe"]]>; |
| 1678 | } |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1679 | |
| 1680 | // 12.13. Vector Single-Width Integer Multiply-Add Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1681 | let HasPolicy = false in { |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1682 | defm vmacc : RVVIntTerBuiltinSet; |
| 1683 | defm vnmsac : RVVIntTerBuiltinSet; |
| 1684 | defm vmadd : RVVIntTerBuiltinSet; |
| 1685 | defm vnmsub : RVVIntTerBuiltinSet; |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1686 | |
| 1687 | // 12.14. Vector Widening Integer Multiply-Add Instructions |
Zakk Chen | fe252b5 | 2021-03-30 08:59:07 -0700 | [diff] [blame] | 1688 | let HasMaskedOffOperand = false, |
| 1689 | Log2LMUL = [-3, -2, -1, 0, 1, 2] in { |
| 1690 | defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", |
| 1691 | [["vv", "Uw", "UwUwUvUv"], |
| 1692 | ["vx", "Uw", "UwUwUeUv"]]>; |
| 1693 | defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", |
| 1694 | [["vv", "w", "wwvv"], |
| 1695 | ["vx", "w", "wwev"]]>; |
| 1696 | defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", |
| 1697 | [["vv", "w", "wwvUv"], |
| 1698 | ["vx", "w", "wweUv"]]>; |
| 1699 | defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", |
| 1700 | [["vx", "w", "wwUev"]]>; |
| 1701 | } |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1702 | } |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1703 | |
| 1704 | // 12.15. Vector Integer Merge Instructions |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 1705 | // C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (op1, op2, mask, vl) |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1706 | let HasMask = false, HasPolicy = false, |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 1707 | ManualCodegen = [{ |
| 1708 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); |
| 1709 | IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()}; |
| 1710 | }] in { |
| 1711 | defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", |
| 1712 | [["vvm", "v", "vmvv"], |
| 1713 | ["vxm", "v", "vmve"], |
| 1714 | ["vvm", "Uv", "UvmUvUv"], |
| 1715 | ["vxm", "Uv", "UvmUvUe"]]>; |
Zakk Chen | 07c3854 | 2021-04-06 07:23:30 -0700 | [diff] [blame] | 1716 | } |
Zakk Chen | 66c0560 | 2021-03-29 07:37:29 -0700 | [diff] [blame] | 1717 | |
| 1718 | // 12.16. Vector Integer Move Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1719 | let HasMask = false, HasPolicy = false in { |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1720 | let MangledName = "vmv_v" in { |
| 1721 | defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", |
| 1722 | [["v", "Uv", "UvUv"]]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1723 | defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfd", |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1724 | [["v", "v", "vv"]]>; |
| 1725 | } |
| 1726 | let HasNoMaskedOverloaded = false in |
| 1727 | defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", |
| 1728 | [["x", "v", "ve"], |
| 1729 | ["x", "Uv", "UvUe"]]>; |
| 1730 | } |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 1731 | |
Zakk Chen | f2a3601 | 2021-03-30 10:12:07 -0700 | [diff] [blame] | 1732 | // 13. Vector Fixed-Point Arithmetic Instructions |
| 1733 | // 13.1. Vector Single-Width Saturating Add and Subtract |
| 1734 | defm vsaddu : RVVUnsignedBinBuiltinSet; |
| 1735 | defm vsadd : RVVSignedBinBuiltinSet; |
| 1736 | defm vssubu : RVVUnsignedBinBuiltinSet; |
| 1737 | defm vssub : RVVSignedBinBuiltinSet; |
| 1738 | |
| 1739 | // 13.2. Vector Single-Width Averaging Add and Subtract |
| 1740 | defm vaaddu : RVVUnsignedBinBuiltinSet; |
| 1741 | defm vaadd : RVVSignedBinBuiltinSet; |
| 1742 | defm vasubu : RVVUnsignedBinBuiltinSet; |
| 1743 | defm vasub : RVVSignedBinBuiltinSet; |
| 1744 | |
| 1745 | // 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation |
| 1746 | defm vsmul : RVVSignedBinBuiltinSet; |
| 1747 | |
| 1748 | // 13.4. Vector Single-Width Scaling Shift Instructions |
| 1749 | defm vssrl : RVVUnsignedShiftBuiltinSet; |
| 1750 | defm vssra : RVVSignedShiftBuiltinSet; |
| 1751 | |
| 1752 | // 13.5. Vector Narrowing Fixed-Point Clip Instructions |
| 1753 | defm vnclipu : RVVUnsignedNShiftBuiltinSet; |
| 1754 | defm vnclip : RVVSignedNShiftBuiltinSet; |
| 1755 | |
Zakk Chen | d6a0560 | 2021-03-05 07:40:28 -0800 | [diff] [blame] | 1756 | // 14. Vector Floating-Point Instructions |
| 1757 | // 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1758 | defm vfadd : RVVFloatingBinBuiltinSet; |
| 1759 | defm vfsub : RVVFloatingBinBuiltinSet; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1760 | defm vfrsub : RVVFloatingBinVFBuiltinSet; |
| 1761 | |
| 1762 | // 14.3. Vector Widening Floating-Point Add/Subtract Instructions |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1763 | // Widening FP add/subtract, 2*SEW = SEW +/- SEW |
| 1764 | defm vfwadd : RVVFloatingWidenBinBuiltinSet; |
| 1765 | defm vfwsub : RVVFloatingWidenBinBuiltinSet; |
| 1766 | // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW |
| 1767 | defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; |
| 1768 | defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1769 | |
| 1770 | // 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1771 | defm vfmul : RVVFloatingBinBuiltinSet; |
| 1772 | defm vfdiv : RVVFloatingBinBuiltinSet; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1773 | defm vfrdiv : RVVFloatingBinVFBuiltinSet; |
| 1774 | |
| 1775 | // 14.5. Vector Widening Floating-Point Multiply |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1776 | let Log2LMUL = [-2, -1, 0, 1, 2] in { |
| 1777 | defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1778 | [["vv", "w", "wvv"], |
| 1779 | ["vf", "w", "wve"]]>; |
| 1780 | } |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1781 | |
| 1782 | // 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1783 | let HasPolicy = false in { |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1784 | defm vfmacc : RVVFloatingTerBuiltinSet; |
| 1785 | defm vfnmacc : RVVFloatingTerBuiltinSet; |
| 1786 | defm vfmsac : RVVFloatingTerBuiltinSet; |
| 1787 | defm vfnmsac : RVVFloatingTerBuiltinSet; |
| 1788 | defm vfmadd : RVVFloatingTerBuiltinSet; |
| 1789 | defm vfnmadd : RVVFloatingTerBuiltinSet; |
| 1790 | defm vfmsub : RVVFloatingTerBuiltinSet; |
| 1791 | defm vfnmsub : RVVFloatingTerBuiltinSet; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1792 | |
| 1793 | // 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1794 | defm vfwmacc : RVVFloatingWidenTerBuiltinSet; |
| 1795 | defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; |
| 1796 | defm vfwmsac : RVVFloatingWidenTerBuiltinSet; |
| 1797 | defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1798 | } |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1799 | |
| 1800 | // 14.8. Vector Floating-Point Square-Root Instruction |
Zakk Chen | 5f7739b | 2021-04-08 08:21:06 -0700 | [diff] [blame] | 1801 | def vfsqrt : RVVFloatingUnaryVVBuiltin; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1802 | |
| 1803 | // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction |
Zakk Chen | 5f7739b | 2021-04-08 08:21:06 -0700 | [diff] [blame] | 1804 | def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1805 | |
| 1806 | // 14.10. Vector Floating-Point Reciprocal Estimate Instruction |
Zakk Chen | 5f7739b | 2021-04-08 08:21:06 -0700 | [diff] [blame] | 1807 | def vfrec7 : RVVFloatingUnaryVVBuiltin; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1808 | |
| 1809 | // 14.11. Vector Floating-Point MIN/MAX Instructions |
| 1810 | defm vfmin : RVVFloatingBinBuiltinSet; |
| 1811 | defm vfmax : RVVFloatingBinBuiltinSet; |
| 1812 | |
| 1813 | // 14.12. Vector Floating-Point Sign-Injection Instructions |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1814 | defm vfsgnj : RVVFloatingBinBuiltinSet; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1815 | defm vfsgnjn : RVVFloatingBinBuiltinSet; |
| 1816 | defm vfsgnjx : RVVFloatingBinBuiltinSet; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1817 | defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; |
| 1818 | defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1819 | |
| 1820 | // 14.13. Vector Floating-Point Compare Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1821 | let HasPolicy = false in { |
Zakk Chen | 98a3ff9 | 2021-04-08 08:09:42 -0700 | [diff] [blame] | 1822 | defm vmfeq : RVVFloatingMaskOutBuiltinSet; |
| 1823 | defm vmfne : RVVFloatingMaskOutBuiltinSet; |
| 1824 | defm vmflt : RVVFloatingMaskOutBuiltinSet; |
| 1825 | defm vmfle : RVVFloatingMaskOutBuiltinSet; |
Hsiangkai Wang | 645c5f2 | 2021-04-20 14:50:42 +0800 | [diff] [blame] | 1826 | defm vmfgt : RVVFloatingMaskOutBuiltinSet; |
| 1827 | defm vmfge : RVVFloatingMaskOutBuiltinSet; |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1828 | } |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1829 | |
| 1830 | // 14.14. Vector Floating-Point Classify Instruction |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1831 | let Name = "vfclass_v", HasPolicy = false in |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1832 | def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1833 | |
| 1834 | // 14.15. Vector Floating-Point Merge Instructio |
Zakk Chen | 07c3854 | 2021-04-06 07:23:30 -0700 | [diff] [blame] | 1835 | // C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1836 | let HasMask = false, HasPolicy = false, |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 1837 | ManualCodegen = [{ |
| 1838 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); |
| 1839 | IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()}; |
| 1840 | }] in { |
Craig Topper | e2b7aab | 2021-10-25 08:56:43 -0700 | [diff] [blame] | 1841 | defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfd", |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 1842 | [["vvm", "v", "vmvv"]]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1843 | defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 1844 | [["vfm", "v", "vmve"]]>; |
Zakk Chen | 07c3854 | 2021-04-06 07:23:30 -0700 | [diff] [blame] | 1845 | } |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1846 | |
| 1847 | // 14.16. Vector Floating-Point Move Instruction |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1848 | let HasMask = false, HasNoMaskedOverloaded = false, HasPolicy = false in |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1849 | defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1850 | [["f", "v", "ve"]]>; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1851 | |
| 1852 | // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 1853 | def vfcvt_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_xu">; |
| 1854 | def vfcvt_x_f_v : RVVConvToSignedBuiltin<"vfcvt_x">; |
| 1855 | def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; |
| 1856 | def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; |
| 1857 | def vfcvt_f_xu_v : RVVConvBuiltin<"Fv", "FvUv", "sil", "vfcvt_f">; |
| 1858 | def vfcvt_f_x_v : RVVConvBuiltin<"Fv", "Fvv", "sil", "vfcvt_f">; |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1859 | |
| 1860 | // 14.18. Widening Floating-Point/Integer Type-Convert Instructions |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 1861 | let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { |
| 1862 | def vfwcvt_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_xu">; |
| 1863 | def vfwcvt_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_x">; |
| 1864 | def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; |
| 1865 | def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; |
| 1866 | def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; |
| 1867 | def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1868 | def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "xf", "vfwcvt_f">; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 1869 | } |
Zakk Chen | 007ea0e | 2021-04-08 07:29:59 -0700 | [diff] [blame] | 1870 | |
| 1871 | // 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 1872 | let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { |
| 1873 | def vfncvt_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_xu">; |
| 1874 | def vfncvt_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_x">; |
| 1875 | def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; |
| 1876 | def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1877 | def vfncvt_f_xu_w : RVVConvBuiltin<"Fv", "FvUw", "csi", "vfncvt_f">; |
| 1878 | def vfncvt_f_x_w : RVVConvBuiltin<"Fv", "Fvw", "csi", "vfncvt_f">; |
| 1879 | def vfncvt_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_f">; |
| 1880 | def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; |
Zakk Chen | 01fa222 | 2021-04-01 09:21:11 -0700 | [diff] [blame] | 1881 | } |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 1882 | |
| 1883 | // 15. Vector Reduction Operations |
| 1884 | // 15.1. Vector Single-Width Integer Reduction Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1885 | let HasPolicy = false in { |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 1886 | defm vredsum : RVVIntReductionBuiltinSet; |
| 1887 | defm vredmaxu : RVVUnsignedReductionBuiltin; |
| 1888 | defm vredmax : RVVSignedReductionBuiltin; |
| 1889 | defm vredminu : RVVUnsignedReductionBuiltin; |
| 1890 | defm vredmin : RVVSignedReductionBuiltin; |
| 1891 | defm vredand : RVVIntReductionBuiltinSet; |
| 1892 | defm vredor : RVVIntReductionBuiltinSet; |
| 1893 | defm vredxor : RVVIntReductionBuiltinSet; |
| 1894 | |
| 1895 | // 15.2. Vector Widening Integer Reduction Instructions |
| 1896 | // Vector Widening Integer Reduction Operations |
| 1897 | let HasMaskedOffOperand = false in { |
| 1898 | defm vwredsum : RVVOutOp1BuiltinSet<"vwredsum", "csi", |
| 1899 | [["vs", "vSw", "SwSwvSw"]]>; |
| 1900 | defm vwredsumu : RVVOutOp1BuiltinSet<"vwredsumu", "csi", |
| 1901 | [["vs", "UvUSw", "USwUSwUvUSw"]]>; |
| 1902 | } |
| 1903 | |
| 1904 | // 15.3. Vector Single-Width Floating-Point Reduction Instructions |
| 1905 | defm vfredmax : RVVFloatingReductionBuiltin; |
| 1906 | defm vfredmin : RVVFloatingReductionBuiltin; |
jacquesguan | 0608bbd | 2021-10-12 06:46:46 +0000 | [diff] [blame] | 1907 | defm vfredusum : RVVFloatingReductionBuiltin; |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 1908 | defm vfredosum : RVVFloatingReductionBuiltin; |
| 1909 | |
| 1910 | // 15.4. Vector Widening Floating-Point Reduction Instructions |
jacquesguan | 0608bbd | 2021-10-12 06:46:46 +0000 | [diff] [blame] | 1911 | defm vfwredusum : RVVFloatingWidenReductionBuiltin; |
Zakk Chen | c680b0d | 2021-04-06 07:57:41 -0700 | [diff] [blame] | 1912 | defm vfwredosum : RVVFloatingWidenReductionBuiltin; |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1913 | } |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 1914 | |
| 1915 | // 16. Vector Mask Instructions |
| 1916 | // 16.1. Vector Mask-Register Logical Instructions |
| 1917 | def vmand : RVVMaskBinBuiltin; |
| 1918 | def vmnand : RVVMaskBinBuiltin; |
Zakk Chen | 0649dfe | 2021-11-04 09:22:34 -0700 | [diff] [blame] | 1919 | def vmandn : RVVMaskBinBuiltin; |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 1920 | def vmxor : RVVMaskBinBuiltin; |
| 1921 | def vmor : RVVMaskBinBuiltin; |
| 1922 | def vmnor : RVVMaskBinBuiltin; |
Zakk Chen | 0649dfe | 2021-11-04 09:22:34 -0700 | [diff] [blame] | 1923 | def vmorn : RVVMaskBinBuiltin; |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 1924 | def vmxnor : RVVMaskBinBuiltin; |
| 1925 | // pseudoinstructions |
| 1926 | def vmclr : RVVMaskNullaryBuiltin; |
| 1927 | def vmset : RVVMaskNullaryBuiltin; |
Hsiangkai Wang | 4b24341 | 2021-04-20 11:12:53 +0800 | [diff] [blame] | 1928 | defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; |
| 1929 | defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 1930 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1931 | let HasPolicy = false in { |
Zakk Chen | 0649dfe | 2021-11-04 09:22:34 -0700 | [diff] [blame] | 1932 | // 16.2. Vector count population in mask vcpop.m |
| 1933 | def vcpop : RVVMaskOp0Builtin<"um">; |
Zakk Chen | a8fc0e4 | 2021-04-08 08:28:15 -0700 | [diff] [blame] | 1934 | |
| 1935 | // 16.3. vfirst find-first-set mask bit |
| 1936 | def vfirst : RVVMaskOp0Builtin<"lm">; |
| 1937 | |
| 1938 | // 16.4. vmsbf.m set-before-first mask bit |
| 1939 | def vmsbf : RVVMaskUnaryBuiltin; |
| 1940 | |
| 1941 | // 16.5. vmsif.m set-including-first mask bit |
| 1942 | def vmsif : RVVMaskUnaryBuiltin; |
| 1943 | |
| 1944 | // 16.6. vmsof.m set-only-first mask bit |
| 1945 | def vmsof : RVVMaskUnaryBuiltin; |
| 1946 | |
| 1947 | let HasNoMaskedOverloaded = false in { |
| 1948 | // 16.8. Vector Iota Instruction |
| 1949 | defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; |
| 1950 | |
| 1951 | // 16.9. Vector Element Index Instruction |
| 1952 | defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], |
| 1953 | ["v", "Uv", "Uv"]]>; |
| 1954 | } |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1955 | } |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 1956 | |
| 1957 | // 17. Vector Permutation Instructions |
| 1958 | // 17.1. Integer Scalar Move Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1959 | let HasMask = false, HasPolicy = false in { |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1960 | let HasVL = false, MangledName = "vmv_x" in |
| 1961 | defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", |
| 1962 | [["s", "ve", "ev"], |
| 1963 | ["s", "UvUe", "UeUv"]]>; |
| 1964 | let MangledName = "vmv_s" in |
| 1965 | defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", |
| 1966 | [["x", "v", "vve"], |
| 1967 | ["x", "Uv", "UvUvUe"]]>; |
| 1968 | } |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 1969 | |
| 1970 | // 17.2. Floating-Point Scalar Move Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1971 | let HasMask = false, HasPolicy = false in { |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1972 | let HasVL = false, MangledName = "vfmv_f" in |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1973 | defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1974 | [["s", "ve", "ev"]]>; |
| 1975 | let MangledName = "vfmv_s" in |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1976 | defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", |
Zakk Chen | ea5d33d | 2021-04-11 06:50:57 -0700 | [diff] [blame] | 1977 | [["f", "v", "vve"], |
| 1978 | ["x", "Uv", "UvUvUe"]]>; |
| 1979 | } |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 1980 | |
| 1981 | // 17.3. Vector Slide Instructions |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1982 | let HasPolicy = false in { |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 1983 | // 17.3.1. Vector Slideup Instructions |
| 1984 | defm vslideup : RVVSlideBuiltinSet; |
| 1985 | // 17.3.2. Vector Slidedown Instructions |
| 1986 | defm vslidedown : RVVSlideBuiltinSet; |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 1987 | } |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 1988 | |
| 1989 | // 17.3.3. Vector Slide1up Instructions |
| 1990 | defm vslide1up : RVVSlideOneBuiltinSet; |
| 1991 | defm vfslide1up : RVVFloatingBinVFBuiltinSet; |
| 1992 | |
| 1993 | // 17.3.4. Vector Slide1down Instruction |
| 1994 | defm vslide1down : RVVSlideOneBuiltinSet; |
| 1995 | defm vfslide1down : RVVFloatingBinVFBuiltinSet; |
| 1996 | |
| 1997 | // 17.4. Vector Register Gather Instructions |
| 1998 | // signed and floating type |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 1999 | defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 2000 | [["vv", "v", "vvUv"]]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2001 | defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 2002 | [["vx", "v", "vvz"]]>; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2003 | defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 2004 | [["vv", "v", "vv(Log2EEW:4)Uv"]]>; |
| 2005 | // unsigned type |
| 2006 | defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", |
| 2007 | [["vv", "Uv", "UvUvUv"]]>; |
| 2008 | defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", |
| 2009 | [["vx", "Uv", "UvUvz"]]>; |
| 2010 | defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", |
| 2011 | [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; |
| 2012 | |
| 2013 | // 17.5. Vector Compress Instruction |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2014 | let HasMask = false, HasPolicy = false, |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 2015 | ManualCodegen = [{ |
| 2016 | std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3); |
Craig Topper | 081ae5f | 2021-06-10 20:42:11 -0700 | [diff] [blame] | 2017 | IntrinsicTypes = {ResultType, Ops[3]->getType()}; |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 2018 | }] in { |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 2019 | // signed and floating type |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2020 | defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 2021 | [["vm", "v", "vmvv"]]>; |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 2022 | // unsigned type |
| 2023 | defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", |
Craig Topper | cfe3b00 | 2021-05-01 13:18:21 -0700 | [diff] [blame] | 2024 | [["vm", "Uv", "UvmUvUv"]]>; |
Zakk Chen | 59d5b8c | 2021-04-08 10:15:09 -0700 | [diff] [blame] | 2025 | } |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2026 | |
| 2027 | // Miscellaneous |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2028 | let HasMask = false, HasVL = false, IRName = "" in { |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2029 | let Name = "vreinterpret_v", HasPolicy = false, |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2030 | ManualCodegen = [{ |
| 2031 | return Builder.CreateBitCast(Ops[0], ResultType); |
| 2032 | }] in { |
| 2033 | // Reinterpret between different type under the same SEW and LMUL |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2034 | def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2035 | def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">; |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2036 | def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2037 | def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">; |
| 2038 | def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">; |
| 2039 | def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">; |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2040 | |
| 2041 | // Reinterpret between different SEW under the same LMUL |
| 2042 | foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", |
| 2043 | "(FixedSEW:64)"] in { |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2044 | def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", |
| 2045 | dst_sew # "vv", "csil", dst_sew # "v">; |
| 2046 | def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", |
| 2047 | dst_sew # "UvUv", "csil", dst_sew # "Uv">; |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2048 | } |
| 2049 | } |
| 2050 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2051 | let Name = "vundefined", HasNoMaskedOverloaded = false, HasPolicy = false, |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2052 | ManualCodegen = [{ |
| 2053 | return llvm::UndefValue::get(ResultType); |
| 2054 | }] in { |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2055 | def vundefined : RVVBuiltin<"v", "v", "csilxfd">; |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2056 | def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; |
| 2057 | } |
| 2058 | |
| 2059 | // LMUL truncation |
| 2060 | // C/C++ Operand: VecTy, IR Operand: VecTy, Index |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2061 | let Name = "vlmul_trunc_v", MangledName = "vlmul_trunc", HasPolicy = false, |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2062 | ManualCodegen = [{ { |
| 2063 | ID = Intrinsic::experimental_vector_extract; |
| 2064 | IntrinsicTypes = {ResultType, Ops[0]->getType()}; |
| 2065 | Ops.push_back(ConstantInt::get(Int64Ty, 0)); |
| 2066 | return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); |
| 2067 | } }] in { |
| 2068 | foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", |
| 2069 | "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2070 | def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2071 | dst_lmul # "vv", "csilxfd", dst_lmul # "v">; |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2072 | def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", |
| 2073 | dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2074 | } |
| 2075 | } |
| 2076 | |
| 2077 | // LMUL extension |
| 2078 | // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2079 | let Name = "vlmul_ext_v", MangledName = "vlmul_ext", HasPolicy = false, |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2080 | ManualCodegen = [{ |
| 2081 | ID = Intrinsic::experimental_vector_insert; |
| 2082 | IntrinsicTypes = {ResultType, Ops[0]->getType()}; |
| 2083 | Ops.push_back(llvm::UndefValue::get(ResultType)); |
| 2084 | std::swap(Ops[0], Ops[1]); |
| 2085 | Ops.push_back(ConstantInt::get(Int64Ty, 0)); |
| 2086 | return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); |
| 2087 | }] in { |
| 2088 | foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", |
| 2089 | "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2090 | def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", |
Hsiangkai Wang | 77bb82d | 2021-06-28 13:38:41 +0800 | [diff] [blame] | 2091 | dst_lmul # "vv", "csilxfd", dst_lmul # "v">; |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2092 | def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", |
| 2093 | dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2094 | } |
| 2095 | } |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2096 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2097 | let Name = "vget_v", HasPolicy = false, |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2098 | ManualCodegen = [{ |
| 2099 | { |
| 2100 | ID = Intrinsic::experimental_vector_extract; |
Craig Topper | e2b7aab | 2021-10-25 08:56:43 -0700 | [diff] [blame] | 2101 | auto *VecTy = cast<ScalableVectorType>(ResultType); |
| 2102 | auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); |
| 2103 | // Mask to only valid indices. |
| 2104 | unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); |
| 2105 | assert(isPowerOf2_32(MaxIndex)); |
| 2106 | Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2107 | Ops[1] = Builder.CreateMul(Ops[1], |
| 2108 | ConstantInt::get(Ops[1]->getType(), |
| 2109 | VecTy->getMinNumElements())); |
| 2110 | IntrinsicTypes = {ResultType, Ops[0]->getType()}; |
| 2111 | return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); |
| 2112 | } |
| 2113 | }] in { |
| 2114 | foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { |
Hsiangkai Wang | 5f99670 | 2021-08-04 07:57:13 +0800 | [diff] [blame] | 2115 | def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">; |
Zakk Chen | 08cf69c | 2021-07-13 20:32:55 -0700 | [diff] [blame] | 2116 | def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2117 | } |
| 2118 | } |
| 2119 | |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2120 | let Name = "vset_v", Log2LMUL = [0, 1, 2], HasPolicy = false, |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2121 | ManualCodegen = [{ |
| 2122 | { |
| 2123 | ID = Intrinsic::experimental_vector_insert; |
| 2124 | IntrinsicTypes = {ResultType, Ops[2]->getType()}; |
Craig Topper | e2b7aab | 2021-10-25 08:56:43 -0700 | [diff] [blame] | 2125 | auto *ResVecTy = cast<ScalableVectorType>(ResultType); |
| 2126 | auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); |
| 2127 | // Mask to only valid indices. |
| 2128 | unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); |
| 2129 | assert(isPowerOf2_32(MaxIndex)); |
| 2130 | Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2131 | Ops[1] = Builder.CreateMul(Ops[1], |
| 2132 | ConstantInt::get(Ops[1]->getType(), |
| 2133 | VecTy->getMinNumElements())); |
| 2134 | std::swap(Ops[1], Ops[2]); |
| 2135 | return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, ""); |
| 2136 | } |
| 2137 | }] in { |
| 2138 | foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { |
Hsiangkai Wang | 5f99670 | 2021-08-04 07:57:13 +0800 | [diff] [blame] | 2139 | def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; |
Craig Topper | f225367 | 2021-06-24 15:53:47 -0700 | [diff] [blame] | 2140 | def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; |
| 2141 | } |
| 2142 | } |
Zakk Chen | 8f68336 | 2021-04-12 19:01:44 -0700 | [diff] [blame] | 2143 | } |
Hsiangkai Wang | 5158cfef | 2021-10-13 14:35:56 +0800 | [diff] [blame] | 2144 | |
| 2145 | let HeaderCode = [{ |
| 2146 | #define VE_TAIL_UNDISTURBED 0 |
| 2147 | #define VE_TAIL_AGNOSTIC 1 |
| 2148 | }] in |
| 2149 | def policy : RVVHeader; |