blob: cc242da7f1ca8a2e5a6691f04a2ce3b10d87f640 [file] [log] [blame]
Zakk Chend6a05602021-03-05 07:40:28 -08001//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the builtins for RISC-V V-extension. See:
10//
11// https://github.com/riscv/rvv-intrinsic-doc
12//
13//===----------------------------------------------------------------------===//
14
15//===----------------------------------------------------------------------===//
16// Instruction definitions
17//===----------------------------------------------------------------------===//
18// Each record of the class RVVBuiltin defines a collection of builtins (i.e.
19// "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1",
20// "vadd_vv_i32m2", etc).
21//
22// The elements of this collection are defined by an instantiation process the
23// range of which is specified by the cross product of the LMUL attribute and
24// every element in the attribute TypeRange. By default builtins have LMUL = [1,
25// 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we
26// use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL.
27//
28// LMUL represents the fact that the types of values used by that builtin are
29// values generated by instructions that are executed under that LMUL. However,
30// this does not mean the builtin is necessarily lowered into an instruction
31// that executes under the specified LMUL. An example where this happens are
32// loads and stores of masks. A mask like `vbool8_t` can be generated, for
33// instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two
34// `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will
35// be performed under LMUL=1 because mask registers are not grouped.
36//
37// TypeRange is a non-empty sequence of basic types:
38//
39// c: int8_t (i8)
40// s: int16_t (i16)
41// i: int32_t (i32)
42// l: int64_t (i64)
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +080043// x: float16_t (half)
Zakk Chend6a05602021-03-05 07:40:28 -080044// f: float32_t (float)
45// d: float64_t (double)
46//
47// This way, given an LMUL, a record with a TypeRange "sil" will cause the
48// definition of 3 builtins. Each type "t" in the TypeRange (in this example
49// they are int16_t, int32_t, int64_t) is used as a parameter that drives the
50// definition of that particular builtin (for the given LMUL).
51//
52// During the instantiation, types can be transformed or modified using type
53// transformers. Given a type "t" the following primitive type transformers can
54// be applied to it to yield another type.
55//
56// e: type of "t" as is (identity)
57// v: computes a vector type whose element type is "t" for the current LMUL
58// w: computes a vector type identical to what 'v' computes except for the
59// element type which is twice as wide as the element type of 'v'
60// q: computes a vector type identical to what 'v' computes except for the
61// element type which is four times as wide as the element type of 'v'
62// o: computes a vector type identical to what 'v' computes except for the
63// element type which is eight times as wide as the element type of 'v'
64// m: computes a vector type identical to what 'v' computes except for the
65// element type which is bool
66// 0: void type, ignores "t"
67// z: size_t, ignores "t"
68// t: ptrdiff_t, ignores "t"
Zakk Chena8fc0e42021-04-08 08:28:15 -070069// u: unsigned long, ignores "t"
70// l: long, ignores "t"
Zakk Chend6a05602021-03-05 07:40:28 -080071//
72// So for instance if t is "i", i.e. int, then "e" will yield int again. "v"
73// will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t.
74// Accordingly "w" would yield __rvv_int64m2_t.
75//
76// A type transformer can be prefixed by other non-primitive type transformers.
77//
78// P: constructs a pointer to the current type
79// C: adds const to the type
80// K: requires the integer type to be a constant expression
81// U: given an integer type or vector type, computes its unsigned variant
82// I: given a vector type, compute the vector type with integer type
83// elements of the same width
84// F: given a vector type, compute the vector type with floating-point type
85// elements of the same width
86// S: given a vector type, computes its equivalent one for LMUL=1. This is a
87// no-op if the vector was already LMUL=1
Zakk Chen88c2d4c2021-03-17 20:25:32 -070088// (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a
89// vector type (SEW and LMUL) and EEW (8/16/32/64), computes its
90// equivalent integer vector type with EEW and corresponding ELMUL (elmul =
91// (eew/sew) * lmul). For example, vector type is __rvv_float16m4
92// (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector
93// type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new
94// builtins if its equivalent type has illegal lmul.
Zakk Chen8f683362021-04-12 19:01:44 -070095// (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another
96// vector type which only changed SEW as given value. Ignore to define a new
97// builtin if its equivalent type has illegal lmul or the SEW does not changed.
98// (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW
99// and LMUL), and computes another vector type which only changed LMUL as
100// given value. The new LMUL should be smaller than the old one. Ignore to
101// define a new builtin if its equivalent type has illegal lmul.
102// (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW
103// and LMUL), and computes another vector type which only changed LMUL as
104// given value. The new LMUL should be larger than the old one. Ignore to
105// define a new builtin if its equivalent type has illegal lmul.
Zakk Chend6a05602021-03-05 07:40:28 -0800106//
107// Following with the example above, if t is "i", then "Ue" will yield unsigned
108// int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would
109// yield __rvv_float64m2_t, etc.
110//
111// Each builtin is then defined by applying each type in TypeRange against the
112// sequence of type transformers described in Suffix and Prototype.
113//
114// The name of the builtin is defined by the Name attribute (which defaults to
115// the name of the class) appended (separated with an underscore) the Suffix
116// attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il",
117// the builtin generated will be __builtin_rvv_foo_i32m1 and
118// __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one
119// type transformer (say "vv") each of the types is separated with an
120// underscore as in "__builtin_rvv_foo_i32m1_i32m1".
121//
122// The C/C++ prototype of the builtin is defined by the Prototype attribute.
123// Prototype is a non-empty sequence of type transformers, the first of which
124// is the return type of the builtin and the rest are the parameters of the
125// builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si"
126// a first builtin will have type
127// __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin
128// will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again
129// under LMUL=1).
130//
131// There are a number of attributes that are used to constraint the number and
132// shape of the builtins generated. Refer to the comments below for them.
133class RVVBuiltin<string suffix, string prototype, string type_range,
Zakk Chen08cf69c2021-07-13 20:32:55 -0700134 string mangled_suffix = ""> {
Zakk Chend6a05602021-03-05 07:40:28 -0800135 // Base name that will be prepended in __builtin_rvv_ and appended the
136 // computed Suffix.
137 string Name = NAME;
138
139 // If not empty, each instantiated builtin will have this appended after an
140 // underscore (_). It is instantiated like Prototype.
141 string Suffix = suffix;
142
143 // If empty, default MangledName is sub string of `Name` which end of first
144 // '_'. For example, the default mangled name is `vadd` for Name `vadd_vv`.
145 // It's used for describe some special naming cases.
146 string MangledName = "";
147
Zakk Chen08cf69c2021-07-13 20:32:55 -0700148 // If not empty, each MangledName will have this appended after an
149 // underscore (_). It is instantiated like Prototype.
150 string MangledSuffix = mangled_suffix;
151
Zakk Chend6a05602021-03-05 07:40:28 -0800152 // The different variants of the builtin, parameterised with a type.
153 string TypeRange = type_range;
154
155 // We use each type described in TypeRange and LMUL with prototype to
156 // instantiate a specific element of the set of builtins being defined.
157 // Prototype attribute defines the C/C++ prototype of the builtin. It is a
158 // non-empty sequence of type transformers, the first of which is the return
159 // type of the builtin and the rest are the parameters of the builtin, in
160 // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a
161 // first builtin will have type
162 // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin
163 // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t).
164 string Prototype = prototype;
165
166 // This builtin has a masked form.
167 bit HasMask = true;
168
169 // If HasMask, this flag states that this builtin has a maskedoff operand. It
170 // is always the first operand in builtin and IR intrinsic.
171 bit HasMaskedOffOperand = true;
172
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800173 // This builtin has a granted vector length parameter.
Zakk Chend6a05602021-03-05 07:40:28 -0800174 bit HasVL = true;
175
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800176 // There are several cases for specifying tail policy.
177 // 1. Add tail policy argument to masked intrinsics. It may have the maskedoff
178 // argument or not.
179 // * Have the maskedoff argument: (HasPolicy = true, HasMaskedOffOperand = true)
180 // Ex: vadd_vv_i8m1_mt(mask, maskedoff, op1, op2, vl, ta);
181 // * Do not have the maskedoff argument: (HasPolicy = true, HasMaskedOffOperand = false)
182 // Ex: vmacc_vv_i8m1_mt(mask, vd, vs1, vs2, vl, ta);
183 // 2. Add dest argument for no mask intrinsics. (TODO)
184 // Ex: vmv_v_x_i8m1_t(dest, src, vl);
185 // 3. Always tail agnostic. (HasPolicy = false)
186 // Ex: vmseq_vv_i8m1_b8_m(mask, maskedoff, op1, op2, vl);
187 // The tail policy argument is located at the last position.
188 bit HasPolicy = true;
189
Zakk Chen821547c2021-03-22 07:51:52 -0700190 // This builtin supports non-masked function overloading api.
191 // All masked operations support overloading api.
192 bit HasNoMaskedOverloaded = true;
Zakk Chend6a05602021-03-05 07:40:28 -0800193
Zakk Chend6a05602021-03-05 07:40:28 -0800194 // This builtin is valid for the given Log2LMULs.
195 list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3];
196
Zakk Chen95c01252021-02-25 00:15:14 -0800197 // Manual code in clang codegen riscv_vector_builtin_cg.inc
198 code ManualCodegen = [{}];
199 code ManualCodegenMask = [{}];
200
201 // When emit the automatic clang codegen, it describes what types we have to use
Zakk Chend6a05602021-03-05 07:40:28 -0800202 // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise,
203 // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd
204 // parameter of the unmasked version. k can't be the mask operand's position.
205 list<int> IntrinsicTypes = [];
206
207 // If these names are not empty, this is the ID of the LLVM intrinsic
208 // we want to lower to.
209 string IRName = NAME;
210
211 // If HasMask, this is the ID of the LLVM intrinsic we want to lower to.
212 string IRNameMask = NAME #"_mask";
Zakk Chen95c01252021-02-25 00:15:14 -0800213
214 // If non empty, this is the code emitted in the header, otherwise
215 // an automatic definition in header is emitted.
216 string HeaderCode = "";
217
Craig Topper6c6abb12021-10-21 16:02:42 -0700218 // Sub extension of vector spec. Currently only support Zvlsseg.
ShihPo Hung11072a02021-04-21 01:48:02 -0700219 string RequiredExtension = "";
220
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800221 // Number of fields for Zvlsseg.
222 int NF = 1;
Zakk Chend6a05602021-03-05 07:40:28 -0800223}
224
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800225class RVVHeader
226{
227 code HeaderCode;
228}
229
Zakk Chend6a05602021-03-05 07:40:28 -0800230//===----------------------------------------------------------------------===//
231// Basic classes with automatic codegen.
232//===----------------------------------------------------------------------===//
233
Zakk Chen5f7739b2021-04-08 08:21:06 -0700234class RVVOutBuiltin<string suffix, string prototype, string type_range>
235 : RVVBuiltin<suffix, prototype, type_range> {
236 let IntrinsicTypes = [-1];
237}
238
239class RVVOp0Builtin<string suffix, string prototype, string type_range>
240 : RVVBuiltin<suffix, prototype, type_range> {
241 let IntrinsicTypes = [0];
242}
243
Zakk Chenccc624b2021-04-06 03:26:44 -0700244class RVVOutOp1Builtin<string suffix, string prototype, string type_range>
245 : RVVBuiltin<suffix, prototype, type_range> {
246 let IntrinsicTypes = [-1, 1];
247}
248
249class RVVOutOp0Op1Builtin<string suffix, string prototype, string type_range>
250 : RVVBuiltin<suffix, prototype, type_range> {
251 let IntrinsicTypes = [-1, 0, 1];
252}
253
Zakk Chen66c05602021-03-29 07:37:29 -0700254multiclass RVVBuiltinSet<string intrinsic_name, string type_range,
255 list<list<string>> suffixes_prototypes,
256 list<int> intrinsic_types> {
257 let IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask",
258 IntrinsicTypes = intrinsic_types in {
Zakk Chend6a05602021-03-05 07:40:28 -0800259 foreach s_p = suffixes_prototypes in {
260 let Name = NAME # "_" # s_p[0] in {
261 defvar suffix = s_p[1];
262 defvar prototype = s_p[2];
Zakk Chen66c05602021-03-29 07:37:29 -0700263 def : RVVBuiltin<suffix, prototype, type_range>;
Zakk Chend6a05602021-03-05 07:40:28 -0800264 }
265 }
266 }
267}
268
Zakk Chen0a18ea02021-03-29 09:38:55 -0700269// IntrinsicTypes is output, op0, op1 [-1, 0, 1]
270multiclass RVVOutOp0Op1BuiltinSet<string intrinsic_name, string type_range,
Craig Topper5f6b3d12021-04-07 17:33:20 -0700271 list<list<string>> suffixes_prototypes>
272 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes,
Zakk Chen0a18ea02021-03-29 09:38:55 -0700273 [-1, 0, 1]>;
Zakk Chen0a18ea02021-03-29 09:38:55 -0700274
Zakk Chena8fc0e42021-04-08 08:28:15 -0700275multiclass RVVOutBuiltinSet<string intrinsic_name, string type_range,
276 list<list<string>> suffixes_prototypes>
277 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1]>;
278
Zakk Chenea5d33d2021-04-11 06:50:57 -0700279multiclass RVVOp0BuiltinSet<string intrinsic_name, string type_range,
280 list<list<string>> suffixes_prototypes>
281 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0]>;
282
Zakk Chen66c05602021-03-29 07:37:29 -0700283// IntrinsicTypes is output, op1 [-1, 1]
284multiclass RVVOutOp1BuiltinSet<string intrinsic_name, string type_range,
Craig Topper5f6b3d12021-04-07 17:33:20 -0700285 list<list<string>> suffixes_prototypes>
286 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1]>;
Zakk Chen66c05602021-03-29 07:37:29 -0700287
Zakk Chenfe252b52021-03-30 08:59:07 -0700288multiclass RVVOp0Op1BuiltinSet<string intrinsic_name, string type_range,
Craig Topper5f6b3d12021-04-07 17:33:20 -0700289 list<list<string>> suffixes_prototypes>
290 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [0, 1]>;
Zakk Chenfe252b52021-03-30 08:59:07 -0700291
292multiclass RVVOutOp1Op2BuiltinSet<string intrinsic_name, string type_range,
Craig Topper5f6b3d12021-04-07 17:33:20 -0700293 list<list<string>> suffixes_prototypes>
294 : RVVBuiltinSet<intrinsic_name, type_range, suffixes_prototypes, [-1, 1, 2]>;
Zakk Chenfe252b52021-03-30 08:59:07 -0700295
Craig Topper5f6b3d12021-04-07 17:33:20 -0700296multiclass RVVSignedBinBuiltinSet
297 : RVVOutOp1BuiltinSet<NAME, "csil",
298 [["vv", "v", "vvv"],
299 ["vx", "v", "vve"]]>;
Zakk Chen66c05602021-03-29 07:37:29 -0700300
Craig Topper5f6b3d12021-04-07 17:33:20 -0700301multiclass RVVUnsignedBinBuiltinSet
302 : RVVOutOp1BuiltinSet<NAME, "csil",
303 [["vv", "Uv", "UvUvUv"],
304 ["vx", "Uv", "UvUvUe"]]>;
Zakk Chen66c05602021-03-29 07:37:29 -0700305
Craig Topper5f6b3d12021-04-07 17:33:20 -0700306multiclass RVVIntBinBuiltinSet
307 : RVVSignedBinBuiltinSet,
308 RVVUnsignedBinBuiltinSet;
Zakk Chen66c05602021-03-29 07:37:29 -0700309
Zakk Chen59d5b8c2021-04-08 10:15:09 -0700310multiclass RVVSlideOneBuiltinSet
311 : RVVOutOp1BuiltinSet<NAME, "csil",
312 [["vx", "v", "vve"],
313 ["vx", "Uv", "UvUve"]]>;
314
Craig Topper5f6b3d12021-04-07 17:33:20 -0700315multiclass RVVSignedShiftBuiltinSet
316 : RVVOutOp1BuiltinSet<NAME, "csil",
317 [["vv", "v", "vvUv"],
318 ["vx", "v", "vvz"]]>;
Zakk Chenf2a36012021-03-30 10:12:07 -0700319
Craig Topper5f6b3d12021-04-07 17:33:20 -0700320multiclass RVVUnsignedShiftBuiltinSet
321 : RVVOutOp1BuiltinSet<NAME, "csil",
322 [["vv", "Uv", "UvUvUv"],
323 ["vx", "Uv", "UvUvz"]]>;
Zakk Chenf2a36012021-03-30 10:12:07 -0700324
Craig Topper5f6b3d12021-04-07 17:33:20 -0700325multiclass RVVShiftBuiltinSet
326 : RVVSignedShiftBuiltinSet,
327 RVVUnsignedShiftBuiltinSet;
Zakk Chenf2a36012021-03-30 10:12:07 -0700328
329let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
Craig Topper5f6b3d12021-04-07 17:33:20 -0700330 multiclass RVVSignedNShiftBuiltinSet
331 : RVVOutOp0Op1BuiltinSet<NAME, "csil",
Zakk Chenf2a36012021-03-30 10:12:07 -0700332 [["wv", "v", "vwUv"],
333 ["wx", "v", "vwz"]]>;
Craig Topper5f6b3d12021-04-07 17:33:20 -0700334 multiclass RVVUnsignedNShiftBuiltinSet
335 : RVVOutOp0Op1BuiltinSet<NAME, "csil",
Zakk Chenf2a36012021-03-30 10:12:07 -0700336 [["wv", "Uv", "UvUwUv"],
337 ["wx", "Uv", "UvUwz"]]>;
Zakk Chenf2a36012021-03-30 10:12:07 -0700338}
339
Craig Topper5f6b3d12021-04-07 17:33:20 -0700340multiclass RVVCarryinBuiltinSet
341 : RVVOutOp1BuiltinSet<NAME, "csil",
342 [["vvm", "v", "vvvm"],
343 ["vxm", "v", "vvem"],
344 ["vvm", "Uv", "UvUvUvm"],
345 ["vxm", "Uv", "UvUvUem"]]>;
Zakk Chenfe252b52021-03-30 08:59:07 -0700346
Craig Topper5f6b3d12021-04-07 17:33:20 -0700347multiclass RVVCarryOutInBuiltinSet<string intrinsic_name>
348 : RVVOp0Op1BuiltinSet<intrinsic_name, "csil",
349 [["vvm", "vm", "mvvm"],
350 ["vxm", "vm", "mvem"],
351 ["vvm", "Uvm", "mUvUvm"],
352 ["vxm", "Uvm", "mUvUem"]]>;
Zakk Chenfe252b52021-03-30 08:59:07 -0700353
Craig Topper5f6b3d12021-04-07 17:33:20 -0700354multiclass RVVSignedMaskOutBuiltinSet
355 : RVVOp0Op1BuiltinSet<NAME, "csil",
356 [["vv", "vm", "mvv"],
357 ["vx", "vm", "mve"]]>;
Zakk Chenfe252b52021-03-30 08:59:07 -0700358
Craig Topper5f6b3d12021-04-07 17:33:20 -0700359multiclass RVVUnsignedMaskOutBuiltinSet
360 : RVVOp0Op1BuiltinSet<NAME, "csil",
361 [["vv", "Uvm", "mUvUv"],
362 ["vx", "Uvm", "mUvUe"]]>;
Zakk Chenfe252b52021-03-30 08:59:07 -0700363
Craig Topper5f6b3d12021-04-07 17:33:20 -0700364multiclass RVVIntMaskOutBuiltinSet
365 : RVVSignedMaskOutBuiltinSet,
366 RVVUnsignedMaskOutBuiltinSet;
Zakk Chenfe252b52021-03-30 08:59:07 -0700367
Zakk Chen98a3ff92021-04-08 08:09:42 -0700368class RVVIntExt<string intrinsic_name, string suffix, string prototype,
369 string type_range>
370 : RVVBuiltin<suffix, prototype, type_range> {
371 let IRName = intrinsic_name;
372 let IRNameMask = intrinsic_name # "_mask";
373 let MangledName = NAME;
374 let IntrinsicTypes = [-1, 0];
375}
376
377let HasMaskedOffOperand = false in {
378 multiclass RVVIntTerBuiltinSet {
379 defm "" : RVVOutOp1BuiltinSet<NAME, "csil",
380 [["vv", "v", "vvvv"],
381 ["vx", "v", "vvev"],
382 ["vv", "Uv", "UvUvUvUv"],
383 ["vx", "Uv", "UvUvUeUv"]]>;
384 }
385 multiclass RVVFloatingTerBuiltinSet {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800386 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd",
Zakk Chen98a3ff92021-04-08 08:09:42 -0700387 [["vv", "v", "vvvv"],
388 ["vf", "v", "vvev"]]>;
389 }
390}
391
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800392let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in {
Zakk Chen98a3ff92021-04-08 08:09:42 -0700393 multiclass RVVFloatingWidenTerBuiltinSet {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800394 defm "" : RVVOutOp1Op2BuiltinSet<NAME, "xf",
Zakk Chen98a3ff92021-04-08 08:09:42 -0700395 [["vv", "w", "wwvv"],
396 ["vf", "w", "wwev"]]>;
397 }
398}
399
Craig Topper5f6b3d12021-04-07 17:33:20 -0700400multiclass RVVFloatingBinBuiltinSet
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800401 : RVVOutOp1BuiltinSet<NAME, "xfd",
Craig Topper5f6b3d12021-04-07 17:33:20 -0700402 [["vv", "v", "vvv"],
403 ["vf", "v", "vve"]]>;
Zakk Chen66c05602021-03-29 07:37:29 -0700404
Zakk Chen007ea0e2021-04-08 07:29:59 -0700405multiclass RVVFloatingBinVFBuiltinSet
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800406 : RVVOutOp1BuiltinSet<NAME, "xfd",
Zakk Chen007ea0e2021-04-08 07:29:59 -0700407 [["vf", "v", "vve"]]>;
408
Zakk Chen98a3ff92021-04-08 08:09:42 -0700409multiclass RVVFloatingMaskOutBuiltinSet
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800410 : RVVOp0Op1BuiltinSet<NAME, "xfd",
Zakk Chen98a3ff92021-04-08 08:09:42 -0700411 [["vv", "vm", "mvv"],
412 ["vf", "vm", "mve"]]>;
413
414multiclass RVVFloatingMaskOutVFBuiltinSet
415 : RVVOp0Op1BuiltinSet<NAME, "fd",
416 [["vf", "vm", "mve"]]>;
Zakk Chenf720c222021-03-30 08:55:46 -0700417
Zakk Chena8fc0e42021-04-08 08:28:15 -0700418class RVVMaskBinBuiltin : RVVOutBuiltin<"m", "mmm", "c"> {
419 let Name = NAME # "_mm";
420 let HasMask = false;
421}
422
423class RVVMaskUnaryBuiltin : RVVOutBuiltin<"m", "mm", "c"> {
424 let Name = NAME # "_m";
425}
426
427class RVVMaskNullaryBuiltin : RVVOutBuiltin<"m", "m", "c"> {
428 let Name = NAME # "_m";
429 let HasMask = false;
430 let HasNoMaskedOverloaded = false;
431}
432
433class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> {
434 let Name = NAME # "_m";
435 let HasMaskedOffOperand = false;
436}
437
Zakk Chen59d5b8c2021-04-08 10:15:09 -0700438let HasMaskedOffOperand = false in {
439 multiclass RVVSlideBuiltinSet {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800440 defm "" : RVVOutBuiltinSet<NAME, "csilxfd",
Zakk Chen59d5b8c2021-04-08 10:15:09 -0700441 [["vx","v", "vvvz"]]>;
442 defm "" : RVVOutBuiltinSet<NAME, "csil",
443 [["vx","Uv", "UvUvUvz"]]>;
444 }
445}
Zakk Chena8fc0e42021-04-08 08:28:15 -0700446
Zakk Chen5f7739b2021-04-08 08:21:06 -0700447class RVVFloatingUnaryBuiltin<string builtin_suffix, string ir_suffix,
448 string prototype>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800449 : RVVOutBuiltin<ir_suffix, prototype, "xfd"> {
Zakk Chen5f7739b2021-04-08 08:21:06 -0700450 let Name = NAME # "_" # builtin_suffix;
451}
452
453class RVVFloatingUnaryVVBuiltin : RVVFloatingUnaryBuiltin<"v", "v", "vv">;
454
Zakk Chen01fa2222021-04-01 09:21:11 -0700455class RVVConvBuiltin<string suffix, string prototype, string type_range,
456 string mangled_name>
457 : RVVBuiltin<suffix, prototype, type_range> {
458 let IntrinsicTypes = [-1, 0];
459 let MangledName = mangled_name;
460}
461
462class RVVConvToSignedBuiltin<string mangled_name>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800463 : RVVConvBuiltin<"Iv", "Ivv", "xfd", mangled_name>;
Zakk Chen01fa2222021-04-01 09:21:11 -0700464
465class RVVConvToUnsignedBuiltin<string mangled_name>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800466 : RVVConvBuiltin<"Uv", "Uvv", "xfd", mangled_name>;
Zakk Chen01fa2222021-04-01 09:21:11 -0700467
468class RVVConvToWidenSignedBuiltin<string mangled_name>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800469 : RVVConvBuiltin<"Iw", "Iwv", "xf", mangled_name>;
Zakk Chen01fa2222021-04-01 09:21:11 -0700470
471class RVVConvToWidenUnsignedBuiltin<string mangled_name>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800472 : RVVConvBuiltin<"Uw", "Uwv", "xf", mangled_name>;
Zakk Chen01fa2222021-04-01 09:21:11 -0700473
474class RVVConvToNarrowingSignedBuiltin<string mangled_name>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800475 : RVVConvBuiltin<"Iv", "IvFw", "csi", mangled_name>;
Zakk Chen01fa2222021-04-01 09:21:11 -0700476
477class RVVConvToNarrowingUnsignedBuiltin<string mangled_name>
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800478 : RVVConvBuiltin<"Uv", "UvFw", "csi", mangled_name>;
Zakk Chen01fa2222021-04-01 09:21:11 -0700479
Zakk Chenc680b0d2021-04-06 07:57:41 -0700480let HasMaskedOffOperand = false in {
481 multiclass RVVSignedReductionBuiltin {
482 defm "" : RVVOutOp1BuiltinSet<NAME, "csil",
483 [["vs", "vSv", "SvSvvSv"]]>;
484 }
485 multiclass RVVUnsignedReductionBuiltin {
486 defm "" : RVVOutOp1BuiltinSet<NAME, "csil",
487 [["vs", "UvUSv", "USvUSvUvUSv"]]>;
488 }
489 multiclass RVVFloatingReductionBuiltin {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800490 defm "" : RVVOutOp1BuiltinSet<NAME, "xfd",
Zakk Chenc680b0d2021-04-06 07:57:41 -0700491 [["vs", "vSv", "SvSvvSv"]]>;
492 }
493 multiclass RVVFloatingWidenReductionBuiltin {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800494 defm "" : RVVOutOp1BuiltinSet<NAME, "xf",
Zakk Chenc680b0d2021-04-06 07:57:41 -0700495 [["vs", "vSw", "SwSwvSw"]]>;
496 }
497}
498
499multiclass RVVIntReductionBuiltinSet
500 : RVVSignedReductionBuiltin,
501 RVVUnsignedReductionBuiltin;
502
Zakk Chenccc624b2021-04-06 03:26:44 -0700503// For widen operation which has different mangling name.
504multiclass RVVWidenBuiltinSet<string intrinsic_name, string type_range,
505 list<list<string>> suffixes_prototypes> {
506 let Log2LMUL = [-3, -2, -1, 0, 1, 2],
507 IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask" in {
508 foreach s_p = suffixes_prototypes in {
509 let Name = NAME # "_" # s_p[0],
510 MangledName = NAME # "_" # s_p[0] in {
511 defvar suffix = s_p[1];
512 defvar prototype = s_p[2];
513 def : RVVOutOp0Op1Builtin<suffix, prototype, type_range>;
514 }
515 }
516 }
517}
518
519// For widen operation with widen operand which has different mangling name.
520multiclass RVVWidenWOp0BuiltinSet<string intrinsic_name, string type_range,
521 list<list<string>> suffixes_prototypes> {
522 let Log2LMUL = [-3, -2, -1, 0, 1, 2],
523 IRName = intrinsic_name, IRNameMask = intrinsic_name # "_mask" in {
524 foreach s_p = suffixes_prototypes in {
525 let Name = NAME # "_" # s_p[0],
526 MangledName = NAME # "_" # s_p[0] in {
527 defvar suffix = s_p[1];
528 defvar prototype = s_p[2];
529 def : RVVOutOp1Builtin<suffix, prototype, type_range>;
530 }
531 }
532 }
533}
534
535multiclass RVVSignedWidenBinBuiltinSet
536 : RVVWidenBuiltinSet<NAME, "csi",
537 [["vv", "w", "wvv"],
538 ["vx", "w", "wve"]]>;
539
540multiclass RVVSignedWidenOp0BinBuiltinSet
541 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi",
542 [["wv", "w", "wwv"],
543 ["wx", "w", "wwe"]]>;
544
545multiclass RVVUnsignedWidenBinBuiltinSet
546 : RVVWidenBuiltinSet<NAME, "csi",
547 [["vv", "Uw", "UwUvUv"],
548 ["vx", "Uw", "UwUvUe"]]>;
549
550multiclass RVVUnsignedWidenOp0BinBuiltinSet
551 : RVVWidenWOp0BuiltinSet<NAME # "_w", "csi",
552 [["wv", "Uw", "UwUwUv"],
553 ["wx", "Uw", "UwUwUe"]]>;
554
Zakk Chen98a3ff92021-04-08 08:09:42 -0700555multiclass RVVFloatingWidenBinBuiltinSet
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800556 : RVVWidenBuiltinSet<NAME, "xf",
Zakk Chen98a3ff92021-04-08 08:09:42 -0700557 [["vv", "w", "wvv"],
558 ["vf", "w", "wve"]]>;
559
560multiclass RVVFloatingWidenOp0BinBuiltinSet
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800561 : RVVWidenWOp0BuiltinSet<NAME # "_w", "xf",
Zakk Chen98a3ff92021-04-08 08:09:42 -0700562 [["wv", "w", "wwv"],
563 ["wf", "w", "wwe"]]>;
564
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800565defvar TypeList = ["c","s","i","l","x","f","d"];
Zakk Chen88c2d4c2021-03-17 20:25:32 -0700566defvar EEWList = [["8", "(Log2EEW:3)"],
567 ["16", "(Log2EEW:4)"],
568 ["32", "(Log2EEW:5)"],
569 ["64", "(Log2EEW:6)"]];
570
Zakk Chenbe947ad2021-03-17 07:56:55 -0700571class IsFloat<string type> {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800572 bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d"));
Zakk Chenbe947ad2021-03-17 07:56:55 -0700573}
574
Zakk Chene5a82192021-04-11 07:25:06 -0700575let HasNoMaskedOverloaded = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800576 HasPolicy = false,
Zakk Chene5a82192021-04-11 07:25:06 -0700577 ManualCodegen = [{
578 IntrinsicTypes = {ResultType, Ops[1]->getType()};
579 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
580 }],
581 ManualCodegenMask= [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700582 // Move mask to right before vl.
583 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Zakk Chene5a82192021-04-11 07:25:06 -0700584 IntrinsicTypes = {ResultType, Ops[3]->getType()};
585 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
586 }] in {
587 class RVVVLEMaskBuiltin : RVVBuiltin<"m", "mPCUe", "c"> {
Hsiangkai Wang80a64562021-10-05 14:20:36 +0800588 let Name = "vlm_v";
589 let IRName = "vlm";
Zakk Chene5a82192021-04-11 07:25:06 -0700590 let HasMask = false;
Zakk Chenbe947ad2021-03-17 07:56:55 -0700591 }
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800592}
593
594let HasNoMaskedOverloaded = false,
595 ManualCodegen = [{
596 IntrinsicTypes = {ResultType, Ops[1]->getType()};
597 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
598 }],
599 ManualCodegenMask= [{
600 // Move mask to right before vl.
601 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
602 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
603 IntrinsicTypes = {ResultType, Ops[3]->getType()};
604 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
605 }] in {
Zakk Chene5a82192021-04-11 07:25:06 -0700606 multiclass RVVVLEBuiltin<list<string> types> {
607 let Name = NAME # "_v",
608 IRName = "vle",
609 IRNameMask ="vle_mask" in {
610 foreach type = types in {
611 def : RVVBuiltin<"v", "vPCe", type>;
612 if !not(IsFloat<type>.val) then {
613 def : RVVBuiltin<"Uv", "UvPCUe", type>;
Zakk Chen88c2d4c2021-03-17 20:25:32 -0700614 }
615 }
Zakk Chene5a82192021-04-11 07:25:06 -0700616 }
Zakk Chen88c2d4c2021-03-17 20:25:32 -0700617 }
618}
619
Hsiangkai Wang471ae422021-04-10 07:02:08 +0800620multiclass RVVVLEFFBuiltin<list<string> types> {
621 let Name = NAME # "_v",
622 IRName = "vleff",
623 IRNameMask = "vleff_mask",
624 HasNoMaskedOverloaded = false,
625 ManualCodegen = [{
626 {
627 IntrinsicTypes = {ResultType, Ops[2]->getType()};
628 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
629 Value *NewVL = Ops[1];
630 Ops.erase(Ops.begin() + 1);
631 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
632 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
633 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0});
634 // Store new_vl.
635 clang::CharUnits Align =
Craig Topper6171f842021-08-12 08:52:20 -0700636 CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType());
Hsiangkai Wang471ae422021-04-10 07:02:08 +0800637 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {1}),
638 Address(NewVL, Align));
639 return V;
640 }
641 }],
642 ManualCodegenMask = [{
643 {
Craig Toppercfe3b002021-05-01 13:18:21 -0700644 // Move mask to right before vl.
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +0800645 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800646 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wang471ae422021-04-10 07:02:08 +0800647 IntrinsicTypes = {ResultType, Ops[4]->getType()};
648 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
649 Value *NewVL = Ops[2];
650 Ops.erase(Ops.begin() + 2);
651 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
652 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
653 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0});
654 // Store new_vl.
655 clang::CharUnits Align =
Craig Topper6171f842021-08-12 08:52:20 -0700656 CGM.getNaturalPointeeTypeAlignment(E->getArg(3)->getType());
Hsiangkai Wang471ae422021-04-10 07:02:08 +0800657 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {1}),
658 Address(NewVL, Align));
659 return V;
660 }
661 }] in {
662 foreach type = types in {
663 def : RVVBuiltin<"v", "vPCePz", type>;
664 // Skip floating types for unsigned versions.
665 if !not(IsFloat<type>.val) then {
666 def : RVVBuiltin<"Uv", "UvPCUePz", type>;
667 }
668 }
669 }
670}
671
Zakk Chene5a82192021-04-11 07:25:06 -0700672multiclass RVVVLSEBuiltin<list<string> types> {
Zakk Chenbe947ad2021-03-17 07:56:55 -0700673 let Name = NAME # "_v",
Zakk Chene5a82192021-04-11 07:25:06 -0700674 IRName = "vlse",
675 IRNameMask ="vlse_mask",
676 HasNoMaskedOverloaded = false,
677 ManualCodegen = [{
678 IntrinsicTypes = {ResultType, Ops[2]->getType()};
679 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
680 }],
681 ManualCodegenMask= [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700682 // Move mask to right before vl.
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +0800683 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800684 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Zakk Chene5a82192021-04-11 07:25:06 -0700685 IntrinsicTypes = {ResultType, Ops[4]->getType()};
686 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
687 }] in {
688 foreach type = types in {
689 def : RVVBuiltin<"v", "vPCet", type>;
690 if !not(IsFloat<type>.val) then {
691 def : RVVBuiltin<"Uv", "UvPCUet", type>;
692 }
693 }
694 }
695}
696
697multiclass RVVIndexedLoad<string op> {
698 let ManualCodegen = [{
699 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType()};
700 Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
701 }],
702 ManualCodegenMask = [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700703 // Move mask to right before vl.
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +0800704 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800705 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Zakk Chene5a82192021-04-11 07:25:06 -0700706 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops[4]->getType()};
707 Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
708 }] in {
709 foreach type = TypeList in {
710 foreach eew_list = EEWList in {
711 defvar eew = eew_list[0];
712 defvar eew_type = eew_list[1];
713 let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in {
714 def: RVVBuiltin<"v", "vPCe" # eew_type # "Uv", type>;
715 if !not(IsFloat<type>.val) then {
716 def: RVVBuiltin<"Uv", "UvPCUe" # eew_type # "Uv", type>;
717 }
718 }
719 }
720 }
721 }
722}
723
724let HasMaskedOffOperand = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800725 HasPolicy = false,
Zakk Chene5a82192021-04-11 07:25:06 -0700726 ManualCodegen = [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700727 // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl)
728 std::swap(Ops[0], Ops[1]);
Zakk Chene5a82192021-04-11 07:25:06 -0700729 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
730 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()};
731 }],
732 ManualCodegenMask= [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700733 // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl)
734 std::swap(Ops[0], Ops[2]);
Zakk Chene5a82192021-04-11 07:25:06 -0700735 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
736 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
737 }] in {
Craig Toppercfe3b002021-05-01 13:18:21 -0700738 class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> {
Hsiangkai Wang80a64562021-10-05 14:20:36 +0800739 let Name = "vsm_v";
740 let IRName = "vsm";
Zakk Chene5a82192021-04-11 07:25:06 -0700741 let HasMask = false;
742 }
743 multiclass RVVVSEBuiltin<list<string> types> {
744 let Name = NAME # "_v",
745 IRName = "vse",
746 IRNameMask = "vse_mask" in {
747 foreach type = types in {
Craig Toppercfe3b002021-05-01 13:18:21 -0700748 def : RVVBuiltin<"v", "0Pev", type>;
Zakk Chene5a82192021-04-11 07:25:06 -0700749 if !not(IsFloat<type>.val) then {
Craig Toppercfe3b002021-05-01 13:18:21 -0700750 def : RVVBuiltin<"Uv", "0PUeUv", type>;
Zakk Chene5a82192021-04-11 07:25:06 -0700751 }
752 }
753 }
754 }
755}
756
757multiclass RVVVSSEBuiltin<list<string> types> {
758 let Name = NAME # "_v",
759 IRName = "vsse",
760 IRNameMask = "vsse_mask",
Zakk Chenbe947ad2021-03-17 07:56:55 -0700761 HasMaskedOffOperand = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800762 HasPolicy = false,
Zakk Chenbe947ad2021-03-17 07:56:55 -0700763 ManualCodegen = [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700764 // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl)
765 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
Zakk Chenbe947ad2021-03-17 07:56:55 -0700766 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
Zakk Chene5a82192021-04-11 07:25:06 -0700767 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
Zakk Chenbe947ad2021-03-17 07:56:55 -0700768 }],
769 ManualCodegenMask= [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700770 // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl)
771 std::swap(Ops[0], Ops[3]);
Zakk Chenbe947ad2021-03-17 07:56:55 -0700772 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
Zakk Chene5a82192021-04-11 07:25:06 -0700773 IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()};
Zakk Chenbe947ad2021-03-17 07:56:55 -0700774 }] in {
775 foreach type = types in {
Craig Toppercfe3b002021-05-01 13:18:21 -0700776 def : RVVBuiltin<"v", "0Petv", type>;
Zakk Chenbe947ad2021-03-17 07:56:55 -0700777 if !not(IsFloat<type>.val) then {
Craig Toppercfe3b002021-05-01 13:18:21 -0700778 def : RVVBuiltin<"Uv", "0PUetUv", type>;
Zakk Chenbe947ad2021-03-17 07:56:55 -0700779 }
780 }
781 }
782}
Zakk Chen95c01252021-02-25 00:15:14 -0800783
Zakk Chene5a82192021-04-11 07:25:06 -0700784multiclass RVVIndexedStore<string op> {
785 let HasMaskedOffOperand = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800786 HasPolicy = false,
Zakk Chene5a82192021-04-11 07:25:06 -0700787 ManualCodegen = [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700788 // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl)
789 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
Zakk Chene5a82192021-04-11 07:25:06 -0700790 Ops[1] = Builder.CreateBitCast(Ops[1],Ops[0]->getType()->getPointerTo());
791 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()};
792 }],
793 ManualCodegenMask= [{
Craig Toppercfe3b002021-05-01 13:18:21 -0700794 // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl)
795 std::swap(Ops[0], Ops[3]);
Zakk Chene5a82192021-04-11 07:25:06 -0700796 Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
797 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()};
798 }] in {
799 foreach type = TypeList in {
800 foreach eew_list = EEWList in {
801 defvar eew = eew_list[0];
802 defvar eew_type = eew_list[1];
803 let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in {
Craig Toppercfe3b002021-05-01 13:18:21 -0700804 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>;
Zakk Chene5a82192021-04-11 07:25:06 -0700805 if !not(IsFloat<type>.val) then {
Craig Toppercfe3b002021-05-01 13:18:21 -0700806 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>;
Zakk Chene5a82192021-04-11 07:25:06 -0700807 }
808 }
809 }
810 }
811 }
812}
813
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800814defvar NFList = [2, 3, 4, 5, 6, 7, 8];
815
816class PVString<int nf, bit signed> {
817 string S =
818 !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"),
819 !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"),
820 !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"),
821 !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"),
822 !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"),
823 !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"),
824 !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv"));
825}
826
827multiclass RVVUnitStridedSegLoad<string op> {
828 foreach type = TypeList in {
829 defvar eew = !cond(!eq(type, "c") : "8",
830 !eq(type, "s") : "16",
831 !eq(type, "i") : "32",
832 !eq(type, "l") : "64",
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800833 !eq(type, "x") : "16",
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800834 !eq(type, "f") : "32",
835 !eq(type, "d") : "64");
836 foreach nf = NFList in {
837 let Name = op # nf # "e" # eew # "_v",
838 IRName = op # nf,
839 IRNameMask = op # nf # "_mask",
840 NF = nf,
841 HasNoMaskedOverloaded = false,
842 ManualCodegen = [{
843 {
844 // builtin: (val0 address, val1 address, ..., ptr, vl)
Craig Topper6171f842021-08-12 08:52:20 -0700845 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800846 Ops[NF + 1]->getType()};
847 // intrinsic: (ptr, vl)
848 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1]};
849 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
850 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -0700851 clang::CharUnits Align =
852 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800853 llvm::Value *V;
854 for (unsigned I = 0; I < NF; ++I) {
855 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
856 Address(Ops[I], Align));
857 }
858 return V;
859 }
860 }],
861 ManualCodegenMask = [{
862 {
863 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl)
864 // intrinsic: (maskedoff0, ..., ptr, mask, vl)
Craig Topper6171f842021-08-12 08:52:20 -0700865 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800866 Ops[2 * NF + 2]->getType()};
867 SmallVector<llvm::Value*, 12> Operands;
868 for (unsigned I = 0; I < NF; ++I)
869 Operands.push_back(Ops[NF + I + 1]);
870 Operands.push_back(Ops[2 * NF + 1]);
871 Operands.push_back(Ops[NF]);
872 Operands.push_back(Ops[2 * NF + 2]);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800873 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
874 assert(Operands.size() == NF + 4);
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800875 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
876 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -0700877 clang::CharUnits Align =
878 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800879 llvm::Value *V;
880 for (unsigned I = 0; I < NF; ++I) {
881 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
882 Address(Ops[I], Align));
883 }
884 return V;
885 }
886 }] in {
887 defvar PV = PVString<nf, /*signed=*/true>.S;
888 defvar PUV = PVString<nf, /*signed=*/false>.S;
889 def : RVVBuiltin<"v", "0" # PV # "PCe", type>;
890 if !not(IsFloat<type>.val) then {
891 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe", type>;
892 }
893 }
894 }
895 }
896}
897
898multiclass RVVUnitStridedSegLoadFF<string op> {
899 foreach type = TypeList in {
900 defvar eew = !cond(!eq(type, "c") : "8",
901 !eq(type, "s") : "16",
902 !eq(type, "i") : "32",
903 !eq(type, "l") : "64",
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +0800904 !eq(type, "x") : "16",
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800905 !eq(type, "f") : "32",
906 !eq(type, "d") : "64");
907 foreach nf = NFList in {
908 let Name = op # nf # "e" # eew # "ff_v",
909 IRName = op # nf # "ff",
910 IRNameMask = op # nf # "ff_mask",
911 NF = nf,
912 HasNoMaskedOverloaded = false,
913 ManualCodegen = [{
914 {
915 // builtin: (val0 address, val1 address, ..., ptr, new_vl, vl)
Craig Topper6171f842021-08-12 08:52:20 -0700916 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800917 Ops[NF + 2]->getType()};
918 // intrinsic: (ptr, vl)
919 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 2]};
920 Value *NewVL = Ops[NF + 1];
921 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
922 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -0700923 clang::CharUnits Align =
924 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800925 for (unsigned I = 0; I < NF; ++I) {
926 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
927 Address(Ops[I], Align));
928 }
929 // Store new_vl.
930 return Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {NF}),
931 Address(NewVL, Align));
932 }
933 }],
934 ManualCodegenMask = [{
935 {
936 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl)
937 // intrinsic: (maskedoff0, ..., ptr, mask, vl)
Craig Topper6171f842021-08-12 08:52:20 -0700938 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800939 Ops[2 * NF + 3]->getType()};
940 SmallVector<llvm::Value*, 12> Operands;
941 for (unsigned I = 0; I < NF; ++I)
942 Operands.push_back(Ops[NF + I + 1]);
943 Operands.push_back(Ops[2 * NF + 1]);
944 Operands.push_back(Ops[NF]);
945 Operands.push_back(Ops[2 * NF + 3]);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800946 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800947 Value *NewVL = Ops[2 * NF + 2];
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +0800948 assert(Operands.size() == NF + 4);
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800949 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
950 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -0700951 clang::CharUnits Align =
952 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +0800953 for (unsigned I = 0; I < NF; ++I) {
954 Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
955 Address(Ops[I], Align));
956 }
957 // Store new_vl.
958 return Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {NF}),
959 Address(NewVL, Align));
960 }
961 }] in {
962 defvar PV = PVString<nf, /*signed=*/true>.S;
963 defvar PUV = PVString<nf, /*signed=*/false>.S;
964 def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>;
965 if !not(IsFloat<type>.val) then {
966 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "Pz", type>;
967 }
968 }
969 }
970 }
971}
972
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +0800973multiclass RVVStridedSegLoad<string op> {
974 foreach type = TypeList in {
975 defvar eew = !cond(!eq(type, "c") : "8",
976 !eq(type, "s") : "16",
977 !eq(type, "i") : "32",
978 !eq(type, "l") : "64",
979 !eq(type, "x") : "16",
980 !eq(type, "f") : "32",
981 !eq(type, "d") : "64");
982 foreach nf = NFList in {
983 let Name = op # nf # "e" # eew # "_v",
984 IRName = op # nf,
985 IRNameMask = op # nf # "_mask",
986 NF = nf,
987 HasNoMaskedOverloaded = false,
988 ManualCodegen = [{
989 {
990 // builtin: (val0 address, val1 address, ..., ptr, stride, vl)
Craig Topper6171f842021-08-12 08:52:20 -0700991 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +0800992 Ops[NF + 2]->getType()};
993 // intrinsic: (ptr, stride, vl)
994 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1], Ops[NF + 2]};
995 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
996 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -0700997 clang::CharUnits Align =
998 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +0800999 llvm::Value *V;
1000 for (unsigned I = 0; I < NF; ++I) {
1001 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
1002 Address(Ops[I], Align));
1003 }
1004 return V;
1005 }
1006 }],
1007 ManualCodegenMask = [{
1008 {
1009 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl)
1010 // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl)
Craig Topper6171f842021-08-12 08:52:20 -07001011 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +08001012 Ops[2 * NF + 3]->getType()};
1013 SmallVector<llvm::Value*, 12> Operands;
1014 for (unsigned I = 0; I < NF; ++I)
1015 Operands.push_back(Ops[NF + I + 1]);
1016 Operands.push_back(Ops[2 * NF + 1]);
1017 Operands.push_back(Ops[2 * NF + 2]);
1018 Operands.push_back(Ops[NF]);
1019 Operands.push_back(Ops[2 * NF + 3]);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001020 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
1021 assert(Operands.size() == NF + 5);
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +08001022 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
1023 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -07001024 clang::CharUnits Align =
1025 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +08001026 llvm::Value *V;
1027 for (unsigned I = 0; I < NF; ++I) {
1028 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
1029 Address(Ops[I], Align));
1030 }
1031 return V;
1032 }
1033 }] in {
1034 defvar PV = PVString<nf, /*signed=*/true>.S;
1035 defvar PUV = PVString<nf, /*signed=*/false>.S;
1036 def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>;
1037 if !not(IsFloat<type>.val) then {
1038 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # "t", type>;
1039 }
1040 }
1041 }
1042 }
1043}
1044
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001045multiclass RVVIndexedSegLoad<string op> {
1046 foreach type = TypeList in {
1047 foreach eew_info = EEWList in {
1048 defvar eew = eew_info[0];
1049 defvar eew_type = eew_info[1];
1050 foreach nf = NFList in {
1051 let Name = op # nf # "ei" # eew # "_v",
1052 IRName = op # nf,
1053 IRNameMask = op # nf # "_mask",
1054 NF = nf,
1055 ManualCodegen = [{
1056 {
1057 // builtin: (val0 address, val1 address, ..., ptr, index, vl)
Craig Topper6171f842021-08-12 08:52:20 -07001058 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001059 Ops[NF + 1]->getType(), Ops[NF + 2]->getType()};
1060 // intrinsic: (ptr, index, vl)
1061 llvm::Value *Operands[] = {Ops[NF], Ops[NF + 1], Ops[NF + 2]};
1062 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
1063 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -07001064 clang::CharUnits Align =
1065 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001066 llvm::Value *V;
1067 for (unsigned I = 0; I < NF; ++I) {
1068 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
1069 Address(Ops[I], Align));
1070 }
1071 return V;
1072 }
1073 }],
1074 ManualCodegenMask = [{
1075 {
1076 // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl)
Craig Topper6171f842021-08-12 08:52:20 -07001077 IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001078 Ops[2 * NF + 2]->getType(), Ops[2 * NF + 3]->getType()};
1079 // intrinsic: (maskedoff0, ..., ptr, index, mask, vl)
1080 SmallVector<llvm::Value*, 12> Operands;
1081 for (unsigned I = 0; I < NF; ++I)
1082 Operands.push_back(Ops[NF + I + 1]);
1083 Operands.push_back(Ops[2 * NF + 1]);
1084 Operands.push_back(Ops[2 * NF + 2]);
1085 Operands.push_back(Ops[NF]);
1086 Operands.push_back(Ops[2 * NF + 3]);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001087 Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
1088 assert(Operands.size() == NF + 5);
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001089 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
1090 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
Craig Topper6171f842021-08-12 08:52:20 -07001091 clang::CharUnits Align =
1092 CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001093 llvm::Value *V;
1094 for (unsigned I = 0; I < NF; ++I) {
1095 V = Builder.CreateStore(Builder.CreateExtractValue(LoadValue, {I}),
1096 Address(Ops[I], Align));
1097 }
1098 return V;
1099 }
1100 }] in {
1101 defvar PV = PVString<nf, /*signed=*/true>.S;
1102 defvar PUV = PVString<nf, /*signed=*/false>.S;
1103 def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>;
1104 if !not(IsFloat<type>.val) then {
1105 def : RVVBuiltin<"Uv", "0" # PUV # "PCUe" # eew_type # "Uv", type>;
1106 }
1107 }
1108 }
1109 }
1110 }
1111}
1112
Hsiangkai Wangd1a401b2021-06-08 13:09:07 +08001113class VString<int nf, bit signed> {
1114 string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"),
1115 !eq(nf, 3): !if(signed, "vvv", "UvUvUv"),
1116 !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"),
1117 !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"),
1118 !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"),
1119 !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"),
1120 !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv"));
1121}
1122
1123multiclass RVVUnitStridedSegStore<string op> {
1124 foreach type = TypeList in {
1125 defvar eew = !cond(!eq(type, "c") : "8",
1126 !eq(type, "s") : "16",
1127 !eq(type, "i") : "32",
1128 !eq(type, "l") : "64",
1129 !eq(type, "x") : "16",
1130 !eq(type, "f") : "32",
1131 !eq(type, "d") : "64");
1132 foreach nf = NFList in {
1133 let Name = op # nf # "e" # eew # "_v",
1134 IRName = op # nf,
1135 IRNameMask = op # nf # "_mask",
1136 NF = nf,
1137 HasMaskedOffOperand = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001138 HasPolicy = false,
Hsiangkai Wangd1a401b2021-06-08 13:09:07 +08001139 ManualCodegen = [{
1140 {
1141 // Builtin: (ptr, val0, val1, ..., vl)
1142 // Intrinsic: (val0, val1, ..., ptr, vl)
1143 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
1144 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
1145 assert(Ops.size() == NF + 2);
1146 }
1147 }],
1148 ManualCodegenMask = [{
1149 {
1150 // Builtin: (mask, ptr, val0, val1, ..., vl)
1151 // Intrinsic: (val0, val1, ..., ptr, mask, vl)
1152 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
1153 std::swap(Ops[NF], Ops[NF + 1]);
1154 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()};
1155 assert(Ops.size() == NF + 3);
1156 }
1157 }] in {
1158 defvar V = VString<nf, /*signed=*/true>.S;
1159 defvar UV = VString<nf, /*signed=*/false>.S;
1160 def : RVVBuiltin<"v", "0Pe" # V, type>;
1161 if !not(IsFloat<type>.val) then {
1162 def : RVVBuiltin<"Uv", "0PUe" # UV, type>;
1163 }
1164 }
1165 }
1166 }
1167}
1168
Hsiangkai Wang915e6dc2021-06-08 13:29:51 +08001169multiclass RVVStridedSegStore<string op> {
1170 foreach type = TypeList in {
1171 defvar eew = !cond(!eq(type, "c") : "8",
1172 !eq(type, "s") : "16",
1173 !eq(type, "i") : "32",
1174 !eq(type, "l") : "64",
1175 !eq(type, "x") : "16",
1176 !eq(type, "f") : "32",
1177 !eq(type, "d") : "64");
1178 foreach nf = NFList in {
1179 let Name = op # nf # "e" # eew # "_v",
1180 IRName = op # nf,
1181 IRNameMask = op # nf # "_mask",
1182 NF = nf,
1183 HasMaskedOffOperand = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001184 HasPolicy = false,
Hsiangkai Wang915e6dc2021-06-08 13:29:51 +08001185 ManualCodegen = [{
1186 {
1187 // Builtin: (ptr, stride, val0, val1, ..., vl).
1188 // Intrinsic: (val0, val1, ..., ptr, stride, vl)
1189 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
1190 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
1191 assert(Ops.size() == NF + 3);
1192 }
1193 }],
1194 ManualCodegenMask = [{
1195 {
1196 // Builtin: (mask, ptr, stride, val0, val1, ..., vl).
1197 // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl)
1198 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1);
1199 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3);
1200 IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
1201 assert(Ops.size() == NF + 4);
1202 }
1203 }] in {
1204 defvar V = VString<nf, /*signed=*/true>.S;
1205 defvar UV = VString<nf, /*signed=*/false>.S;
1206 def : RVVBuiltin<"v", "0Pet" # V, type>;
1207 if !not(IsFloat<type>.val) then {
1208 def : RVVBuiltin<"Uv", "0PUet" # UV, type>;
1209 }
1210 }
1211 }
1212 }
1213}
1214
Hsiangkai Wang698f2882021-06-08 14:29:40 +08001215multiclass RVVIndexedSegStore<string op> {
1216 foreach type = TypeList in {
1217 foreach eew_info = EEWList in {
1218 defvar eew = eew_info[0];
1219 defvar eew_type = eew_info[1];
1220 foreach nf = NFList in {
1221 let Name = op # nf # "ei" # eew # "_v",
1222 IRName = op # nf,
1223 IRNameMask = op # nf # "_mask",
1224 NF = nf,
1225 HasMaskedOffOperand = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001226 HasPolicy = false,
Hsiangkai Wang698f2882021-06-08 14:29:40 +08001227 ManualCodegen = [{
1228 {
1229 // Builtin: (ptr, index, val0, val1, ..., vl)
1230 // Intrinsic: (val0, val1, ..., ptr, index, vl)
1231 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
1232 IntrinsicTypes = {Ops[0]->getType(),
1233 Ops[NF + 1]->getType(), Ops[NF + 2]->getType()};
1234 assert(Ops.size() == NF + 3);
1235 }
1236 }],
1237 ManualCodegenMask = [{
1238 {
1239 // Builtin: (mask, ptr, index, val0, val1, ..., vl)
1240 // Intrinsic: (val0, val1, ..., ptr, index, mask, vl)
1241 std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1);
1242 std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3);
1243 IntrinsicTypes = {Ops[0]->getType(),
1244 Ops[NF + 1]->getType(), Ops[NF + 3]->getType()};
1245 assert(Ops.size() == NF + 4);
1246 }
1247 }] in {
1248 defvar V = VString<nf, /*signed=*/true>.S;
1249 defvar UV = VString<nf, /*signed=*/false>.S;
1250 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uv" # V, type>;
1251 if !not(IsFloat<type>.val) then {
1252 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "Uv" # UV, type>;
1253 }
1254 }
1255 }
1256 }
1257 }
1258}
1259
Hsiangkai Wang14cc1cb22021-04-20 10:53:19 +08001260multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> {
1261 let Name = NAME,
1262 IRName = IR,
1263 IRNameMask = IR # "_mask",
1264 ManualCodegen = [{
1265 {
1266 // op1, vl
1267 IntrinsicTypes = {ResultType,
1268 cast<llvm::VectorType>(ResultType)->getElementType(),
1269 Ops[1]->getType()};
1270 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[1]));
1271 break;
1272 }
1273 }],
1274 ManualCodegenMask = [{
1275 {
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +08001276 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001277 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wang14cc1cb22021-04-20 10:53:19 +08001278 // maskedoff, op1, mask, vl
1279 IntrinsicTypes = {ResultType,
1280 cast<llvm::VectorType>(ResultType)->getElementType(),
1281 Ops[3]->getType()};
1282 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[1]));
1283 break;
1284 }
1285 }] in {
1286 def : RVVBuiltin<"v", "vv", type_range>;
1287 }
1288}
1289
Hsiangkai Wang43cd5882021-04-20 11:01:22 +08001290multiclass RVVPseudoVNotBuiltin<string IR, string type_range> {
1291 let Name = NAME,
1292 IRName = IR,
1293 IRNameMask = IR # "_mask",
1294 ManualCodegen = [{
1295 {
1296 // op1, vl
1297 IntrinsicTypes = {ResultType,
1298 cast<llvm::VectorType>(ResultType)->getElementType(),
1299 Ops[1]->getType()};
1300 Ops.insert(Ops.begin() + 1,
1301 llvm::Constant::getAllOnesValue(IntrinsicTypes[1]));
1302 break;
1303 }
1304 }],
1305 ManualCodegenMask = [{
1306 {
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +08001307 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001308 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wang43cd5882021-04-20 11:01:22 +08001309 // maskedoff, op1, mask, vl
1310 IntrinsicTypes = {ResultType,
1311 cast<llvm::VectorType>(ResultType)->getElementType(),
1312 Ops[3]->getType()};
1313 Ops.insert(Ops.begin() + 2,
1314 llvm::Constant::getAllOnesValue(IntrinsicTypes[1]));
1315 break;
1316 }
1317 }] in {
1318 def : RVVBuiltin<"v", "vv", type_range>;
1319 def : RVVBuiltin<"Uv", "UvUv", type_range>;
1320 }
1321}
1322
Hsiangkai Wang4b243412021-04-20 11:12:53 +08001323multiclass RVVPseudoMaskBuiltin<string IR, string type_range> {
1324 let Name = NAME,
1325 IRName = IR,
1326 HasMask = false,
1327 ManualCodegen = [{
1328 {
1329 // op1, vl
1330 IntrinsicTypes = {ResultType,
1331 Ops[1]->getType()};
1332 Ops.insert(Ops.begin() + 1, Ops[0]);
1333 break;
1334 }
1335 }] in {
1336 def : RVVBuiltin<"m", "mm", type_range>;
1337 }
1338}
1339
Hsiangkai Wangbfb3fca2021-04-20 13:17:40 +08001340multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> {
1341 let Name = NAME,
1342 IRName = IR,
1343 IRNameMask = IR # "_mask",
1344 ManualCodegen = [{
1345 {
1346 // op1, vl
1347 IntrinsicTypes = {ResultType,
1348 Ops[0]->getType(), Ops[1]->getType()};
1349 Ops.insert(Ops.begin() + 1, Ops[0]);
1350 break;
1351 }
1352 }],
1353 ManualCodegenMask = [{
1354 {
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +08001355 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001356 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wangbfb3fca2021-04-20 13:17:40 +08001357 // maskedoff, op1, mask, vl
1358 IntrinsicTypes = {ResultType,
1359 Ops[1]->getType(),
1360 Ops[3]->getType()};
1361 Ops.insert(Ops.begin() + 2, Ops[1]);
1362 break;
1363 }
1364 }] in {
1365 def : RVVBuiltin<"v", "vv", type_range>;
1366 }
1367}
1368
Hsiangkai Wangbd32c2d2021-04-20 15:31:13 +08001369multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range,
1370 list<list<string>> suffixes_prototypes> {
1371 let Name = NAME,
1372 MangledName = MName,
1373 IRName = IR,
1374 IRNameMask = IR # "_mask",
1375 ManualCodegen = [{
1376 {
1377 // op1, vl
1378 IntrinsicTypes = {ResultType,
1379 Ops[0]->getType(),
1380 cast<llvm::VectorType>(Ops[0]->getType())->getElementType(),
1381 Ops[1]->getType()};
1382 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2]));
1383 break;
1384 }
1385 }],
1386 ManualCodegenMask = [{
1387 {
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +08001388 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001389 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wangbd32c2d2021-04-20 15:31:13 +08001390 // maskedoff, op1, mask, vl
1391 IntrinsicTypes = {ResultType,
1392 Ops[1]->getType(),
1393 cast<llvm::VectorType>(Ops[1]->getType())->getElementType(),
1394 Ops[3]->getType()};
1395 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2]));
1396 break;
1397 }
1398 }] in {
1399 foreach s_p = suffixes_prototypes in {
1400 def : RVVBuiltin<s_p[0], s_p[1], type_range>;
1401 }
1402 }
1403}
1404
1405multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range,
1406 list<list<string>> suffixes_prototypes> {
1407 let Name = NAME,
1408 MangledName = MName,
1409 IRName = IR,
1410 IRNameMask = IR # "_mask",
1411 ManualCodegen = [{
1412 {
1413 // op1, vl
1414 IntrinsicTypes = {ResultType,
1415 Ops[0]->getType(),
1416 Ops[1]->getType(),
1417 Ops[1]->getType()};
1418 Ops.insert(Ops.begin() + 1, llvm::Constant::getNullValue(IntrinsicTypes[2]));
1419 break;
1420 }
1421 }],
1422 ManualCodegenMask = [{
1423 {
Hsiangkai Wang7ccd31c2021-10-13 14:50:45 +08001424 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001425 Ops.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
Hsiangkai Wangbd32c2d2021-04-20 15:31:13 +08001426 // maskedoff, op1, mask, vl
1427 IntrinsicTypes = {ResultType,
1428 Ops[1]->getType(),
1429 Ops[3]->getType(),
1430 Ops[3]->getType()};
1431 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(IntrinsicTypes[2]));
1432 break;
1433 }
1434 }] in {
1435 foreach s_p = suffixes_prototypes in {
1436 def : RVVBuiltin<s_p[0], s_p[1], type_range>;
1437 }
1438 }
1439}
1440
Zakk Chen95c01252021-02-25 00:15:14 -08001441// 6. Configuration-Setting Instructions
1442// 6.1. vsetvli/vsetvl instructions
1443let HasVL = false,
1444 HasMask = false,
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001445 HasPolicy = false,
Zakk Chen95c01252021-02-25 00:15:14 -08001446 Log2LMUL = [0],
1447 ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type
1448{
1449 // vsetvl is a macro because for it require constant integers in SEW and LMUL.
1450 let HeaderCode =
1451[{
1452#define vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5)
1453#define vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6)
1454#define vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7)
1455#define vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0)
1456#define vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1)
1457#define vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2)
1458#define vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3)
1459
1460#define vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6)
1461#define vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7)
1462#define vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0)
1463#define vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1)
1464#define vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2)
1465#define vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3)
1466
1467#define vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7)
1468#define vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0)
1469#define vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1)
1470#define vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2)
1471#define vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3)
1472
1473#define vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0)
1474#define vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1)
1475#define vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2)
1476#define vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3)
1477
1478}] in
1479 def vsetvli : RVVBuiltin<"", "zzKzKz", "i">;
1480
1481 let HeaderCode =
1482[{
1483#define vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5)
1484#define vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6)
1485#define vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7)
1486#define vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0)
1487#define vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1)
1488#define vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2)
1489#define vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3)
1490
1491#define vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6)
1492#define vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7)
1493#define vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0)
1494#define vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1)
1495#define vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2)
1496#define vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3)
1497
1498#define vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7)
1499#define vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0)
1500#define vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1)
1501#define vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2)
1502#define vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3)
1503
1504#define vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0)
1505#define vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1)
1506#define vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2)
1507#define vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3)
1508
1509}] in
1510 def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">;
1511}
1512
Zakk Chenbe947ad2021-03-17 07:56:55 -07001513// 7. Vector Loads and Stores
1514// 7.4. Vector Unit-Stride Instructions
Hsiangkai Wang80a64562021-10-05 14:20:36 +08001515def vlm: RVVVLEMaskBuiltin;
Zakk Chenbe947ad2021-03-17 07:56:55 -07001516defm vle8: RVVVLEBuiltin<["c"]>;
Hsiangkai Wang89ce64492021-07-20 11:36:48 +08001517defm vle16: RVVVLEBuiltin<["s","x"]>;
Zakk Chenbe947ad2021-03-17 07:56:55 -07001518defm vle32: RVVVLEBuiltin<["i","f"]>;
1519defm vle64: RVVVLEBuiltin<["l","d"]>;
1520
Hsiangkai Wang80a64562021-10-05 14:20:36 +08001521def vsm : RVVVSEMaskBuiltin;
Zakk Chenbe947ad2021-03-17 07:56:55 -07001522defm vse8 : RVVVSEBuiltin<["c"]>;
Hsiangkai Wang89ce64492021-07-20 11:36:48 +08001523defm vse16: RVVVSEBuiltin<["s","x"]>;
Zakk Chenbe947ad2021-03-17 07:56:55 -07001524defm vse32: RVVVSEBuiltin<["i","f"]>;
1525defm vse64: RVVVSEBuiltin<["l","d"]>;
1526
Zakk Chene5a82192021-04-11 07:25:06 -07001527// 7.5. Vector Strided Instructions
1528defm vlse8: RVVVLSEBuiltin<["c"]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001529defm vlse16: RVVVLSEBuiltin<["s","x"]>;
Zakk Chene5a82192021-04-11 07:25:06 -07001530defm vlse32: RVVVLSEBuiltin<["i","f"]>;
1531defm vlse64: RVVVLSEBuiltin<["l","d"]>;
1532
1533defm vsse8 : RVVVSSEBuiltin<["c"]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001534defm vsse16: RVVVSSEBuiltin<["s","x"]>;
Zakk Chene5a82192021-04-11 07:25:06 -07001535defm vsse32: RVVVSSEBuiltin<["i","f"]>;
1536defm vsse64: RVVVSSEBuiltin<["l","d"]>;
1537
Zakk Chen88c2d4c2021-03-17 20:25:32 -07001538// 7.6. Vector Indexed Instructions
1539defm : RVVIndexedLoad<"vluxei">;
1540defm : RVVIndexedLoad<"vloxei">;
1541
Zakk Chene5a82192021-04-11 07:25:06 -07001542defm : RVVIndexedStore<"vsuxei">;
1543defm : RVVIndexedStore<"vsoxei">;
1544
1545// 7.7. Unit-stride Fault-Only-First Loads
1546defm vle8ff: RVVVLEFFBuiltin<["c"]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001547defm vle16ff: RVVVLEFFBuiltin<["s","x"]>;
Zakk Chene5a82192021-04-11 07:25:06 -07001548defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>;
1549defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>;
1550
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +08001551// 7.8 Vector Load/Store Segment Instructions
1552let RequiredExtension = "Zvlsseg" in {
1553defm : RVVUnitStridedSegLoad<"vlseg">;
1554defm : RVVUnitStridedSegLoadFF<"vlseg">;
Hsiangkai Wanga9de8f72021-06-07 17:54:00 +08001555defm : RVVStridedSegLoad<"vlsseg">;
Hsiangkai Wang1c550332021-06-07 21:53:37 +08001556defm : RVVIndexedSegLoad<"vluxseg">;
1557defm : RVVIndexedSegLoad<"vloxseg">;
Hsiangkai Wangd1a401b2021-06-08 13:09:07 +08001558defm : RVVUnitStridedSegStore<"vsseg">;
Hsiangkai Wang915e6dc2021-06-08 13:29:51 +08001559defm : RVVStridedSegStore<"vssseg">;
Hsiangkai Wang698f2882021-06-08 14:29:40 +08001560defm : RVVIndexedSegStore<"vsuxseg">;
1561defm : RVVIndexedSegStore<"vsoxseg">;
Hsiangkai Wang593bf9b2021-05-25 16:13:34 +08001562}
1563
Zakk Chend6a05602021-03-05 07:40:28 -08001564// 12. Vector Integer Arithmetic Instructions
1565// 12.1. Vector Single-Width Integer Add and Subtract
Zakk Chen66c05602021-03-29 07:37:29 -07001566defm vadd : RVVIntBinBuiltinSet;
1567defm vsub : RVVIntBinBuiltinSet;
1568defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil",
1569 [["vx", "v", "vve"],
1570 ["vx", "Uv", "UvUvUe"]]>;
Hsiangkai Wang14cc1cb22021-04-20 10:53:19 +08001571defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">;
Zakk Chen66c05602021-03-29 07:37:29 -07001572
1573// 12.2. Vector Widening Integer Add/Subtract
Zakk Chenccc624b2021-04-06 03:26:44 -07001574// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW
1575defm vwaddu : RVVUnsignedWidenBinBuiltinSet;
1576defm vwsubu : RVVUnsignedWidenBinBuiltinSet;
1577// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW
1578defm vwadd : RVVSignedWidenBinBuiltinSet;
1579defm vwsub : RVVSignedWidenBinBuiltinSet;
1580// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW
1581defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet;
1582defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet;
1583// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW
1584defm vwadd : RVVSignedWidenOp0BinBuiltinSet;
1585defm vwsub : RVVSignedWidenOp0BinBuiltinSet;
Hsiangkai Wangbd32c2d2021-04-20 15:31:13 +08001586defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi",
1587 [["Uw", "UwUv"]]>;
1588defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi",
1589 [["w", "wv"]]>;
Zakk Chen66c05602021-03-29 07:37:29 -07001590
1591// 12.3. Vector Integer Extension
Zakk Chenf720c222021-03-30 08:55:46 -07001592let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
Craig Topper5f6b3d12021-04-07 17:33:20 -07001593 def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">;
1594 def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">;
Zakk Chenf720c222021-03-30 08:55:46 -07001595}
1596let Log2LMUL = [-3, -2, -1, 0, 1] in {
Craig Topper5f6b3d12021-04-07 17:33:20 -07001597 def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">;
1598 def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">;
Zakk Chenf720c222021-03-30 08:55:46 -07001599}
1600let Log2LMUL = [-3, -2, -1, 0] in {
Craig Topper5f6b3d12021-04-07 17:33:20 -07001601 def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">;
1602 def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">;
Zakk Chenf720c222021-03-30 08:55:46 -07001603}
Zakk Chen66c05602021-03-29 07:37:29 -07001604
1605// 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001606let HasMask = false, HasPolicy = false in {
Zakk Chenfe252b52021-03-30 08:59:07 -07001607 defm vadc : RVVCarryinBuiltinSet;
1608 defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">;
1609 defm vmadc : RVVIntMaskOutBuiltinSet;
1610 defm vsbc : RVVCarryinBuiltinSet;
1611 defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">;
1612 defm vmsbc : RVVIntMaskOutBuiltinSet;
1613}
Zakk Chen66c05602021-03-29 07:37:29 -07001614
1615// 12.5. Vector Bitwise Logical Instructions
1616defm vand : RVVIntBinBuiltinSet;
1617defm vxor : RVVIntBinBuiltinSet;
1618defm vor : RVVIntBinBuiltinSet;
Hsiangkai Wang43cd5882021-04-20 11:01:22 +08001619defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">;
Zakk Chen66c05602021-03-29 07:37:29 -07001620
1621// 12.6. Vector Single-Width Bit Shift Instructions
Zakk Chenf2a36012021-03-30 10:12:07 -07001622defm vsll : RVVShiftBuiltinSet;
1623defm vsrl : RVVUnsignedShiftBuiltinSet;
1624defm vsra : RVVSignedShiftBuiltinSet;
Zakk Chen66c05602021-03-29 07:37:29 -07001625
1626// 12.7. Vector Narrowing Integer Right Shift Instructions
Zakk Chenf2a36012021-03-30 10:12:07 -07001627defm vnsrl : RVVUnsignedNShiftBuiltinSet;
1628defm vnsra : RVVSignedNShiftBuiltinSet;
Hsiangkai Wangbd32c2d2021-04-20 15:31:13 +08001629defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi",
1630 [["v", "vw"],
1631 ["Uv", "UvUw"]]>;
Zakk Chen66c05602021-03-29 07:37:29 -07001632
1633// 12.8. Vector Integer Comparison Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001634let HasPolicy = false in {
Zakk Chenfe252b52021-03-30 08:59:07 -07001635defm vmseq : RVVIntMaskOutBuiltinSet;
1636defm vmsne : RVVIntMaskOutBuiltinSet;
1637defm vmsltu : RVVUnsignedMaskOutBuiltinSet;
1638defm vmslt : RVVSignedMaskOutBuiltinSet;
1639defm vmsleu : RVVUnsignedMaskOutBuiltinSet;
1640defm vmsle : RVVSignedMaskOutBuiltinSet;
Hsiangkai Wang645c5f22021-04-20 14:50:42 +08001641defm vmsgtu : RVVUnsignedMaskOutBuiltinSet;
1642defm vmsgt : RVVSignedMaskOutBuiltinSet;
1643defm vmsgeu : RVVUnsignedMaskOutBuiltinSet;
1644defm vmsge : RVVSignedMaskOutBuiltinSet;
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001645}
Zakk Chen66c05602021-03-29 07:37:29 -07001646
1647// 12.9. Vector Integer Min/Max Instructions
1648defm vminu : RVVUnsignedBinBuiltinSet;
1649defm vmin : RVVSignedBinBuiltinSet;
1650defm vmaxu : RVVUnsignedBinBuiltinSet;
1651defm vmax : RVVSignedBinBuiltinSet;
1652
1653// 12.10. Vector Single-Width Integer Multiply Instructions
1654defm vmul : RVVIntBinBuiltinSet;
1655defm vmulh : RVVSignedBinBuiltinSet;
1656defm vmulhu : RVVUnsignedBinBuiltinSet;
1657defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil",
1658 [["vv", "v", "vvUv"],
1659 ["vx", "v", "vvUe"]]>;
1660
1661// 12.11. Vector Integer Divide Instructions
1662defm vdivu : RVVUnsignedBinBuiltinSet;
1663defm vdiv : RVVSignedBinBuiltinSet;
1664defm vremu : RVVUnsignedBinBuiltinSet;
1665defm vrem : RVVSignedBinBuiltinSet;
1666
1667// 12.12. Vector Widening Integer Multiply Instructions
Zakk Chen0a18ea02021-03-29 09:38:55 -07001668let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
1669defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi",
1670 [["vv", "w", "wvv"],
1671 ["vx", "w", "wve"]]>;
1672defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi",
1673 [["vv", "Uw", "UwUvUv"],
1674 ["vx", "Uw", "UwUvUe"]]>;
1675defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi",
1676 [["vv", "w", "wvUv"],
1677 ["vx", "w", "wvUe"]]>;
1678}
Zakk Chen66c05602021-03-29 07:37:29 -07001679
1680// 12.13. Vector Single-Width Integer Multiply-Add Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001681let HasPolicy = false in {
Zakk Chen98a3ff92021-04-08 08:09:42 -07001682defm vmacc : RVVIntTerBuiltinSet;
1683defm vnmsac : RVVIntTerBuiltinSet;
1684defm vmadd : RVVIntTerBuiltinSet;
1685defm vnmsub : RVVIntTerBuiltinSet;
Zakk Chen66c05602021-03-29 07:37:29 -07001686
1687// 12.14. Vector Widening Integer Multiply-Add Instructions
Zakk Chenfe252b52021-03-30 08:59:07 -07001688let HasMaskedOffOperand = false,
1689 Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
1690defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi",
1691 [["vv", "Uw", "UwUwUvUv"],
1692 ["vx", "Uw", "UwUwUeUv"]]>;
1693defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi",
1694 [["vv", "w", "wwvv"],
1695 ["vx", "w", "wwev"]]>;
1696defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi",
1697 [["vv", "w", "wwvUv"],
1698 ["vx", "w", "wweUv"]]>;
1699defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi",
1700 [["vx", "w", "wwUev"]]>;
1701}
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001702}
Zakk Chen66c05602021-03-29 07:37:29 -07001703
1704// 12.15. Vector Integer Merge Instructions
Craig Toppercfe3b002021-05-01 13:18:21 -07001705// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (op1, op2, mask, vl)
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001706let HasMask = false, HasPolicy = false,
Craig Toppercfe3b002021-05-01 13:18:21 -07001707 ManualCodegen = [{
1708 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3);
1709 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()};
1710 }] in {
1711 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil",
1712 [["vvm", "v", "vmvv"],
1713 ["vxm", "v", "vmve"],
1714 ["vvm", "Uv", "UvmUvUv"],
1715 ["vxm", "Uv", "UvmUvUe"]]>;
Zakk Chen07c38542021-04-06 07:23:30 -07001716}
Zakk Chen66c05602021-03-29 07:37:29 -07001717
1718// 12.16. Vector Integer Move Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001719let HasMask = false, HasPolicy = false in {
Zakk Chenea5d33d2021-04-11 06:50:57 -07001720 let MangledName = "vmv_v" in {
1721 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil",
1722 [["v", "Uv", "UvUv"]]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001723 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfd",
Zakk Chenea5d33d2021-04-11 06:50:57 -07001724 [["v", "v", "vv"]]>;
1725 }
1726 let HasNoMaskedOverloaded = false in
1727 defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil",
1728 [["x", "v", "ve"],
1729 ["x", "Uv", "UvUe"]]>;
1730}
Zakk Chend6a05602021-03-05 07:40:28 -08001731
Zakk Chenf2a36012021-03-30 10:12:07 -07001732// 13. Vector Fixed-Point Arithmetic Instructions
1733// 13.1. Vector Single-Width Saturating Add and Subtract
1734defm vsaddu : RVVUnsignedBinBuiltinSet;
1735defm vsadd : RVVSignedBinBuiltinSet;
1736defm vssubu : RVVUnsignedBinBuiltinSet;
1737defm vssub : RVVSignedBinBuiltinSet;
1738
1739// 13.2. Vector Single-Width Averaging Add and Subtract
1740defm vaaddu : RVVUnsignedBinBuiltinSet;
1741defm vaadd : RVVSignedBinBuiltinSet;
1742defm vasubu : RVVUnsignedBinBuiltinSet;
1743defm vasub : RVVSignedBinBuiltinSet;
1744
1745// 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
1746defm vsmul : RVVSignedBinBuiltinSet;
1747
1748// 13.4. Vector Single-Width Scaling Shift Instructions
1749defm vssrl : RVVUnsignedShiftBuiltinSet;
1750defm vssra : RVVSignedShiftBuiltinSet;
1751
1752// 13.5. Vector Narrowing Fixed-Point Clip Instructions
1753defm vnclipu : RVVUnsignedNShiftBuiltinSet;
1754defm vnclip : RVVSignedNShiftBuiltinSet;
1755
Zakk Chend6a05602021-03-05 07:40:28 -08001756// 14. Vector Floating-Point Instructions
1757// 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions
Zakk Chen98a3ff92021-04-08 08:09:42 -07001758defm vfadd : RVVFloatingBinBuiltinSet;
1759defm vfsub : RVVFloatingBinBuiltinSet;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001760defm vfrsub : RVVFloatingBinVFBuiltinSet;
1761
1762// 14.3. Vector Widening Floating-Point Add/Subtract Instructions
Zakk Chen98a3ff92021-04-08 08:09:42 -07001763// Widening FP add/subtract, 2*SEW = SEW +/- SEW
1764defm vfwadd : RVVFloatingWidenBinBuiltinSet;
1765defm vfwsub : RVVFloatingWidenBinBuiltinSet;
1766// Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW
1767defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet;
1768defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001769
1770// 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
Zakk Chen98a3ff92021-04-08 08:09:42 -07001771defm vfmul : RVVFloatingBinBuiltinSet;
1772defm vfdiv : RVVFloatingBinBuiltinSet;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001773defm vfrdiv : RVVFloatingBinVFBuiltinSet;
1774
1775// 14.5. Vector Widening Floating-Point Multiply
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001776let Log2LMUL = [-2, -1, 0, 1, 2] in {
1777 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf",
Zakk Chen98a3ff92021-04-08 08:09:42 -07001778 [["vv", "w", "wvv"],
1779 ["vf", "w", "wve"]]>;
1780}
Zakk Chen007ea0e2021-04-08 07:29:59 -07001781
1782// 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001783let HasPolicy = false in {
Zakk Chen98a3ff92021-04-08 08:09:42 -07001784defm vfmacc : RVVFloatingTerBuiltinSet;
1785defm vfnmacc : RVVFloatingTerBuiltinSet;
1786defm vfmsac : RVVFloatingTerBuiltinSet;
1787defm vfnmsac : RVVFloatingTerBuiltinSet;
1788defm vfmadd : RVVFloatingTerBuiltinSet;
1789defm vfnmadd : RVVFloatingTerBuiltinSet;
1790defm vfmsub : RVVFloatingTerBuiltinSet;
1791defm vfnmsub : RVVFloatingTerBuiltinSet;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001792
1793// 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
Zakk Chen98a3ff92021-04-08 08:09:42 -07001794defm vfwmacc : RVVFloatingWidenTerBuiltinSet;
1795defm vfwnmacc : RVVFloatingWidenTerBuiltinSet;
1796defm vfwmsac : RVVFloatingWidenTerBuiltinSet;
1797defm vfwnmsac : RVVFloatingWidenTerBuiltinSet;
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001798}
Zakk Chen007ea0e2021-04-08 07:29:59 -07001799
1800// 14.8. Vector Floating-Point Square-Root Instruction
Zakk Chen5f7739b2021-04-08 08:21:06 -07001801def vfsqrt : RVVFloatingUnaryVVBuiltin;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001802
1803// 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
Zakk Chen5f7739b2021-04-08 08:21:06 -07001804def vfrsqrt7 : RVVFloatingUnaryVVBuiltin;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001805
1806// 14.10. Vector Floating-Point Reciprocal Estimate Instruction
Zakk Chen5f7739b2021-04-08 08:21:06 -07001807def vfrec7 : RVVFloatingUnaryVVBuiltin;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001808
1809// 14.11. Vector Floating-Point MIN/MAX Instructions
1810defm vfmin : RVVFloatingBinBuiltinSet;
1811defm vfmax : RVVFloatingBinBuiltinSet;
1812
1813// 14.12. Vector Floating-Point Sign-Injection Instructions
Zakk Chen98a3ff92021-04-08 08:09:42 -07001814defm vfsgnj : RVVFloatingBinBuiltinSet;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001815defm vfsgnjn : RVVFloatingBinBuiltinSet;
1816defm vfsgnjx : RVVFloatingBinBuiltinSet;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001817defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">;
1818defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001819
1820// 14.13. Vector Floating-Point Compare Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001821let HasPolicy = false in {
Zakk Chen98a3ff92021-04-08 08:09:42 -07001822defm vmfeq : RVVFloatingMaskOutBuiltinSet;
1823defm vmfne : RVVFloatingMaskOutBuiltinSet;
1824defm vmflt : RVVFloatingMaskOutBuiltinSet;
1825defm vmfle : RVVFloatingMaskOutBuiltinSet;
Hsiangkai Wang645c5f22021-04-20 14:50:42 +08001826defm vmfgt : RVVFloatingMaskOutBuiltinSet;
1827defm vmfge : RVVFloatingMaskOutBuiltinSet;
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001828}
Zakk Chen007ea0e2021-04-08 07:29:59 -07001829
1830// 14.14. Vector Floating-Point Classify Instruction
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001831let Name = "vfclass_v", HasPolicy = false in
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001832 def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001833
1834// 14.15. Vector Floating-Point Merge Instructio
Zakk Chen07c38542021-04-06 07:23:30 -07001835// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl)
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001836let HasMask = false, HasPolicy = false,
Craig Toppercfe3b002021-05-01 13:18:21 -07001837 ManualCodegen = [{
1838 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3);
1839 IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType()};
1840 }] in {
Craig Toppere2b7aab2021-10-25 08:56:43 -07001841 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfd",
Craig Toppercfe3b002021-05-01 13:18:21 -07001842 [["vvm", "v", "vmvv"]]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001843 defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd",
Craig Toppercfe3b002021-05-01 13:18:21 -07001844 [["vfm", "v", "vmve"]]>;
Zakk Chen07c38542021-04-06 07:23:30 -07001845}
Zakk Chen007ea0e2021-04-08 07:29:59 -07001846
1847// 14.16. Vector Floating-Point Move Instruction
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001848let HasMask = false, HasNoMaskedOverloaded = false, HasPolicy = false in
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001849 defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd",
Zakk Chenea5d33d2021-04-11 06:50:57 -07001850 [["f", "v", "ve"]]>;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001851
1852// 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions
Zakk Chen01fa2222021-04-01 09:21:11 -07001853def vfcvt_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_xu">;
1854def vfcvt_x_f_v : RVVConvToSignedBuiltin<"vfcvt_x">;
1855def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">;
1856def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">;
1857def vfcvt_f_xu_v : RVVConvBuiltin<"Fv", "FvUv", "sil", "vfcvt_f">;
1858def vfcvt_f_x_v : RVVConvBuiltin<"Fv", "Fvv", "sil", "vfcvt_f">;
Zakk Chen007ea0e2021-04-08 07:29:59 -07001859
1860// 14.18. Widening Floating-Point/Integer Type-Convert Instructions
Zakk Chen01fa2222021-04-01 09:21:11 -07001861let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
1862 def vfwcvt_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_xu">;
1863 def vfwcvt_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_x">;
1864 def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">;
1865 def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">;
1866 def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">;
1867 def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001868 def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "xf", "vfwcvt_f">;
Zakk Chen01fa2222021-04-01 09:21:11 -07001869}
Zakk Chen007ea0e2021-04-08 07:29:59 -07001870
1871// 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions
Zakk Chen01fa2222021-04-01 09:21:11 -07001872let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
1873 def vfncvt_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_xu">;
1874 def vfncvt_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_x">;
1875 def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">;
1876 def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001877 def vfncvt_f_xu_w : RVVConvBuiltin<"Fv", "FvUw", "csi", "vfncvt_f">;
1878 def vfncvt_f_x_w : RVVConvBuiltin<"Fv", "Fvw", "csi", "vfncvt_f">;
1879 def vfncvt_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_f">;
1880 def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">;
Zakk Chen01fa2222021-04-01 09:21:11 -07001881}
Zakk Chenc680b0d2021-04-06 07:57:41 -07001882
1883// 15. Vector Reduction Operations
1884// 15.1. Vector Single-Width Integer Reduction Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001885let HasPolicy = false in {
Zakk Chenc680b0d2021-04-06 07:57:41 -07001886defm vredsum : RVVIntReductionBuiltinSet;
1887defm vredmaxu : RVVUnsignedReductionBuiltin;
1888defm vredmax : RVVSignedReductionBuiltin;
1889defm vredminu : RVVUnsignedReductionBuiltin;
1890defm vredmin : RVVSignedReductionBuiltin;
1891defm vredand : RVVIntReductionBuiltinSet;
1892defm vredor : RVVIntReductionBuiltinSet;
1893defm vredxor : RVVIntReductionBuiltinSet;
1894
1895// 15.2. Vector Widening Integer Reduction Instructions
1896// Vector Widening Integer Reduction Operations
1897let HasMaskedOffOperand = false in {
1898 defm vwredsum : RVVOutOp1BuiltinSet<"vwredsum", "csi",
1899 [["vs", "vSw", "SwSwvSw"]]>;
1900 defm vwredsumu : RVVOutOp1BuiltinSet<"vwredsumu", "csi",
1901 [["vs", "UvUSw", "USwUSwUvUSw"]]>;
1902}
1903
1904// 15.3. Vector Single-Width Floating-Point Reduction Instructions
1905defm vfredmax : RVVFloatingReductionBuiltin;
1906defm vfredmin : RVVFloatingReductionBuiltin;
jacquesguan0608bbd2021-10-12 06:46:46 +00001907defm vfredusum : RVVFloatingReductionBuiltin;
Zakk Chenc680b0d2021-04-06 07:57:41 -07001908defm vfredosum : RVVFloatingReductionBuiltin;
1909
1910// 15.4. Vector Widening Floating-Point Reduction Instructions
jacquesguan0608bbd2021-10-12 06:46:46 +00001911defm vfwredusum : RVVFloatingWidenReductionBuiltin;
Zakk Chenc680b0d2021-04-06 07:57:41 -07001912defm vfwredosum : RVVFloatingWidenReductionBuiltin;
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001913}
Zakk Chena8fc0e42021-04-08 08:28:15 -07001914
1915// 16. Vector Mask Instructions
1916// 16.1. Vector Mask-Register Logical Instructions
1917def vmand : RVVMaskBinBuiltin;
1918def vmnand : RVVMaskBinBuiltin;
Zakk Chen0649dfe2021-11-04 09:22:34 -07001919def vmandn : RVVMaskBinBuiltin;
Zakk Chena8fc0e42021-04-08 08:28:15 -07001920def vmxor : RVVMaskBinBuiltin;
1921def vmor : RVVMaskBinBuiltin;
1922def vmnor : RVVMaskBinBuiltin;
Zakk Chen0649dfe2021-11-04 09:22:34 -07001923def vmorn : RVVMaskBinBuiltin;
Zakk Chena8fc0e42021-04-08 08:28:15 -07001924def vmxnor : RVVMaskBinBuiltin;
1925// pseudoinstructions
1926def vmclr : RVVMaskNullaryBuiltin;
1927def vmset : RVVMaskNullaryBuiltin;
Hsiangkai Wang4b243412021-04-20 11:12:53 +08001928defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">;
1929defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">;
Zakk Chena8fc0e42021-04-08 08:28:15 -07001930
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001931let HasPolicy = false in {
Zakk Chen0649dfe2021-11-04 09:22:34 -07001932// 16.2. Vector count population in mask vcpop.m
1933def vcpop : RVVMaskOp0Builtin<"um">;
Zakk Chena8fc0e42021-04-08 08:28:15 -07001934
1935// 16.3. vfirst find-first-set mask bit
1936def vfirst : RVVMaskOp0Builtin<"lm">;
1937
1938// 16.4. vmsbf.m set-before-first mask bit
1939def vmsbf : RVVMaskUnaryBuiltin;
1940
1941// 16.5. vmsif.m set-including-first mask bit
1942def vmsif : RVVMaskUnaryBuiltin;
1943
1944// 16.6. vmsof.m set-only-first mask bit
1945def vmsof : RVVMaskUnaryBuiltin;
1946
1947let HasNoMaskedOverloaded = false in {
1948 // 16.8. Vector Iota Instruction
1949 defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>;
1950
1951 // 16.9. Vector Element Index Instruction
1952 defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"],
1953 ["v", "Uv", "Uv"]]>;
1954}
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001955}
Zakk Chen59d5b8c2021-04-08 10:15:09 -07001956
1957// 17. Vector Permutation Instructions
1958// 17.1. Integer Scalar Move Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001959let HasMask = false, HasPolicy = false in {
Zakk Chenea5d33d2021-04-11 06:50:57 -07001960 let HasVL = false, MangledName = "vmv_x" in
1961 defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil",
1962 [["s", "ve", "ev"],
1963 ["s", "UvUe", "UeUv"]]>;
1964 let MangledName = "vmv_s" in
1965 defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil",
1966 [["x", "v", "vve"],
1967 ["x", "Uv", "UvUvUe"]]>;
1968}
Zakk Chen59d5b8c2021-04-08 10:15:09 -07001969
1970// 17.2. Floating-Point Scalar Move Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001971let HasMask = false, HasPolicy = false in {
Zakk Chenea5d33d2021-04-11 06:50:57 -07001972 let HasVL = false, MangledName = "vfmv_f" in
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001973 defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd",
Zakk Chenea5d33d2021-04-11 06:50:57 -07001974 [["s", "ve", "ev"]]>;
1975 let MangledName = "vfmv_s" in
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001976 defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd",
Zakk Chenea5d33d2021-04-11 06:50:57 -07001977 [["f", "v", "vve"],
1978 ["x", "Uv", "UvUvUe"]]>;
1979}
Zakk Chen59d5b8c2021-04-08 10:15:09 -07001980
1981// 17.3. Vector Slide Instructions
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001982let HasPolicy = false in {
Zakk Chen59d5b8c2021-04-08 10:15:09 -07001983// 17.3.1. Vector Slideup Instructions
1984defm vslideup : RVVSlideBuiltinSet;
1985// 17.3.2. Vector Slidedown Instructions
1986defm vslidedown : RVVSlideBuiltinSet;
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08001987}
Zakk Chen59d5b8c2021-04-08 10:15:09 -07001988
1989// 17.3.3. Vector Slide1up Instructions
1990defm vslide1up : RVVSlideOneBuiltinSet;
1991defm vfslide1up : RVVFloatingBinVFBuiltinSet;
1992
1993// 17.3.4. Vector Slide1down Instruction
1994defm vslide1down : RVVSlideOneBuiltinSet;
1995defm vfslide1down : RVVFloatingBinVFBuiltinSet;
1996
1997// 17.4. Vector Register Gather Instructions
1998// signed and floating type
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08001999defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd",
Zakk Chen59d5b8c2021-04-08 10:15:09 -07002000 [["vv", "v", "vvUv"]]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002001defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd",
Zakk Chen59d5b8c2021-04-08 10:15:09 -07002002 [["vx", "v", "vvz"]]>;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002003defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd",
Zakk Chen59d5b8c2021-04-08 10:15:09 -07002004 [["vv", "v", "vv(Log2EEW:4)Uv"]]>;
2005// unsigned type
2006defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil",
2007 [["vv", "Uv", "UvUvUv"]]>;
2008defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil",
2009 [["vx", "Uv", "UvUvz"]]>;
2010defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil",
2011 [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>;
2012
2013// 17.5. Vector Compress Instruction
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002014let HasMask = false, HasPolicy = false,
Craig Toppercfe3b002021-05-01 13:18:21 -07002015 ManualCodegen = [{
2016 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 3);
Craig Topper081ae5f2021-06-10 20:42:11 -07002017 IntrinsicTypes = {ResultType, Ops[3]->getType()};
Craig Toppercfe3b002021-05-01 13:18:21 -07002018 }] in {
Zakk Chen59d5b8c2021-04-08 10:15:09 -07002019 // signed and floating type
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002020 defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd",
Craig Toppercfe3b002021-05-01 13:18:21 -07002021 [["vm", "v", "vmvv"]]>;
Zakk Chen59d5b8c2021-04-08 10:15:09 -07002022 // unsigned type
2023 defm vcompress : RVVOutBuiltinSet<"vcompress", "csil",
Craig Toppercfe3b002021-05-01 13:18:21 -07002024 [["vm", "Uv", "UvmUvUv"]]>;
Zakk Chen59d5b8c2021-04-08 10:15:09 -07002025}
Zakk Chen8f683362021-04-12 19:01:44 -07002026
2027// Miscellaneous
Zakk Chen08cf69c2021-07-13 20:32:55 -07002028let HasMask = false, HasVL = false, IRName = "" in {
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002029 let Name = "vreinterpret_v", HasPolicy = false,
Zakk Chen8f683362021-04-12 19:01:44 -07002030 ManualCodegen = [{
2031 return Builder.CreateBitCast(Ops[0], ResultType);
2032 }] in {
2033 // Reinterpret between different type under the same SEW and LMUL
Zakk Chen08cf69c2021-07-13 20:32:55 -07002034 def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002035 def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">;
Zakk Chen08cf69c2021-07-13 20:32:55 -07002036 def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">;
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002037 def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">;
2038 def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">;
2039 def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">;
Zakk Chen8f683362021-04-12 19:01:44 -07002040
2041 // Reinterpret between different SEW under the same LMUL
2042 foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)",
2043 "(FixedSEW:64)"] in {
Zakk Chen08cf69c2021-07-13 20:32:55 -07002044 def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v",
2045 dst_sew # "vv", "csil", dst_sew # "v">;
2046 def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv",
2047 dst_sew # "UvUv", "csil", dst_sew # "Uv">;
Zakk Chen8f683362021-04-12 19:01:44 -07002048 }
2049 }
2050
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002051 let Name = "vundefined", HasNoMaskedOverloaded = false, HasPolicy = false,
Zakk Chen8f683362021-04-12 19:01:44 -07002052 ManualCodegen = [{
2053 return llvm::UndefValue::get(ResultType);
2054 }] in {
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002055 def vundefined : RVVBuiltin<"v", "v", "csilxfd">;
Zakk Chen8f683362021-04-12 19:01:44 -07002056 def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">;
2057 }
2058
2059 // LMUL truncation
2060 // C/C++ Operand: VecTy, IR Operand: VecTy, Index
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002061 let Name = "vlmul_trunc_v", MangledName = "vlmul_trunc", HasPolicy = false,
Zakk Chen8f683362021-04-12 19:01:44 -07002062 ManualCodegen = [{ {
2063 ID = Intrinsic::experimental_vector_extract;
2064 IntrinsicTypes = {ResultType, Ops[0]->getType()};
2065 Ops.push_back(ConstantInt::get(Int64Ty, 0));
2066 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, "");
2067 } }] in {
2068 foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)",
2069 "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in {
Zakk Chen08cf69c2021-07-13 20:32:55 -07002070 def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002071 dst_lmul # "vv", "csilxfd", dst_lmul # "v">;
Zakk Chen08cf69c2021-07-13 20:32:55 -07002072 def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv",
2073 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">;
Zakk Chen8f683362021-04-12 19:01:44 -07002074 }
2075 }
2076
2077 // LMUL extension
2078 // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002079 let Name = "vlmul_ext_v", MangledName = "vlmul_ext", HasPolicy = false,
Zakk Chen8f683362021-04-12 19:01:44 -07002080 ManualCodegen = [{
2081 ID = Intrinsic::experimental_vector_insert;
2082 IntrinsicTypes = {ResultType, Ops[0]->getType()};
2083 Ops.push_back(llvm::UndefValue::get(ResultType));
2084 std::swap(Ops[0], Ops[1]);
2085 Ops.push_back(ConstantInt::get(Int64Ty, 0));
2086 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, "");
2087 }] in {
2088 foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)",
2089 "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in {
Zakk Chen08cf69c2021-07-13 20:32:55 -07002090 def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
Hsiangkai Wang77bb82d2021-06-28 13:38:41 +08002091 dst_lmul # "vv", "csilxfd", dst_lmul # "v">;
Zakk Chen08cf69c2021-07-13 20:32:55 -07002092 def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv",
2093 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">;
Zakk Chen8f683362021-04-12 19:01:44 -07002094 }
2095 }
Craig Topperf2253672021-06-24 15:53:47 -07002096
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002097 let Name = "vget_v", HasPolicy = false,
Craig Topperf2253672021-06-24 15:53:47 -07002098 ManualCodegen = [{
2099 {
2100 ID = Intrinsic::experimental_vector_extract;
Craig Toppere2b7aab2021-10-25 08:56:43 -07002101 auto *VecTy = cast<ScalableVectorType>(ResultType);
2102 auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType());
2103 // Mask to only valid indices.
2104 unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements();
2105 assert(isPowerOf2_32(MaxIndex));
2106 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1);
Craig Topperf2253672021-06-24 15:53:47 -07002107 Ops[1] = Builder.CreateMul(Ops[1],
2108 ConstantInt::get(Ops[1]->getType(),
2109 VecTy->getMinNumElements()));
2110 IntrinsicTypes = {ResultType, Ops[0]->getType()};
2111 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, "");
2112 }
2113 }] in {
2114 foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in {
Hsiangkai Wang5f996702021-08-04 07:57:13 +08002115 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">;
Zakk Chen08cf69c2021-07-13 20:32:55 -07002116 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">;
Craig Topperf2253672021-06-24 15:53:47 -07002117 }
2118 }
2119
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002120 let Name = "vset_v", Log2LMUL = [0, 1, 2], HasPolicy = false,
Craig Topperf2253672021-06-24 15:53:47 -07002121 ManualCodegen = [{
2122 {
2123 ID = Intrinsic::experimental_vector_insert;
2124 IntrinsicTypes = {ResultType, Ops[2]->getType()};
Craig Toppere2b7aab2021-10-25 08:56:43 -07002125 auto *ResVecTy = cast<ScalableVectorType>(ResultType);
2126 auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType());
2127 // Mask to only valid indices.
2128 unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements();
2129 assert(isPowerOf2_32(MaxIndex));
2130 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1);
Craig Topperf2253672021-06-24 15:53:47 -07002131 Ops[1] = Builder.CreateMul(Ops[1],
2132 ConstantInt::get(Ops[1]->getType(),
2133 VecTy->getMinNumElements()));
2134 std::swap(Ops[1], Ops[2]);
2135 return Builder.CreateCall(CGM.getIntrinsic(ID, IntrinsicTypes), Ops, "");
2136 }
2137 }] in {
2138 foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in {
Hsiangkai Wang5f996702021-08-04 07:57:13 +08002139 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">;
Craig Topperf2253672021-06-24 15:53:47 -07002140 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">;
2141 }
2142 }
Zakk Chen8f683362021-04-12 19:01:44 -07002143}
Hsiangkai Wang5158cfef2021-10-13 14:35:56 +08002144
2145let HeaderCode = [{
2146#define VE_TAIL_UNDISTURBED 0
2147#define VE_TAIL_AGNOSTIC 1
2148}] in
2149def policy : RVVHeader;