| //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file implements the auto-upgrade helper functions. |
| // This is where deprecated IR intrinsics and other IR features are updated to |
| // current specifications. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/IR/AutoUpgrade.h" |
| #include "llvm/ADT/StringSwitch.h" |
| #include "llvm/IR/Constants.h" |
| #include "llvm/IR/DIBuilder.h" |
| #include "llvm/IR/DebugInfo.h" |
| #include "llvm/IR/DiagnosticInfo.h" |
| #include "llvm/IR/Function.h" |
| #include "llvm/IR/IRBuilder.h" |
| #include "llvm/IR/Instruction.h" |
| #include "llvm/IR/IntrinsicInst.h" |
| #include "llvm/IR/LLVMContext.h" |
| #include "llvm/IR/Module.h" |
| #include "llvm/IR/Verifier.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/Regex.h" |
| #include <cstring> |
| using namespace llvm; |
| |
| static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); } |
| |
| // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have |
| // changed their type from v4f32 to v2i64. |
| static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID, |
| Function *&NewFn) { |
| // Check whether this is an old version of the function, which received |
| // v4f32 arguments. |
| Type *Arg0Type = F->getFunctionType()->getParamType(0); |
| if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) |
| return false; |
| |
| // Yes, it's old, replace it with new version. |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), IID); |
| return true; |
| } |
| |
| // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask |
| // arguments have changed their type from i32 to i8. |
| static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, |
| Function *&NewFn) { |
| // Check that the last argument is an i32. |
| Type *LastArgType = F->getFunctionType()->getParamType( |
| F->getFunctionType()->getNumParams() - 1); |
| if (!LastArgType->isIntegerTy(32)) |
| return false; |
| |
| // Move this function aside and map down. |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), IID); |
| return true; |
| } |
| |
| static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { |
| // All of the intrinsics matches below should be marked with which llvm |
| // version started autoupgrading them. At some point in the future we would |
| // like to use this information to remove upgrade code for some older |
| // intrinsics. It is currently undecided how we will determine that future |
| // point. |
| if (Name == "addcarryx.u32" || // Added in 8.0 |
| Name == "addcarryx.u64" || // Added in 8.0 |
| Name == "addcarry.u32" || // Added in 8.0 |
| Name == "addcarry.u64" || // Added in 8.0 |
| Name == "subborrow.u32" || // Added in 8.0 |
| Name == "subborrow.u64" || // Added in 8.0 |
| Name.startswith("sse2.padds.") || // Added in 8.0 |
| Name.startswith("sse2.psubs.") || // Added in 8.0 |
| Name.startswith("sse2.paddus.") || // Added in 8.0 |
| Name.startswith("sse2.psubus.") || // Added in 8.0 |
| Name.startswith("avx2.padds.") || // Added in 8.0 |
| Name.startswith("avx2.psubs.") || // Added in 8.0 |
| Name.startswith("avx2.paddus.") || // Added in 8.0 |
| Name.startswith("avx2.psubus.") || // Added in 8.0 |
| Name.startswith("avx512.padds.") || // Added in 8.0 |
| Name.startswith("avx512.psubs.") || // Added in 8.0 |
| Name.startswith("avx512.mask.padds.") || // Added in 8.0 |
| Name.startswith("avx512.mask.psubs.") || // Added in 8.0 |
| Name.startswith("avx512.mask.paddus.") || // Added in 8.0 |
| Name.startswith("avx512.mask.psubus.") || // Added in 8.0 |
| Name=="ssse3.pabs.b.128" || // Added in 6.0 |
| Name=="ssse3.pabs.w.128" || // Added in 6.0 |
| Name=="ssse3.pabs.d.128" || // Added in 6.0 |
| Name.startswith("fma4.vfmadd.s") || // Added in 7.0 |
| Name.startswith("fma.vfmadd.") || // Added in 7.0 |
| Name.startswith("fma.vfmsub.") || // Added in 7.0 |
| Name.startswith("fma.vfmaddsub.") || // Added in 7.0 |
| Name.startswith("fma.vfmsubadd.") || // Added in 7.0 |
| Name.startswith("fma.vfnmadd.") || // Added in 7.0 |
| Name.startswith("fma.vfnmsub.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0 |
| Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0 |
| Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0 |
| Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0 |
| Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0 |
| Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0 |
| Name.startswith("avx512.mask.shuf.i") || // Added in 6.0 |
| Name.startswith("avx512.mask.shuf.f") || // Added in 6.0 |
| Name.startswith("avx512.kunpck") || //added in 6.0 |
| Name.startswith("avx2.pabs.") || // Added in 6.0 |
| Name.startswith("avx512.mask.pabs.") || // Added in 6.0 |
| Name.startswith("avx512.broadcastm") || // Added in 6.0 |
| Name == "sse.sqrt.ss" || // Added in 7.0 |
| Name == "sse2.sqrt.sd" || // Added in 7.0 |
| Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0 |
| Name.startswith("avx.sqrt.p") || // Added in 7.0 |
| Name.startswith("sse2.sqrt.p") || // Added in 7.0 |
| Name.startswith("sse.sqrt.p") || // Added in 7.0 |
| Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0 |
| Name.startswith("sse2.pcmpeq.") || // Added in 3.1 |
| Name.startswith("sse2.pcmpgt.") || // Added in 3.1 |
| Name.startswith("avx2.pcmpeq.") || // Added in 3.1 |
| Name.startswith("avx2.pcmpgt.") || // Added in 3.1 |
| Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9 |
| Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9 |
| Name.startswith("avx.vperm2f128.") || // Added in 6.0 |
| Name == "avx2.vperm2i128" || // Added in 6.0 |
| Name == "sse.add.ss" || // Added in 4.0 |
| Name == "sse2.add.sd" || // Added in 4.0 |
| Name == "sse.sub.ss" || // Added in 4.0 |
| Name == "sse2.sub.sd" || // Added in 4.0 |
| Name == "sse.mul.ss" || // Added in 4.0 |
| Name == "sse2.mul.sd" || // Added in 4.0 |
| Name == "sse.div.ss" || // Added in 4.0 |
| Name == "sse2.div.sd" || // Added in 4.0 |
| Name == "sse41.pmaxsb" || // Added in 3.9 |
| Name == "sse2.pmaxs.w" || // Added in 3.9 |
| Name == "sse41.pmaxsd" || // Added in 3.9 |
| Name == "sse2.pmaxu.b" || // Added in 3.9 |
| Name == "sse41.pmaxuw" || // Added in 3.9 |
| Name == "sse41.pmaxud" || // Added in 3.9 |
| Name == "sse41.pminsb" || // Added in 3.9 |
| Name == "sse2.pmins.w" || // Added in 3.9 |
| Name == "sse41.pminsd" || // Added in 3.9 |
| Name == "sse2.pminu.b" || // Added in 3.9 |
| Name == "sse41.pminuw" || // Added in 3.9 |
| Name == "sse41.pminud" || // Added in 3.9 |
| Name == "avx512.kand.w" || // Added in 7.0 |
| Name == "avx512.kandn.w" || // Added in 7.0 |
| Name == "avx512.knot.w" || // Added in 7.0 |
| Name == "avx512.kor.w" || // Added in 7.0 |
| Name == "avx512.kxor.w" || // Added in 7.0 |
| Name == "avx512.kxnor.w" || // Added in 7.0 |
| Name == "avx512.kortestc.w" || // Added in 7.0 |
| Name == "avx512.kortestz.w" || // Added in 7.0 |
| Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0 |
| Name.startswith("avx2.pmax") || // Added in 3.9 |
| Name.startswith("avx2.pmin") || // Added in 3.9 |
| Name.startswith("avx512.mask.pmax") || // Added in 4.0 |
| Name.startswith("avx512.mask.pmin") || // Added in 4.0 |
| Name.startswith("avx2.vbroadcast") || // Added in 3.8 |
| Name.startswith("avx2.pbroadcast") || // Added in 3.8 |
| Name.startswith("avx.vpermil.") || // Added in 3.1 |
| Name.startswith("sse2.pshuf") || // Added in 3.9 |
| Name.startswith("avx512.pbroadcast") || // Added in 3.9 |
| Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9 |
| Name.startswith("avx512.mask.movddup") || // Added in 3.9 |
| Name.startswith("avx512.mask.movshdup") || // Added in 3.9 |
| Name.startswith("avx512.mask.movsldup") || // Added in 3.9 |
| Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9 |
| Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9 |
| Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9 |
| Name.startswith("avx512.mask.shuf.p") || // Added in 4.0 |
| Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9 |
| Name.startswith("avx512.mask.perm.df.") || // Added in 3.9 |
| Name.startswith("avx512.mask.perm.di.") || // Added in 3.9 |
| Name.startswith("avx512.mask.punpckl") || // Added in 3.9 |
| Name.startswith("avx512.mask.punpckh") || // Added in 3.9 |
| Name.startswith("avx512.mask.unpckl.") || // Added in 3.9 |
| Name.startswith("avx512.mask.unpckh.") || // Added in 3.9 |
| Name.startswith("avx512.mask.pand.") || // Added in 3.9 |
| Name.startswith("avx512.mask.pandn.") || // Added in 3.9 |
| Name.startswith("avx512.mask.por.") || // Added in 3.9 |
| Name.startswith("avx512.mask.pxor.") || // Added in 3.9 |
| Name.startswith("avx512.mask.and.") || // Added in 3.9 |
| Name.startswith("avx512.mask.andn.") || // Added in 3.9 |
| Name.startswith("avx512.mask.or.") || // Added in 3.9 |
| Name.startswith("avx512.mask.xor.") || // Added in 3.9 |
| Name.startswith("avx512.mask.padd.") || // Added in 4.0 |
| Name.startswith("avx512.mask.psub.") || // Added in 4.0 |
| Name.startswith("avx512.mask.pmull.") || // Added in 4.0 |
| Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0 |
| Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0 |
| Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0 |
| Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0 |
| Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0 |
| Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0 |
| Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0 |
| Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0 |
| Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0 |
| Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0 |
| Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0 |
| Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0 |
| Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0 |
| Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0 |
| Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0 |
| Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0 |
| Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0 |
| Name == "avx512.cvtusi2sd" || // Added in 7.0 |
| Name.startswith("avx512.mask.permvar.") || // Added in 7.0 |
| Name == "sse2.pmulu.dq" || // Added in 7.0 |
| Name == "sse41.pmuldq" || // Added in 7.0 |
| Name == "avx2.pmulu.dq" || // Added in 7.0 |
| Name == "avx2.pmul.dq" || // Added in 7.0 |
| Name == "avx512.pmulu.dq.512" || // Added in 7.0 |
| Name == "avx512.pmul.dq.512" || // Added in 7.0 |
| Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0 |
| Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0 |
| Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0 |
| Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0 |
| Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0 |
| Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0 |
| Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0 |
| Name.startswith("avx512.mask.packsswb.") || // Added in 5.0 |
| Name.startswith("avx512.mask.packssdw.") || // Added in 5.0 |
| Name.startswith("avx512.mask.packuswb.") || // Added in 5.0 |
| Name.startswith("avx512.mask.packusdw.") || // Added in 5.0 |
| Name.startswith("avx512.mask.cmp.b") || // Added in 5.0 |
| Name.startswith("avx512.mask.cmp.d") || // Added in 5.0 |
| Name.startswith("avx512.mask.cmp.q") || // Added in 5.0 |
| Name.startswith("avx512.mask.cmp.w") || // Added in 5.0 |
| Name.startswith("avx512.mask.cmp.p") || // Added in 7.0 |
| Name.startswith("avx512.mask.ucmp.") || // Added in 5.0 |
| Name.startswith("avx512.cvtb2mask.") || // Added in 7.0 |
| Name.startswith("avx512.cvtw2mask.") || // Added in 7.0 |
| Name.startswith("avx512.cvtd2mask.") || // Added in 7.0 |
| Name.startswith("avx512.cvtq2mask.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0 |
| Name.startswith("avx512.mask.psll.d") || // Added in 4.0 |
| Name.startswith("avx512.mask.psll.q") || // Added in 4.0 |
| Name.startswith("avx512.mask.psll.w") || // Added in 4.0 |
| Name.startswith("avx512.mask.psra.d") || // Added in 4.0 |
| Name.startswith("avx512.mask.psra.q") || // Added in 4.0 |
| Name.startswith("avx512.mask.psra.w") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrl.d") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrl.q") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrl.w") || // Added in 4.0 |
| Name.startswith("avx512.mask.pslli") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrai") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrli") || // Added in 4.0 |
| Name.startswith("avx512.mask.psllv") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrav") || // Added in 4.0 |
| Name.startswith("avx512.mask.psrlv") || // Added in 4.0 |
| Name.startswith("sse41.pmovsx") || // Added in 3.8 |
| Name.startswith("sse41.pmovzx") || // Added in 3.9 |
| Name.startswith("avx2.pmovsx") || // Added in 3.9 |
| Name.startswith("avx2.pmovzx") || // Added in 3.9 |
| Name.startswith("avx512.mask.pmovsx") || // Added in 4.0 |
| Name.startswith("avx512.mask.pmovzx") || // Added in 4.0 |
| Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0 |
| Name.startswith("avx512.mask.pternlog.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0 |
| Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0 |
| Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpshld.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0 |
| Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0 |
| Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0 |
| Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0 |
| Name.startswith("avx512.vpshld.") || // Added in 8.0 |
| Name.startswith("avx512.vpshrd.") || // Added in 8.0 |
| Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0 |
| Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0 |
| Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0 |
| Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0 |
| Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0 |
| Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0 |
| Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0 |
| Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0 |
| Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0 |
| Name.startswith("avx512.mask.conflict.") || // Added in 9.0 |
| Name == "avx512.mask.pmov.qd.256" || // Added in 9.0 |
| Name == "avx512.mask.pmov.qd.512" || // Added in 9.0 |
| Name == "avx512.mask.pmov.wb.256" || // Added in 9.0 |
| Name == "avx512.mask.pmov.wb.512" || // Added in 9.0 |
| Name == "sse.cvtsi2ss" || // Added in 7.0 |
| Name == "sse.cvtsi642ss" || // Added in 7.0 |
| Name == "sse2.cvtsi2sd" || // Added in 7.0 |
| Name == "sse2.cvtsi642sd" || // Added in 7.0 |
| Name == "sse2.cvtss2sd" || // Added in 7.0 |
| Name == "sse2.cvtdq2pd" || // Added in 3.9 |
| Name == "sse2.cvtdq2ps" || // Added in 7.0 |
| Name == "sse2.cvtps2pd" || // Added in 3.9 |
| Name == "avx.cvtdq2.pd.256" || // Added in 3.9 |
| Name == "avx.cvtdq2.ps.256" || // Added in 7.0 |
| Name == "avx.cvt.ps2.pd.256" || // Added in 3.9 |
| Name.startswith("avx.vinsertf128.") || // Added in 3.7 |
| Name == "avx2.vinserti128" || // Added in 3.7 |
| Name.startswith("avx512.mask.insert") || // Added in 4.0 |
| Name.startswith("avx.vextractf128.") || // Added in 3.7 |
| Name == "avx2.vextracti128" || // Added in 3.7 |
| Name.startswith("avx512.mask.vextract") || // Added in 4.0 |
| Name.startswith("sse4a.movnt.") || // Added in 3.9 |
| Name.startswith("avx.movnt.") || // Added in 3.2 |
| Name.startswith("avx512.storent.") || // Added in 3.9 |
| Name == "sse41.movntdqa" || // Added in 5.0 |
| Name == "avx2.movntdqa" || // Added in 5.0 |
| Name == "avx512.movntdqa" || // Added in 5.0 |
| Name == "sse2.storel.dq" || // Added in 3.9 |
| Name.startswith("sse.storeu.") || // Added in 3.9 |
| Name.startswith("sse2.storeu.") || // Added in 3.9 |
| Name.startswith("avx.storeu.") || // Added in 3.9 |
| Name.startswith("avx512.mask.storeu.") || // Added in 3.9 |
| Name.startswith("avx512.mask.store.p") || // Added in 3.9 |
| Name.startswith("avx512.mask.store.b.") || // Added in 3.9 |
| Name.startswith("avx512.mask.store.w.") || // Added in 3.9 |
| Name.startswith("avx512.mask.store.d.") || // Added in 3.9 |
| Name.startswith("avx512.mask.store.q.") || // Added in 3.9 |
| Name == "avx512.mask.store.ss" || // Added in 7.0 |
| Name.startswith("avx512.mask.loadu.") || // Added in 3.9 |
| Name.startswith("avx512.mask.load.") || // Added in 3.9 |
| Name.startswith("avx512.mask.expand.load.") || // Added in 7.0 |
| Name.startswith("avx512.mask.compress.store.") || // Added in 7.0 |
| Name.startswith("avx512.mask.expand.b") || // Added in 9.0 |
| Name.startswith("avx512.mask.expand.w") || // Added in 9.0 |
| Name.startswith("avx512.mask.expand.d") || // Added in 9.0 |
| Name.startswith("avx512.mask.expand.q") || // Added in 9.0 |
| Name.startswith("avx512.mask.expand.p") || // Added in 9.0 |
| Name.startswith("avx512.mask.compress.b") || // Added in 9.0 |
| Name.startswith("avx512.mask.compress.w") || // Added in 9.0 |
| Name.startswith("avx512.mask.compress.d") || // Added in 9.0 |
| Name.startswith("avx512.mask.compress.q") || // Added in 9.0 |
| Name.startswith("avx512.mask.compress.p") || // Added in 9.0 |
| Name == "sse42.crc32.64.8" || // Added in 3.4 |
| Name.startswith("avx.vbroadcast.s") || // Added in 3.5 |
| Name.startswith("avx512.vbroadcast.s") || // Added in 7.0 |
| Name.startswith("avx512.mask.palignr.") || // Added in 3.9 |
| Name.startswith("avx512.mask.valign.") || // Added in 4.0 |
| Name.startswith("sse2.psll.dq") || // Added in 3.7 |
| Name.startswith("sse2.psrl.dq") || // Added in 3.7 |
| Name.startswith("avx2.psll.dq") || // Added in 3.7 |
| Name.startswith("avx2.psrl.dq") || // Added in 3.7 |
| Name.startswith("avx512.psll.dq") || // Added in 3.9 |
| Name.startswith("avx512.psrl.dq") || // Added in 3.9 |
| Name == "sse41.pblendw" || // Added in 3.7 |
| Name.startswith("sse41.blendp") || // Added in 3.7 |
| Name.startswith("avx.blend.p") || // Added in 3.7 |
| Name == "avx2.pblendw" || // Added in 3.7 |
| Name.startswith("avx2.pblendd.") || // Added in 3.7 |
| Name.startswith("avx.vbroadcastf128") || // Added in 4.0 |
| Name == "avx2.vbroadcasti128" || // Added in 3.7 |
| Name.startswith("avx512.mask.broadcastf") || // Added in 6.0 |
| Name.startswith("avx512.mask.broadcasti") || // Added in 6.0 |
| Name == "xop.vpcmov" || // Added in 3.8 |
| Name == "xop.vpcmov.256" || // Added in 5.0 |
| Name.startswith("avx512.mask.move.s") || // Added in 4.0 |
| Name.startswith("avx512.cvtmask2") || // Added in 5.0 |
| Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0 |
| Name.startswith("xop.vprot") || // Added in 8.0 |
| Name.startswith("avx512.prol") || // Added in 8.0 |
| Name.startswith("avx512.pror") || // Added in 8.0 |
| Name.startswith("avx512.mask.prorv.") || // Added in 8.0 |
| Name.startswith("avx512.mask.pror.") || // Added in 8.0 |
| Name.startswith("avx512.mask.prolv.") || // Added in 8.0 |
| Name.startswith("avx512.mask.prol.") || // Added in 8.0 |
| Name.startswith("avx512.ptestm") || //Added in 6.0 |
| Name.startswith("avx512.ptestnm") || //Added in 6.0 |
| Name.startswith("avx512.mask.pavg")) // Added in 6.0 |
| return true; |
| |
| return false; |
| } |
| |
| static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, |
| Function *&NewFn) { |
| // Only handle intrinsics that start with "x86.". |
| if (!Name.startswith("x86.")) |
| return false; |
| // Remove "x86." prefix. |
| Name = Name.substr(4); |
| |
| if (ShouldUpgradeX86Intrinsic(F, Name)) { |
| NewFn = nullptr; |
| return true; |
| } |
| |
| if (Name == "rdtscp") { // Added in 8.0 |
| // If this intrinsic has 0 operands, it's the new version. |
| if (F->getFunctionType()->getNumParams() == 0) |
| return false; |
| |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::x86_rdtscp); |
| return true; |
| } |
| |
| // SSE4.1 ptest functions may have an old signature. |
| if (Name.startswith("sse41.ptest")) { // Added in 3.2 |
| if (Name.substr(11) == "c") |
| return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn); |
| if (Name.substr(11) == "z") |
| return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn); |
| if (Name.substr(11) == "nzc") |
| return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn); |
| } |
| // Several blend and other instructions with masks used the wrong number of |
| // bits. |
| if (Name == "sse41.insertps") // Added in 3.6 |
| return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, |
| NewFn); |
| if (Name == "sse41.dppd") // Added in 3.6 |
| return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, |
| NewFn); |
| if (Name == "sse41.dpps") // Added in 3.6 |
| return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, |
| NewFn); |
| if (Name == "sse41.mpsadbw") // Added in 3.6 |
| return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, |
| NewFn); |
| if (Name == "avx.dp.ps.256") // Added in 3.6 |
| return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, |
| NewFn); |
| if (Name == "avx2.mpsadbw") // Added in 3.6 |
| return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, |
| NewFn); |
| |
| // frcz.ss/sd may need to have an argument dropped. Added in 3.2 |
| if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::x86_xop_vfrcz_ss); |
| return true; |
| } |
| if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::x86_xop_vfrcz_sd); |
| return true; |
| } |
| // Upgrade any XOP PERMIL2 index operand still using a float/double vector. |
| if (Name.startswith("xop.vpermil2")) { // Added in 3.9 |
| auto Idx = F->getFunctionType()->getParamType(2); |
| if (Idx->isFPOrFPVectorTy()) { |
| rename(F); |
| unsigned IdxSize = Idx->getPrimitiveSizeInBits(); |
| unsigned EltSize = Idx->getScalarSizeInBits(); |
| Intrinsic::ID Permil2ID; |
| if (EltSize == 64 && IdxSize == 128) |
| Permil2ID = Intrinsic::x86_xop_vpermil2pd; |
| else if (EltSize == 32 && IdxSize == 128) |
| Permil2ID = Intrinsic::x86_xop_vpermil2ps; |
| else if (EltSize == 64 && IdxSize == 256) |
| Permil2ID = Intrinsic::x86_xop_vpermil2pd_256; |
| else |
| Permil2ID = Intrinsic::x86_xop_vpermil2ps_256; |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID); |
| return true; |
| } |
| } |
| |
| if (Name == "seh.recoverfp") { |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); |
| return true; |
| } |
| |
| return false; |
| } |
| |
| static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { |
| assert(F && "Illegal to upgrade a non-existent Function."); |
| |
| // Quickly eliminate it, if it's not a candidate. |
| StringRef Name = F->getName(); |
| if (Name.size() <= 8 || !Name.startswith("llvm.")) |
| return false; |
| Name = Name.substr(5); // Strip off "llvm." |
| |
| switch (Name[0]) { |
| default: break; |
| case 'a': { |
| if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) { |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, |
| F->arg_begin()->getType()); |
| return true; |
| } |
| if (Name.startswith("arm.neon.vclz")) { |
| Type* args[2] = { |
| F->arg_begin()->getType(), |
| Type::getInt1Ty(F->getContext()) |
| }; |
| // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to |
| // the end of the name. Change name from llvm.arm.neon.vclz.* to |
| // llvm.ctlz.* |
| FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); |
| NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(), |
| "llvm.ctlz." + Name.substr(14), F->getParent()); |
| return true; |
| } |
| if (Name.startswith("arm.neon.vcnt")) { |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, |
| F->arg_begin()->getType()); |
| return true; |
| } |
| static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$"); |
| if (vldRegex.match(Name)) { |
| auto fArgs = F->getFunctionType()->params(); |
| SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end()); |
| // Can't use Intrinsic::getDeclaration here as the return types might |
| // then only be structurally equal. |
| FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false); |
| NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(), |
| "llvm." + Name + ".p0i8", F->getParent()); |
| return true; |
| } |
| static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$"); |
| if (vstRegex.match(Name)) { |
| static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1, |
| Intrinsic::arm_neon_vst2, |
| Intrinsic::arm_neon_vst3, |
| Intrinsic::arm_neon_vst4}; |
| |
| static const Intrinsic::ID StoreLaneInts[] = { |
| Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, |
| Intrinsic::arm_neon_vst4lane |
| }; |
| |
| auto fArgs = F->getFunctionType()->params(); |
| Type *Tys[] = {fArgs[0], fArgs[1]}; |
| if (Name.find("lane") == StringRef::npos) |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| StoreInts[fArgs.size() - 3], Tys); |
| else |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| StoreLaneInts[fArgs.size() - 5], Tys); |
| return true; |
| } |
| if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") { |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); |
| return true; |
| } |
| if (Name.startswith("aarch64.neon.addp")) { |
| if (F->arg_size() != 2) |
| break; // Invalid IR. |
| auto fArgs = F->getFunctionType()->params(); |
| VectorType *ArgTy = dyn_cast<VectorType>(fArgs[0]); |
| if (ArgTy && ArgTy->getElementType()->isFloatingPointTy()) { |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::aarch64_neon_faddp, fArgs); |
| return true; |
| } |
| } |
| break; |
| } |
| |
| case 'c': { |
| if (Name.startswith("ctlz.") && F->arg_size() == 1) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, |
| F->arg_begin()->getType()); |
| return true; |
| } |
| if (Name.startswith("cttz.") && F->arg_size() == 1) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, |
| F->arg_begin()->getType()); |
| return true; |
| } |
| break; |
| } |
| case 'd': { |
| if (Name == "dbg.value" && F->arg_size() == 4) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); |
| return true; |
| } |
| break; |
| } |
| case 'e': { |
| SmallVector<StringRef, 2> Groups; |
| static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+"); |
| if (R.match(Name, &Groups)) { |
| Intrinsic::ID ID = Intrinsic::not_intrinsic; |
| if (Groups[1] == "fadd") |
| ID = Intrinsic::experimental_vector_reduce_v2_fadd; |
| if (Groups[1] == "fmul") |
| ID = Intrinsic::experimental_vector_reduce_v2_fmul; |
| |
| if (ID != Intrinsic::not_intrinsic) { |
| rename(F); |
| auto Args = F->getFunctionType()->params(); |
| Type *Tys[] = {F->getFunctionType()->getReturnType(), Args[1]}; |
| NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); |
| return true; |
| } |
| } |
| break; |
| } |
| case 'i': |
| case 'l': { |
| bool IsLifetimeStart = Name.startswith("lifetime.start"); |
| if (IsLifetimeStart || Name.startswith("invariant.start")) { |
| Intrinsic::ID ID = IsLifetimeStart ? |
| Intrinsic::lifetime_start : Intrinsic::invariant_start; |
| auto Args = F->getFunctionType()->params(); |
| Type* ObjectPtr[1] = {Args[1]}; |
| if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); |
| return true; |
| } |
| } |
| |
| bool IsLifetimeEnd = Name.startswith("lifetime.end"); |
| if (IsLifetimeEnd || Name.startswith("invariant.end")) { |
| Intrinsic::ID ID = IsLifetimeEnd ? |
| Intrinsic::lifetime_end : Intrinsic::invariant_end; |
| |
| auto Args = F->getFunctionType()->params(); |
| Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]}; |
| if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); |
| return true; |
| } |
| } |
| if (Name.startswith("invariant.group.barrier")) { |
| // Rename invariant.group.barrier to launder.invariant.group |
| auto Args = F->getFunctionType()->params(); |
| Type* ObjectPtr[1] = {Args[0]}; |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::launder_invariant_group, ObjectPtr); |
| return true; |
| |
| } |
| |
| break; |
| } |
| case 'm': { |
| if (Name.startswith("masked.load.")) { |
| Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() }; |
| if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::masked_load, |
| Tys); |
| return true; |
| } |
| } |
| if (Name.startswith("masked.store.")) { |
| auto Args = F->getFunctionType()->params(); |
| Type *Tys[] = { Args[0], Args[1] }; |
| if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::masked_store, |
| Tys); |
| return true; |
| } |
| } |
| // Renaming gather/scatter intrinsics with no address space overloading |
| // to the new overload which includes an address space |
| if (Name.startswith("masked.gather.")) { |
| Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()}; |
| if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::masked_gather, Tys); |
| return true; |
| } |
| } |
| if (Name.startswith("masked.scatter.")) { |
| auto Args = F->getFunctionType()->params(); |
| Type *Tys[] = {Args[0], Args[1]}; |
| if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::masked_scatter, Tys); |
| return true; |
| } |
| } |
| // Updating the memory intrinsics (memcpy/memmove/memset) that have an |
| // alignment parameter to embedding the alignment as an attribute of |
| // the pointer args. |
| if (Name.startswith("memcpy.") && F->arg_size() == 5) { |
| rename(F); |
| // Get the types of dest, src, and len |
| ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy, |
| ParamTypes); |
| return true; |
| } |
| if (Name.startswith("memmove.") && F->arg_size() == 5) { |
| rename(F); |
| // Get the types of dest, src, and len |
| ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove, |
| ParamTypes); |
| return true; |
| } |
| if (Name.startswith("memset.") && F->arg_size() == 5) { |
| rename(F); |
| // Get the types of dest, and len |
| const auto *FT = F->getFunctionType(); |
| Type *ParamTypes[2] = { |
| FT->getParamType(0), // Dest |
| FT->getParamType(2) // len |
| }; |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset, |
| ParamTypes); |
| return true; |
| } |
| break; |
| } |
| case 'n': { |
| if (Name.startswith("nvvm.")) { |
| Name = Name.substr(5); |
| |
| // The following nvvm intrinsics correspond exactly to an LLVM intrinsic. |
| Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name) |
| .Cases("brev32", "brev64", Intrinsic::bitreverse) |
| .Case("clz.i", Intrinsic::ctlz) |
| .Case("popc.i", Intrinsic::ctpop) |
| .Default(Intrinsic::not_intrinsic); |
| if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) { |
| NewFn = Intrinsic::getDeclaration(F->getParent(), IID, |
| {F->getReturnType()}); |
| return true; |
| } |
| |
| // The following nvvm intrinsics correspond exactly to an LLVM idiom, but |
| // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall. |
| // |
| // TODO: We could add lohi.i2d. |
| bool Expand = StringSwitch<bool>(Name) |
| .Cases("abs.i", "abs.ll", true) |
| .Cases("clz.ll", "popc.ll", "h2f", true) |
| .Cases("max.i", "max.ll", "max.ui", "max.ull", true) |
| .Cases("min.i", "min.ll", "min.ui", "min.ull", true) |
| .StartsWith("atomic.load.add.f32.p", true) |
| .StartsWith("atomic.load.add.f64.p", true) |
| .Default(false); |
| if (Expand) { |
| NewFn = nullptr; |
| return true; |
| } |
| } |
| break; |
| } |
| case 'o': |
| // We only need to change the name to match the mangling including the |
| // address space. |
| if (Name.startswith("objectsize.")) { |
| Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() }; |
| if (F->arg_size() == 2 || F->arg_size() == 3 || |
| F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) { |
| rename(F); |
| NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, |
| Tys); |
| return true; |
| } |
| } |
| break; |
| |
| case 'p': |
| if (Name == "prefetch") { |
| // Handle address space overloading. |
| Type *Tys[] = {F->arg_begin()->getType()}; |
| if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) { |
| rename(F); |
| NewFn = |
| Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys); |
| return true; |
| } |
| } |
| break; |
| |
| case 's': |
| if (Name == "stackprotectorcheck") { |
| NewFn = nullptr; |
| return true; |
| } |
| break; |
| |
| case 'x': |
| if (UpgradeX86IntrinsicFunction(F, Name, NewFn)) |
| return true; |
| } |
| // Remangle our intrinsic since we upgrade the mangling |
| auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F); |
| if (Result != None) { |
| NewFn = Result.getValue(); |
| return true; |
| } |
| |
| // This may not belong here. This function is effectively being overloaded |
| // to both detect an intrinsic which needs upgrading, and to provide the |
| // upgraded form of the intrinsic. We should perhaps have two separate |
| // functions for this. |
| return false; |
| } |
| |
| bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { |
| NewFn = nullptr; |
| bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); |
| assert(F != NewFn && "Intrinsic function upgraded to the same function"); |
| |
| // Upgrade intrinsic attributes. This does not change the function. |
| if (NewFn) |
| F = NewFn; |
| if (Intrinsic::ID id = F->getIntrinsicID()) |
| F->setAttributes(Intrinsic::getAttributes(F->getContext(), id)); |
| return Upgraded; |
| } |
| |
| GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) { |
| if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" || |
| GV->getName() == "llvm.global_dtors")) || |
| !GV->hasInitializer()) |
| return nullptr; |
| ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType()); |
| if (!ATy) |
| return nullptr; |
| StructType *STy = dyn_cast<StructType>(ATy->getElementType()); |
| if (!STy || STy->getNumElements() != 2) |
| return nullptr; |
| |
| LLVMContext &C = GV->getContext(); |
| IRBuilder<> IRB(C); |
| auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1), |
| IRB.getInt8PtrTy()); |
| Constant *Init = GV->getInitializer(); |
| unsigned N = Init->getNumOperands(); |
| std::vector<Constant *> NewCtors(N); |
| for (unsigned i = 0; i != N; ++i) { |
| auto Ctor = cast<Constant>(Init->getOperand(i)); |
| NewCtors[i] = ConstantStruct::get( |
| EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1), |
| Constant::getNullValue(IRB.getInt8PtrTy())); |
| } |
| Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors); |
| |
| return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(), |
| NewInit, GV->getName()); |
| } |
| |
| // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them |
| // to byte shuffles. |
| static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, |
| Value *Op, unsigned Shift) { |
| Type *ResultTy = Op->getType(); |
| unsigned NumElts = ResultTy->getVectorNumElements() * 8; |
| |
| // Bitcast from a 64-bit element type to a byte element type. |
| Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); |
| Op = Builder.CreateBitCast(Op, VecTy, "cast"); |
| |
| // We'll be shuffling in zeroes. |
| Value *Res = Constant::getNullValue(VecTy); |
| |
| // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, |
| // we'll just return the zero vector. |
| if (Shift < 16) { |
| uint32_t Idxs[64]; |
| // 256/512-bit version is split into 2/4 16-byte lanes. |
| for (unsigned l = 0; l != NumElts; l += 16) |
| for (unsigned i = 0; i != 16; ++i) { |
| unsigned Idx = NumElts + i - Shift; |
| if (Idx < NumElts) |
| Idx -= NumElts - 16; // end of lane, switch operand. |
| Idxs[l + i] = Idx + l; |
| } |
| |
| Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts)); |
| } |
| |
| // Bitcast back to a 64-bit element type. |
| return Builder.CreateBitCast(Res, ResultTy, "cast"); |
| } |
| |
| // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them |
| // to byte shuffles. |
| static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, |
| unsigned Shift) { |
| Type *ResultTy = Op->getType(); |
| unsigned NumElts = ResultTy->getVectorNumElements() * 8; |
| |
| // Bitcast from a 64-bit element type to a byte element type. |
| Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); |
| Op = Builder.CreateBitCast(Op, VecTy, "cast"); |
| |
| // We'll be shuffling in zeroes. |
| Value *Res = Constant::getNullValue(VecTy); |
| |
| // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, |
| // we'll just return the zero vector. |
| if (Shift < 16) { |
| uint32_t Idxs[64]; |
| // 256/512-bit version is split into 2/4 16-byte lanes. |
| for (unsigned l = 0; l != NumElts; l += 16) |
| for (unsigned i = 0; i != 16; ++i) { |
| unsigned Idx = i + Shift; |
| if (Idx >= 16) |
| Idx += NumElts - 16; // end of lane, switch operand. |
| Idxs[l + i] = Idx + l; |
| } |
| |
| Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts)); |
| } |
| |
| // Bitcast back to a 64-bit element type. |
| return Builder.CreateBitCast(Res, ResultTy, "cast"); |
| } |
| |
| static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask, |
| unsigned NumElts) { |
| llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), |
| cast<IntegerType>(Mask->getType())->getBitWidth()); |
| Mask = Builder.CreateBitCast(Mask, MaskTy); |
| |
| // If we have less than 8 elements, then the starting mask was an i8 and |
| // we need to extract down to the right number of elements. |
| if (NumElts < 8) { |
| uint32_t Indices[4]; |
| for (unsigned i = 0; i != NumElts; ++i) |
| Indices[i] = i; |
| Mask = Builder.CreateShuffleVector(Mask, Mask, |
| makeArrayRef(Indices, NumElts), |
| "extract"); |
| } |
| |
| return Mask; |
| } |
| |
| static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask, |
| Value *Op0, Value *Op1) { |
| // If the mask is all ones just emit the first operation. |
| if (const auto *C = dyn_cast<Constant>(Mask)) |
| if (C->isAllOnesValue()) |
| return Op0; |
| |
| Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements()); |
| return Builder.CreateSelect(Mask, Op0, Op1); |
| } |
| |
| static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, |
| Value *Op0, Value *Op1) { |
| // If the mask is all ones just emit the first operation. |
| if (const auto *C = dyn_cast<Constant>(Mask)) |
| if (C->isAllOnesValue()) |
| return Op0; |
| |
| llvm::VectorType *MaskTy = |
| llvm::VectorType::get(Builder.getInt1Ty(), |
| Mask->getType()->getIntegerBitWidth()); |
| Mask = Builder.CreateBitCast(Mask, MaskTy); |
| Mask = Builder.CreateExtractElement(Mask, (uint64_t)0); |
| return Builder.CreateSelect(Mask, Op0, Op1); |
| } |
| |
| // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics. |
| // PALIGNR handles large immediates by shifting while VALIGN masks the immediate |
| // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes. |
| static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, |
| Value *Op1, Value *Shift, |
| Value *Passthru, Value *Mask, |
| bool IsVALIGN) { |
| unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); |
| |
| unsigned NumElts = Op0->getType()->getVectorNumElements(); |
| assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!"); |
| assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!"); |
| assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!"); |
| |
| // Mask the immediate for VALIGN. |
| if (IsVALIGN) |
| ShiftVal &= (NumElts - 1); |
| |
| // If palignr is shifting the pair of vectors more than the size of two |
| // lanes, emit zero. |
| if (ShiftVal >= 32) |
| return llvm::Constant::getNullValue(Op0->getType()); |
| |
| // If palignr is shifting the pair of input vectors more than one lane, |
| // but less than two lanes, convert to shifting in zeroes. |
| if (ShiftVal > 16) { |
| ShiftVal -= 16; |
| Op1 = Op0; |
| Op0 = llvm::Constant::getNullValue(Op0->getType()); |
| } |
| |
| uint32_t Indices[64]; |
| // 256-bit palignr operates on 128-bit lanes so we need to handle that |
| for (unsigned l = 0; l < NumElts; l += 16) { |
| for (unsigned i = 0; i != 16; ++i) { |
| unsigned Idx = ShiftVal + i; |
| if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN. |
| Idx += NumElts - 16; // End of lane, switch operand. |
| Indices[l + i] = Idx + l; |
| } |
| } |
| |
| Value *Align = Builder.CreateShuffleVector(Op1, Op0, |
| makeArrayRef(Indices, NumElts), |
| "palignr"); |
| |
| return EmitX86Select(Builder, Mask, Align, Passthru); |
| } |
| |
| static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI, |
| bool ZeroMask, bool IndexForm) { |
| Type *Ty = CI.getType(); |
| unsigned VecWidth = Ty->getPrimitiveSizeInBits(); |
| unsigned EltWidth = Ty->getScalarSizeInBits(); |
| bool IsFloat = Ty->isFPOrFPVectorTy(); |
| Intrinsic::ID IID; |
| if (VecWidth == 128 && EltWidth == 32 && IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_ps_128; |
| else if (VecWidth == 128 && EltWidth == 32 && !IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_d_128; |
| else if (VecWidth == 128 && EltWidth == 64 && IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_pd_128; |
| else if (VecWidth == 128 && EltWidth == 64 && !IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_q_128; |
| else if (VecWidth == 256 && EltWidth == 32 && IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_ps_256; |
| else if (VecWidth == 256 && EltWidth == 32 && !IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_d_256; |
| else if (VecWidth == 256 && EltWidth == 64 && IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_pd_256; |
| else if (VecWidth == 256 && EltWidth == 64 && !IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_q_256; |
| else if (VecWidth == 512 && EltWidth == 32 && IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_ps_512; |
| else if (VecWidth == 512 && EltWidth == 32 && !IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_d_512; |
| else if (VecWidth == 512 && EltWidth == 64 && IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_pd_512; |
| else if (VecWidth == 512 && EltWidth == 64 && !IsFloat) |
| IID = Intrinsic::x86_avx512_vpermi2var_q_512; |
| else if (VecWidth == 128 && EltWidth == 16) |
| IID = Intrinsic::x86_avx512_vpermi2var_hi_128; |
| else if (VecWidth == 256 && EltWidth == 16) |
| IID = Intrinsic::x86_avx512_vpermi2var_hi_256; |
| else if (VecWidth == 512 && EltWidth == 16) |
| IID = Intrinsic::x86_avx512_vpermi2var_hi_512; |
| else if (VecWidth == 128 && EltWidth == 8) |
| IID = Intrinsic::x86_avx512_vpermi2var_qi_128; |
| else if (VecWidth == 256 && EltWidth == 8) |
| IID = Intrinsic::x86_avx512_vpermi2var_qi_256; |
| else if (VecWidth == 512 && EltWidth == 8) |
| IID = Intrinsic::x86_avx512_vpermi2var_qi_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| |
| Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), |
| CI.getArgOperand(2) }; |
| |
| // If this isn't index form we need to swap operand 0 and 1. |
| if (!IndexForm) |
| std::swap(Args[0], Args[1]); |
| |
| Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), |
| Args); |
| Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) |
| : Builder.CreateBitCast(CI.getArgOperand(1), |
| Ty); |
| return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru); |
| } |
| |
| static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI, |
| bool IsSigned, bool IsAddition) { |
| Type *Ty = CI.getType(); |
| Value *Op0 = CI.getOperand(0); |
| Value *Op1 = CI.getOperand(1); |
| |
| Intrinsic::ID IID = |
| IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat) |
| : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat); |
| Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); |
| Value *Res = Builder.CreateCall(Intrin, {Op0, Op1}); |
| |
| if (CI.getNumArgOperands() == 4) { // For masked intrinsics. |
| Value *VecSrc = CI.getOperand(2); |
| Value *Mask = CI.getOperand(3); |
| Res = EmitX86Select(Builder, Mask, Res, VecSrc); |
| } |
| return Res; |
| } |
| |
| static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI, |
| bool IsRotateRight) { |
| Type *Ty = CI.getType(); |
| Value *Src = CI.getArgOperand(0); |
| Value *Amt = CI.getArgOperand(1); |
| |
| // Amount may be scalar immediate, in which case create a splat vector. |
| // Funnel shifts amounts are treated as modulo and types are all power-of-2 so |
| // we only care about the lowest log2 bits anyway. |
| if (Amt->getType() != Ty) { |
| unsigned NumElts = Ty->getVectorNumElements(); |
| Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); |
| Amt = Builder.CreateVectorSplat(NumElts, Amt); |
| } |
| |
| Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl; |
| Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); |
| Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt}); |
| |
| if (CI.getNumArgOperands() == 4) { // For masked intrinsics. |
| Value *VecSrc = CI.getOperand(2); |
| Value *Mask = CI.getOperand(3); |
| Res = EmitX86Select(Builder, Mask, Res, VecSrc); |
| } |
| return Res; |
| } |
| |
| static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm, |
| bool IsSigned) { |
| Type *Ty = CI.getType(); |
| Value *LHS = CI.getArgOperand(0); |
| Value *RHS = CI.getArgOperand(1); |
| |
| CmpInst::Predicate Pred; |
| switch (Imm) { |
| case 0x0: |
| Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; |
| break; |
| case 0x1: |
| Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; |
| break; |
| case 0x2: |
| Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; |
| break; |
| case 0x3: |
| Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; |
| break; |
| case 0x4: |
| Pred = ICmpInst::ICMP_EQ; |
| break; |
| case 0x5: |
| Pred = ICmpInst::ICMP_NE; |
| break; |
| case 0x6: |
| return Constant::getNullValue(Ty); // FALSE |
| case 0x7: |
| return Constant::getAllOnesValue(Ty); // TRUE |
| default: |
| llvm_unreachable("Unknown XOP vpcom/vpcomu predicate"); |
| } |
| |
| Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS); |
| Value *Ext = Builder.CreateSExt(Cmp, Ty); |
| return Ext; |
| } |
| |
| static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI, |
| bool IsShiftRight, bool ZeroMask) { |
| Type *Ty = CI.getType(); |
| Value *Op0 = CI.getArgOperand(0); |
| Value *Op1 = CI.getArgOperand(1); |
| Value *Amt = CI.getArgOperand(2); |
| |
| if (IsShiftRight) |
| std::swap(Op0, Op1); |
| |
| // Amount may be scalar immediate, in which case create a splat vector. |
| // Funnel shifts amounts are treated as modulo and types are all power-of-2 so |
| // we only care about the lowest log2 bits anyway. |
| if (Amt->getType() != Ty) { |
| unsigned NumElts = Ty->getVectorNumElements(); |
| Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); |
| Amt = Builder.CreateVectorSplat(NumElts, Amt); |
| } |
| |
| Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl; |
| Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); |
| Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt}); |
| |
| unsigned NumArgs = CI.getNumArgOperands(); |
| if (NumArgs >= 4) { // For masked intrinsics. |
| Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : |
| ZeroMask ? ConstantAggregateZero::get(CI.getType()) : |
| CI.getArgOperand(0); |
| Value *Mask = CI.getOperand(NumArgs - 1); |
| Res = EmitX86Select(Builder, Mask, Res, VecSrc); |
| } |
| return Res; |
| } |
| |
| static Value *UpgradeMaskedStore(IRBuilder<> &Builder, |
| Value *Ptr, Value *Data, Value *Mask, |
| bool Aligned) { |
| // Cast the pointer to the right type. |
| Ptr = Builder.CreateBitCast(Ptr, |
| llvm::PointerType::getUnqual(Data->getType())); |
| unsigned Align = |
| Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1; |
| |
| // If the mask is all ones just emit a regular store. |
| if (const auto *C = dyn_cast<Constant>(Mask)) |
| if (C->isAllOnesValue()) |
| return Builder.CreateAlignedStore(Data, Ptr, Align); |
| |
| // Convert the mask from an integer type to a vector of i1. |
| unsigned NumElts = Data->getType()->getVectorNumElements(); |
| Mask = getX86MaskVec(Builder, Mask, NumElts); |
| return Builder.CreateMaskedStore(Data, Ptr, Align, Mask); |
| } |
| |
| static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, |
| Value *Ptr, Value *Passthru, Value *Mask, |
| bool Aligned) { |
| Type *ValTy = Passthru->getType(); |
| // Cast the pointer to the right type. |
| Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy)); |
| unsigned Align = |
| Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1; |
| |
| // If the mask is all ones just emit a regular store. |
| if (const auto *C = dyn_cast<Constant>(Mask)) |
| if (C->isAllOnesValue()) |
| return Builder.CreateAlignedLoad(ValTy, Ptr, Align); |
| |
| // Convert the mask from an integer type to a vector of i1. |
| unsigned NumElts = Passthru->getType()->getVectorNumElements(); |
| Mask = getX86MaskVec(Builder, Mask, NumElts); |
| return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru); |
| } |
| |
| static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) { |
| Value *Op0 = CI.getArgOperand(0); |
| llvm::Type *Ty = Op0->getType(); |
| Value *Zero = llvm::Constant::getNullValue(Ty); |
| Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Op0, Zero); |
| Value *Neg = Builder.CreateNeg(Op0); |
| Value *Res = Builder.CreateSelect(Cmp, Op0, Neg); |
| |
| if (CI.getNumArgOperands() == 3) |
| Res = EmitX86Select(Builder,CI.getArgOperand(2), Res, CI.getArgOperand(1)); |
| |
| return Res; |
| } |
| |
| static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI, |
| ICmpInst::Predicate Pred) { |
| Value *Op0 = CI.getArgOperand(0); |
| Value *Op1 = CI.getArgOperand(1); |
| Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1); |
| Value *Res = Builder.CreateSelect(Cmp, Op0, Op1); |
| |
| if (CI.getNumArgOperands() == 4) |
| Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); |
| |
| return Res; |
| } |
| |
| static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) { |
| Type *Ty = CI.getType(); |
| |
| // Arguments have a vXi32 type so cast to vXi64. |
| Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty); |
| Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty); |
| |
| if (IsSigned) { |
| // Shift left then arithmetic shift right. |
| Constant *ShiftAmt = ConstantInt::get(Ty, 32); |
| LHS = Builder.CreateShl(LHS, ShiftAmt); |
| LHS = Builder.CreateAShr(LHS, ShiftAmt); |
| RHS = Builder.CreateShl(RHS, ShiftAmt); |
| RHS = Builder.CreateAShr(RHS, ShiftAmt); |
| } else { |
| // Clear the upper bits. |
| Constant *Mask = ConstantInt::get(Ty, 0xffffffff); |
| LHS = Builder.CreateAnd(LHS, Mask); |
| RHS = Builder.CreateAnd(RHS, Mask); |
| } |
| |
| Value *Res = Builder.CreateMul(LHS, RHS); |
| |
| if (CI.getNumArgOperands() == 4) |
| Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); |
| |
| return Res; |
| } |
| |
| // Applying mask on vector of i1's and make sure result is at least 8 bits wide. |
| static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, |
| Value *Mask) { |
| unsigned NumElts = Vec->getType()->getVectorNumElements(); |
| if (Mask) { |
| const auto *C = dyn_cast<Constant>(Mask); |
| if (!C || !C->isAllOnesValue()) |
| Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts)); |
| } |
| |
| if (NumElts < 8) { |
| uint32_t Indices[8]; |
| for (unsigned i = 0; i != NumElts; ++i) |
| Indices[i] = i; |
| for (unsigned i = NumElts; i != 8; ++i) |
| Indices[i] = NumElts + i % NumElts; |
| Vec = Builder.CreateShuffleVector(Vec, |
| Constant::getNullValue(Vec->getType()), |
| Indices); |
| } |
| return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U))); |
| } |
| |
| static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, |
| unsigned CC, bool Signed) { |
| Value *Op0 = CI.getArgOperand(0); |
| unsigned NumElts = Op0->getType()->getVectorNumElements(); |
| |
| Value *Cmp; |
| if (CC == 3) { |
| Cmp = Constant::getNullValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); |
| } else if (CC == 7) { |
| Cmp = Constant::getAllOnesValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); |
| } else { |
| ICmpInst::Predicate Pred; |
| switch (CC) { |
| default: llvm_unreachable("Unknown condition code"); |
| case 0: Pred = ICmpInst::ICMP_EQ; break; |
| case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break; |
| case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break; |
| case 4: Pred = ICmpInst::ICMP_NE; break; |
| case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break; |
| case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break; |
| } |
| Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); |
| } |
| |
| Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1); |
| |
| return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask); |
| } |
| |
| // Replace a masked intrinsic with an older unmasked intrinsic. |
| static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI, |
| Intrinsic::ID IID) { |
| Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); |
| Value *Rep = Builder.CreateCall(Intrin, |
| { CI.getArgOperand(0), CI.getArgOperand(1) }); |
| return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); |
| } |
| |
| static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) { |
| Value* A = CI.getArgOperand(0); |
| Value* B = CI.getArgOperand(1); |
| Value* Src = CI.getArgOperand(2); |
| Value* Mask = CI.getArgOperand(3); |
| |
| Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1)); |
| Value* Cmp = Builder.CreateIsNotNull(AndNode); |
| Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0); |
| Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0); |
| Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2); |
| return Builder.CreateInsertElement(A, Select, (uint64_t)0); |
| } |
| |
| |
| static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) { |
| Value* Op = CI.getArgOperand(0); |
| Type* ReturnOp = CI.getType(); |
| unsigned NumElts = CI.getType()->getVectorNumElements(); |
| Value *Mask = getX86MaskVec(Builder, Op, NumElts); |
| return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2"); |
| } |
| |
| // Replace intrinsic with unmasked version and a select. |
| static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, |
| CallInst &CI, Value *&Rep) { |
| Name = Name.substr(12); // Remove avx512.mask. |
| |
| unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); |
| unsigned EltWidth = CI.getType()->getScalarSizeInBits(); |
| Intrinsic::ID IID; |
| if (Name.startswith("max.p")) { |
| if (VecWidth == 128 && EltWidth == 32) |
| IID = Intrinsic::x86_sse_max_ps; |
| else if (VecWidth == 128 && EltWidth == 64) |
| IID = Intrinsic::x86_sse2_max_pd; |
| else if (VecWidth == 256 && EltWidth == 32) |
| IID = Intrinsic::x86_avx_max_ps_256; |
| else if (VecWidth == 256 && EltWidth == 64) |
| IID = Intrinsic::x86_avx_max_pd_256; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("min.p")) { |
| if (VecWidth == 128 && EltWidth == 32) |
| IID = Intrinsic::x86_sse_min_ps; |
| else if (VecWidth == 128 && EltWidth == 64) |
| IID = Intrinsic::x86_sse2_min_pd; |
| else if (VecWidth == 256 && EltWidth == 32) |
| IID = Intrinsic::x86_avx_min_ps_256; |
| else if (VecWidth == 256 && EltWidth == 64) |
| IID = Intrinsic::x86_avx_min_pd_256; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pshuf.b.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_ssse3_pshuf_b_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pshuf_b; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pshuf_b_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pmul.hr.sw.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_ssse3_pmul_hr_sw_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pmul_hr_sw; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pmul_hr_sw_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pmulh.w.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse2_pmulh_w; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pmulh_w; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pmulh_w_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pmulhu.w.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse2_pmulhu_w; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pmulhu_w; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pmulhu_w_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pmaddw.d.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse2_pmadd_wd; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pmadd_wd; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pmaddw_d_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pmaddubs.w.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pmadd_ub_sw; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pmaddubs_w_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("packsswb.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse2_packsswb_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_packsswb; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_packsswb_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("packssdw.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse2_packssdw_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_packssdw; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_packssdw_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("packuswb.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse2_packuswb_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_packuswb; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_packuswb_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("packusdw.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_sse41_packusdw; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx2_packusdw; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_packusdw_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("vpermilvar.")) { |
| if (VecWidth == 128 && EltWidth == 32) |
| IID = Intrinsic::x86_avx_vpermilvar_ps; |
| else if (VecWidth == 128 && EltWidth == 64) |
| IID = Intrinsic::x86_avx_vpermilvar_pd; |
| else if (VecWidth == 256 && EltWidth == 32) |
| IID = Intrinsic::x86_avx_vpermilvar_ps_256; |
| else if (VecWidth == 256 && EltWidth == 64) |
| IID = Intrinsic::x86_avx_vpermilvar_pd_256; |
| else if (VecWidth == 512 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_vpermilvar_ps_512; |
| else if (VecWidth == 512 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_vpermilvar_pd_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name == "cvtpd2dq.256") { |
| IID = Intrinsic::x86_avx_cvt_pd2dq_256; |
| } else if (Name == "cvtpd2ps.256") { |
| IID = Intrinsic::x86_avx_cvt_pd2_ps_256; |
| } else if (Name == "cvttpd2dq.256") { |
| IID = Intrinsic::x86_avx_cvtt_pd2dq_256; |
| } else if (Name == "cvttps2dq.128") { |
| IID = Intrinsic::x86_sse2_cvttps2dq; |
| } else if (Name == "cvttps2dq.256") { |
| IID = Intrinsic::x86_avx_cvtt_ps2dq_256; |
| } else if (Name.startswith("permvar.")) { |
| bool IsFloat = CI.getType()->isFPOrFPVectorTy(); |
| if (VecWidth == 256 && EltWidth == 32 && IsFloat) |
| IID = Intrinsic::x86_avx2_permps; |
| else if (VecWidth == 256 && EltWidth == 32 && !IsFloat) |
| IID = Intrinsic::x86_avx2_permd; |
| else if (VecWidth == 256 && EltWidth == 64 && IsFloat) |
| IID = Intrinsic::x86_avx512_permvar_df_256; |
| else if (VecWidth == 256 && EltWidth == 64 && !IsFloat) |
| IID = Intrinsic::x86_avx512_permvar_di_256; |
| else if (VecWidth == 512 && EltWidth == 32 && IsFloat) |
| IID = Intrinsic::x86_avx512_permvar_sf_512; |
| else if (VecWidth == 512 && EltWidth == 32 && !IsFloat) |
| IID = Intrinsic::x86_avx512_permvar_si_512; |
| else if (VecWidth == 512 && EltWidth == 64 && IsFloat) |
| IID = Intrinsic::x86_avx512_permvar_df_512; |
| else if (VecWidth == 512 && EltWidth == 64 && !IsFloat) |
| IID = Intrinsic::x86_avx512_permvar_di_512; |
| else if (VecWidth == 128 && EltWidth == 16) |
| IID = Intrinsic::x86_avx512_permvar_hi_128; |
| else if (VecWidth == 256 && EltWidth == 16) |
| IID = Intrinsic::x86_avx512_permvar_hi_256; |
| else if (VecWidth == 512 && EltWidth == 16) |
| IID = Intrinsic::x86_avx512_permvar_hi_512; |
| else if (VecWidth == 128 && EltWidth == 8) |
| IID = Intrinsic::x86_avx512_permvar_qi_128; |
| else if (VecWidth == 256 && EltWidth == 8) |
| IID = Intrinsic::x86_avx512_permvar_qi_256; |
| else if (VecWidth == 512 && EltWidth == 8) |
| IID = Intrinsic::x86_avx512_permvar_qi_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("dbpsadbw.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_avx512_dbpsadbw_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx512_dbpsadbw_256; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_dbpsadbw_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pmultishift.qb.")) { |
| if (VecWidth == 128) |
| IID = Intrinsic::x86_avx512_pmultishift_qb_128; |
| else if (VecWidth == 256) |
| IID = Intrinsic::x86_avx512_pmultishift_qb_256; |
| else if (VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pmultishift_qb_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("conflict.")) { |
| if (Name[9] == 'd' && VecWidth == 128) |
| IID = Intrinsic::x86_avx512_conflict_d_128; |
| else if (Name[9] == 'd' && VecWidth == 256) |
| IID = Intrinsic::x86_avx512_conflict_d_256; |
| else if (Name[9] == 'd' && VecWidth == 512) |
| IID = Intrinsic::x86_avx512_conflict_d_512; |
| else if (Name[9] == 'q' && VecWidth == 128) |
| IID = Intrinsic::x86_avx512_conflict_q_128; |
| else if (Name[9] == 'q' && VecWidth == 256) |
| IID = Intrinsic::x86_avx512_conflict_q_256; |
| else if (Name[9] == 'q' && VecWidth == 512) |
| IID = Intrinsic::x86_avx512_conflict_q_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else if (Name.startswith("pavg.")) { |
| if (Name[5] == 'b' && VecWidth == 128) |
| IID = Intrinsic::x86_sse2_pavg_b; |
| else if (Name[5] == 'b' && VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pavg_b; |
| else if (Name[5] == 'b' && VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pavg_b_512; |
| else if (Name[5] == 'w' && VecWidth == 128) |
| IID = Intrinsic::x86_sse2_pavg_w; |
| else if (Name[5] == 'w' && VecWidth == 256) |
| IID = Intrinsic::x86_avx2_pavg_w; |
| else if (Name[5] == 'w' && VecWidth == 512) |
| IID = Intrinsic::x86_avx512_pavg_w_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| } else |
| return false; |
| |
| SmallVector<Value *, 4> Args(CI.arg_operands().begin(), |
| CI.arg_operands().end()); |
| Args.pop_back(); |
| Args.pop_back(); |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), |
| Args); |
| unsigned NumArgs = CI.getNumArgOperands(); |
| Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, |
| CI.getArgOperand(NumArgs - 2)); |
| return true; |
| } |
| |
| /// Upgrade comment in call to inline asm that represents an objc retain release |
| /// marker. |
| void llvm::UpgradeInlineAsmString(std::string *AsmStr) { |
| size_t Pos; |
| if (AsmStr->find("mov\tfp") == 0 && |
| AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos && |
| (Pos = AsmStr->find("# marker")) != std::string::npos) { |
| AsmStr->replace(Pos, 1, ";"); |
| } |
| return; |
| } |
| |
| /// Upgrade a call to an old intrinsic. All argument and return casting must be |
| /// provided to seamlessly integrate with existing context. |
| void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { |
| Function *F = CI->getCalledFunction(); |
| LLVMContext &C = CI->getContext(); |
| IRBuilder<> Builder(C); |
| Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); |
| |
| assert(F && "Intrinsic call is not direct?"); |
| |
| if (!NewFn) { |
| // Get the Function's name. |
| StringRef Name = F->getName(); |
| |
| assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'"); |
| Name = Name.substr(5); |
| |
| bool IsX86 = Name.startswith("x86."); |
| if (IsX86) |
| Name = Name.substr(4); |
| bool IsNVVM = Name.startswith("nvvm."); |
| if (IsNVVM) |
| Name = Name.substr(5); |
| |
| if (IsX86 && Name.startswith("sse4a.movnt.")) { |
| Module *M = F->getParent(); |
| SmallVector<Metadata *, 1> Elts; |
| Elts.push_back( |
| ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); |
| MDNode *Node = MDNode::get(C, Elts); |
| |
| Value *Arg0 = CI->getArgOperand(0); |
| Value *Arg1 = CI->getArgOperand(1); |
| |
| // Nontemporal (unaligned) store of the 0'th element of the float/double |
| // vector. |
| Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType(); |
| PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy); |
| Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast"); |
| Value *Extract = |
| Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement"); |
| |
| StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1); |
| SI->setMetadata(M->getMDKindID("nontemporal"), Node); |
| |
| // Remove intrinsic. |
| CI->eraseFromParent(); |
| return; |
| } |
| |
| if (IsX86 && (Name.startswith("avx.movnt.") || |
| Name.startswith("avx512.storent."))) { |
| Module *M = F->getParent(); |
| SmallVector<Metadata *, 1> Elts; |
| Elts.push_back( |
| ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); |
| MDNode *Node = MDNode::get(C, Elts); |
| |
| Value *Arg0 = CI->getArgOperand(0); |
| Value *Arg1 = CI->getArgOperand(1); |
| |
| // Convert the type of the pointer to a pointer to the stored type. |
| Value *BC = Builder.CreateBitCast(Arg0, |
| PointerType::getUnqual(Arg1->getType()), |
| "cast"); |
| VectorType *VTy = cast<VectorType>(Arg1->getType()); |
| StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC, |
| VTy->getBitWidth() / 8); |
| SI->setMetadata(M->getMDKindID("nontemporal"), Node); |
| |
| // Remove intrinsic. |
| CI->eraseFromParent(); |
| return; |
| } |
| |
| if (IsX86 && Name == "sse2.storel.dq") { |
| Value *Arg0 = CI->getArgOperand(0); |
| Value *Arg1 = CI->getArgOperand(1); |
| |
| Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); |
| Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); |
| Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0); |
| Value *BC = Builder.CreateBitCast(Arg0, |
| PointerType::getUnqual(Elt->getType()), |
| "cast"); |
| Builder.CreateAlignedStore(Elt, BC, 1); |
| |
| // Remove intrinsic. |
| CI->eraseFromParent(); |
| return; |
| } |
| |
| if (IsX86 && (Name.startswith("sse.storeu.") || |
| Name.startswith("sse2.storeu.") || |
| Name.startswith("avx.storeu."))) { |
| Value *Arg0 = CI->getArgOperand(0); |
| Value *Arg1 = CI->getArgOperand(1); |
| |
| Arg0 = Builder.CreateBitCast(Arg0, |
| PointerType::getUnqual(Arg1->getType()), |
| "cast"); |
| Builder.CreateAlignedStore(Arg1, Arg0, 1); |
| |
| // Remove intrinsic. |
| CI->eraseFromParent(); |
| return; |
| } |
| |
| if (IsX86 && Name == "avx512.mask.store.ss") { |
| Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); |
| UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), |
| Mask, false); |
| |
| // Remove intrinsic. |
| CI->eraseFromParent(); |
| return; |
| } |
| |
| if (IsX86 && (Name.startswith("avx512.mask.store"))) { |
| // "avx512.mask.storeu." or "avx512.mask.store." |
| bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu". |
| UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), |
| CI->getArgOperand(2), Aligned); |
| |
| // Remove intrinsic. |
| CI->eraseFromParent(); |
| return; |
| } |
| |
| Value *Rep; |
| // Upgrade packed integer vector compare intrinsics to compare instructions. |
| if (IsX86 && (Name.startswith("sse2.pcmp") || |
| Name.startswith("avx2.pcmp"))) { |
| // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt." |
| bool CmpEq = Name[9] == 'e'; |
| Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT, |
| CI->getArgOperand(0), CI->getArgOperand(1)); |
| Rep = Builder.CreateSExt(Rep, CI->getType(), ""); |
| } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) { |
| Type *ExtTy = Type::getInt32Ty(C); |
| if (CI->getOperand(0)->getType()->isIntegerTy(8)) |
| ExtTy = Type::getInt64Ty(C); |
| unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / |
| ExtTy->getPrimitiveSizeInBits(); |
| Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); |
| Rep = Builder.CreateVectorSplat(NumElts, Rep); |
| } else if (IsX86 && (Name == "sse.sqrt.ss" || |
| Name == "sse2.sqrt.sd")) { |
| Value *Vec = CI->getArgOperand(0); |
| Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0); |
| Function *Intr = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::sqrt, Elt0->getType()); |
| Elt0 = Builder.CreateCall(Intr, Elt0); |
| Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0); |
| } else if (IsX86 && (Name.startswith("avx.sqrt.p") || |
| Name.startswith("sse2.sqrt.p") || |
| Name.startswith("sse.sqrt.p"))) { |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::sqrt, |
| CI->getType()), |
| {CI->getArgOperand(0)}); |
| } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) { |
| if (CI->getNumArgOperands() == 4 && |
| (!isa<ConstantInt>(CI->getArgOperand(3)) || |
| cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { |
| Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512 |
| : Intrinsic::x86_avx512_sqrt_pd_512; |
| |
| Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) }; |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), |
| IID), Args); |
| } else { |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::sqrt, |
| CI->getType()), |
| {CI->getArgOperand(0)}); |
| } |
| Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, |
| CI->getArgOperand(1)); |
| } else if (IsX86 && (Name.startswith("avx512.ptestm") || |
| Name.startswith("avx512.ptestnm"))) { |
| Value *Op0 = CI->getArgOperand(0); |
| Value *Op1 = CI->getArgOperand(1); |
| Value *Mask = CI->getArgOperand(2); |
| Rep = Builder.CreateAnd(Op0, Op1); |
| llvm::Type *Ty = Op0->getType(); |
| Value *Zero = llvm::Constant::getNullValue(Ty); |
| ICmpInst::Predicate Pred = |
| Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ; |
| Rep = Builder.CreateICmp(Pred, Rep, Zero); |
| Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask); |
| } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){ |
| unsigned NumElts = |
| CI->getArgOperand(1)->getType()->getVectorNumElements(); |
| Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); |
| Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, |
| CI->getArgOperand(1)); |
| } else if (IsX86 && (Name.startswith("avx512.kunpck"))) { |
| unsigned NumElts = CI->getType()->getScalarSizeInBits(); |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); |
| uint32_t Indices[64]; |
| for (unsigned i = 0; i != NumElts; ++i) |
| Indices[i] = i; |
| |
| // First extract half of each vector. This gives better codegen than |
| // doing it in a single shuffle. |
| LHS = Builder.CreateShuffleVector(LHS, LHS, |
| makeArrayRef(Indices, NumElts / 2)); |
| RHS = Builder.CreateShuffleVector(RHS, RHS, |
| makeArrayRef(Indices, NumElts / 2)); |
| // Concat the vectors. |
| // NOTE: Operands have to be swapped to match intrinsic definition. |
| Rep = Builder.CreateShuffleVector(RHS, LHS, |
| makeArrayRef(Indices, NumElts)); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && Name == "avx512.kand.w") { |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); |
| Rep = Builder.CreateAnd(LHS, RHS); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && Name == "avx512.kandn.w") { |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); |
| LHS = Builder.CreateNot(LHS); |
| Rep = Builder.CreateAnd(LHS, RHS); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && Name == "avx512.kor.w") { |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); |
| Rep = Builder.CreateOr(LHS, RHS); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && Name == "avx512.kxor.w") { |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); |
| Rep = Builder.CreateXor(LHS, RHS); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && Name == "avx512.kxnor.w") { |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); |
| LHS = Builder.CreateNot(LHS); |
| Rep = Builder.CreateXor(LHS, RHS); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && Name == "avx512.knot.w") { |
| Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Rep = Builder.CreateNot(Rep); |
| Rep = Builder.CreateBitCast(Rep, CI->getType()); |
| } else if (IsX86 && |
| (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) { |
| Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); |
| Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); |
| Rep = Builder.CreateOr(LHS, RHS); |
| Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty()); |
| Value *C; |
| if (Name[14] == 'c') |
| C = ConstantInt::getAllOnesValue(Builder.getInt16Ty()); |
| else |
| C = ConstantInt::getNullValue(Builder.getInt16Ty()); |
| Rep = Builder.CreateICmpEQ(Rep, C); |
| Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty()); |
| } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" || |
| Name == "sse.sub.ss" || Name == "sse2.sub.sd" || |
| Name == "sse.mul.ss" || Name == "sse2.mul.sd" || |
| Name == "sse.div.ss" || Name == "sse2.div.sd")) { |
| Type *I32Ty = Type::getInt32Ty(C); |
| Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), |
| ConstantInt::get(I32Ty, 0)); |
| Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), |
| ConstantInt::get(I32Ty, 0)); |
| Value *EltOp; |
| if (Name.contains(".add.")) |
| EltOp = Builder.CreateFAdd(Elt0, Elt1); |
| else if (Name.contains(".sub.")) |
| EltOp = Builder.CreateFSub(Elt0, Elt1); |
| else if (Name.contains(".mul.")) |
| EltOp = Builder.CreateFMul(Elt0, Elt1); |
| else |
| EltOp = Builder.CreateFDiv(Elt0, Elt1); |
| Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, |
| ConstantInt::get(I32Ty, 0)); |
| } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) { |
| // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt." |
| bool CmpEq = Name[16] == 'e'; |
| Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); |
| } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) { |
| Type *OpTy = CI->getArgOperand(0)->getType(); |
| unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); |
| Intrinsic::ID IID; |
| switch (VecWidth) { |
| default: llvm_unreachable("Unexpected intrinsic"); |
| case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break; |
| case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break; |
| case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break; |
| } |
| |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), |
| { CI->getOperand(0), CI->getArgOperand(1) }); |
| Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); |
| } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) { |
| Type *OpTy = CI->getArgOperand(0)->getType(); |
| unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); |
| unsigned EltWidth = OpTy->getScalarSizeInBits(); |
| Intrinsic::ID IID; |
| if (VecWidth == 128 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_fpclass_ps_128; |
| else if (VecWidth == 256 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_fpclass_ps_256; |
| else if (VecWidth == 512 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_fpclass_ps_512; |
| else if (VecWidth == 128 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_fpclass_pd_128; |
| else if (VecWidth == 256 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_fpclass_pd_256; |
| else if (VecWidth == 512 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_fpclass_pd_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), |
| { CI->getOperand(0), CI->getArgOperand(1) }); |
| Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); |
| } else if (IsX86 && Name.startswith("avx512.mask.cmp.p")) { |
| Type *OpTy = CI->getArgOperand(0)->getType(); |
| unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); |
| unsigned EltWidth = OpTy->getScalarSizeInBits(); |
| Intrinsic::ID IID; |
| if (VecWidth == 128 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_cmp_ps_128; |
| else if (VecWidth == 256 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_cmp_ps_256; |
| else if (VecWidth == 512 && EltWidth == 32) |
| IID = Intrinsic::x86_avx512_cmp_ps_512; |
| else if (VecWidth == 128 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_cmp_pd_128; |
| else if (VecWidth == 256 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_cmp_pd_256; |
| else if (VecWidth == 512 && EltWidth == 64) |
| IID = Intrinsic::x86_avx512_cmp_pd_512; |
| else |
| llvm_unreachable("Unexpected intrinsic"); |
| |
| SmallVector<Value *, 4> Args; |
| Args.push_back(CI->getArgOperand(0)); |
| Args.push_back(CI->getArgOperand(1)); |
| Args.push_back(CI->getArgOperand(2)); |
| if (CI->getNumArgOperands() == 5) |
| Args.push_back(CI->getArgOperand(4)); |
| |
| Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), |
| Args); |
| Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3)); |
| } else if (IsX86 && Name.startswith("avx512.mask.cmp.") && |
| Name[16] != 'p') { |
| // Integer compare intrinsics. |
| unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); |
| Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); |
| } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) { |
| unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); |
| Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); |
| } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") || |
| Name.startswith("avx512.cvtw2mask.") || |
| Name.startswith("avx512.cvtd2mask.") || |
| Name.startswith("avx512.cvtq2mask."))) { |
| Value *Op = CI->getArgOperand(0); |
| Value *Zero = llvm::Constant::getNullValue(Op->getType()); |
| Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero); |
| Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr); |
| } else if(IsX86 && (Name == "ssse3.pabs.b.128" || |
| Name == "ssse3.pabs.w.128" || |
| Name == "ssse3.pabs.d.128" || |
| Name.startswith("avx2.pabs") || |
| Name.startswith("avx512.mask.pabs"))) { |
| Rep = upgradeAbs(Builder, *CI); |
| } else if (IsX86 && (Name == "sse41.pmaxsb" || |
| Name == "sse2.pmaxs.w" || |
| Name == "sse41.pmaxsd" || |
| Name.startswith("avx2.pmaxs") || |
| Name.startswith("avx512.mask.pmaxs"))) { |
| Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT); |
| } else if (IsX86 && (Name == "sse2.pmaxu.b" || |
| Name == "sse41.pmaxuw" || |
| Name == "sse41.pmaxud" || |
| Name.startswith("avx2.pmaxu") || |
| Name.startswith("avx512.mask.pmaxu"))) { |
| Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT); |
| } else if (IsX86 && (Name == "sse41.pminsb" || |
| Name == "sse2.pmins.w" || |
| Name == "sse41.pminsd" || |
| Name.startswith("avx2.pmins") || |
| Name.startswith("avx512.mask.pmins"))) { |
| Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT); |
| } else if (IsX86 && (Name == "sse2.pminu.b" || |
| Name == "sse41.pminuw" || |
| Name == "sse41.pminud" || |
| Name.startswith("avx2.pminu") || |
| Name.startswith("avx512.mask.pminu"))) { |
| Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT); |
| } else if (IsX86 && (Name == "sse2.pmulu.dq" || |
| Name == "avx2.pmulu.dq" || |
| Name == "avx512.pmulu.dq.512" || |
| Name.startswith("avx512.mask.pmulu.dq."))) { |
| Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false); |
| } else if (IsX86 && (Name == "sse41.pmuldq" || |
| Name == "avx2.pmul.dq" || |
| Name == "avx512.pmul.dq.512" || |
| Name.startswith("avx512.mask.pmul.dq."))) { |
| Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true); |
| } else if (IsX86 && (Name == "sse.cvtsi2ss" || |
| Name == "sse2.cvtsi2sd" || |
| Name == "sse.cvtsi642ss" || |
| Name == "sse2.cvtsi642sd")) { |
| Rep = Builder.CreateSIToFP(CI->getArgOperand(1), |
| CI->getType()->getVectorElementType()); |
| Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); |
| } else if (IsX86 && Name == "avx512.cvtusi2sd") { |
| Rep = Builder.CreateUIToFP(CI->getArgOperand(1), |
| CI->getType()->getVectorElementType()); |
| Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); |
| } else if (IsX86 && Name == "sse2.cvtss2sd") { |
| Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); |
| Rep = Builder.CreateFPExt(Rep, CI->getType()->getVectorElementType()); |
| Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); |
| } else if (IsX86 && (Name == "sse2.cvtdq2pd" || |
| Name == "sse2.cvtdq2ps" || |
| Name == "avx.cvtdq2.pd.256" || |
| Name == "avx.cvtdq2.ps.256" || |
| Name.startswith("avx512.mask.cvtdq2pd.") || |
| Name.startswith("avx512.mask.cvtudq2pd.") || |
| Name.startswith("avx512.mask.cvtdq2ps.") || |
| Name.startswith("avx512.mask.cvtudq2ps.") || |
| Name.startswith("avx512.mask.cvtqq2pd.") || |
| Name.startswith("avx512.mask.cvtuqq2pd.") || |
| Name == "avx512.mask.cvtqq2ps.256" || |
| Name == "avx512.mask.cvtqq2ps.512" || |
| Name == "avx512.mask.cvtuqq2ps.256" || |
| Name == "avx512.mask.cvtuqq2ps.512" || |
| Name == "sse2.cvtps2pd" || |
| Name == "avx.cvt.ps2.pd.256" || |
| Name == "avx512.mask.cvtps2pd.128" || |
| Name == "avx512.mask.cvtps2pd.256")) { |
| Type *DstTy = CI->getType(); |
| Rep = CI->getArgOperand(0); |
| Type *SrcTy = Rep->getType(); |
| |
| unsigned NumDstElts = DstTy->getVectorNumElements(); |
| if (NumDstElts < SrcTy->getVectorNumElements()) { |
| assert(NumDstElts == 2 && "Unexpected vector size"); |
| uint32_t ShuffleMask[2] = { 0, 1 }; |
| Rep = Builder.CreateShuffleVector(Rep, Rep, ShuffleMask); |
| } |
| |
| bool IsPS2PD = SrcTy->getVectorElementType()->isFloatTy(); |
| bool IsUnsigned = (StringRef::npos != Name.find("cvtu")); |
| if (IsPS2PD) |
| Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); |
| else if (CI->getNumArgOperands() == 4 && |
| (!isa<ConstantInt>(CI->getArgOperand(3)) || |
| cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { |
| Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round |
| : Intrinsic::x86_avx512_sitofp_round; |
| Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, |
| { DstTy, SrcTy }); |
| Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) }); |
| } else { |
| Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt") |
| : Builder.CreateSIToFP(Rep, DstTy, "cvt"); |
| } |
| |
| if (CI->getNumArgOperands() >= 3) |
| Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, |
| CI->getArgOperand(1)); |
| } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) { |
| Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), |
| CI->getArgOperand(1), CI->getArgOperand(2), |
| /*Aligned*/false); |
| } else if (IsX86 && (Name.startswith("avx512.mask.load."))) { |
| Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), |
| CI->getArgOperand(1),CI->getArgOperand(2), |
| /*Aligned*/true); |
| } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) { |
| Type *ResultTy = CI->getType(); |
| Type *PtrTy = ResultTy->getVectorElementType(); |
| |
| // Cast the pointer to element type. |
| Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), |
| llvm::PointerType::getUnqual(PtrTy)); |
| |
| Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), |
| ResultTy->getVectorNumElements()); |
| |
| Function *ELd = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::masked_expandload, |
| ResultTy); |
| Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) }); |
| } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) { |
| Type *ResultTy = CI->getArgOperand(1)->getType(); |
| Type *PtrTy = ResultTy->getVectorElementType(); |
| |
| // Cast the pointer to element type. |
| Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), |
| llvm::PointerType::getUnqual(PtrTy)); |
| |
| Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), |
| ResultTy->getVectorNumElements()); |
| |
| Function *CSt = Intrinsic::getDeclaration(F->getParent(), |
| Intrinsic::masked_compressstore, |
| ResultTy); |
| Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec }); |
| } else if (IsX86 && (Name.startswith("avx512.mask.compress.") || |
| Name.startswith("avx512.mask.expand."))) { |
| Type *ResultTy = CI->getType(); |
| |
| Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), |
| ResultTy->getVectorNumElements()); |
| |
| bool IsCompress = Name[12] == 'c'; |
| Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress |
| : Intrinsic::x86_avx512_mask_expand; |
| Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy); |
| Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1), |
| MaskVec }); |
| } else if (IsX86 && Name.startswith("xop.vpcom")) { |
| bool IsSigned; |
| if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") || |
| Name.endswith("uq")) |
| IsSigned = false; |
| else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") || |
| Name.endswith("q")) |
| IsSigned = true; |
| else |
| llvm_unreachable("Unknown suffix"); |
| |
| unsigne
|