| //===-- PPCIntrinsicCall.cpp ----------------------------------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // Helper routines for constructing the FIR dialect of MLIR for PowerPC |
| // intrinsics. Extensive use of MLIR interfaces and MLIR's coding style |
| // (https://mlir.llvm.org/getting_started/DeveloperGuide/) is used in this |
| // module. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "flang/Optimizer/Builder/PPCIntrinsicCall.h" |
| #include "flang/Evaluate/common.h" |
| #include "flang/Optimizer/Builder/FIRBuilder.h" |
| #include "flang/Optimizer/Builder/MutableBox.h" |
| #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| |
| namespace fir { |
| |
| using PI = PPCIntrinsicLibrary; |
| |
| // PPC specific intrinsic handlers. |
| static constexpr IntrinsicHandler ppcHandlers[]{ |
| {"__ppc_mma_assemble_acc", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::AssembleAcc, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"arg1", asValue}, |
| {"arg2", asValue}, |
| {"arg3", asValue}, |
| {"arg4", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_assemble_pair", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::AssemblePair, MMAHandlerOp::SubToFunc>), |
| {{{"pair", asAddr}, {"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_build_acc", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::AssembleAcc, |
| MMAHandlerOp::SubToFuncReverseArgOnLE>), |
| {{{"acc", asAddr}, |
| {"arg1", asValue}, |
| {"arg2", asValue}, |
| {"arg3", asValue}, |
| {"arg4", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_disassemble_acc", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::DisassembleAcc, MMAHandlerOp::SubToFunc>), |
| {{{"data", asAddr}, {"acc", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_disassemble_pair", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::DisassemblePair, MMAHandlerOp::SubToFunc>), |
| {{{"data", asAddr}, {"pair", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvbf16ger2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvbf16ger2, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvbf16ger2nn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvbf16ger2nn, |
| MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvbf16ger2np", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvbf16ger2np, |
| MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvbf16ger2pn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvbf16ger2pn, |
| MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvbf16ger2pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvbf16ger2pp, |
| MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf16ger2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf16ger2, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf16ger2nn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf16ger2nn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf16ger2np", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf16ger2np, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf16ger2pn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf16ger2pn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf16ger2pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf16ger2pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf32ger", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf32ger, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf32gernn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf32gernn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf32gernp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf32gernp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf32gerpn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf32gerpn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf32gerpp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf32gerpp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf64ger", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf64ger, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf64gernn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf64gernn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf64gernp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf64gernp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf64gerpn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf64gerpn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvf64gerpp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvf64gerpp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi16ger2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi16ger2, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi16ger2pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi16ger2pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi16ger2s", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi16ger2s, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi16ger2spp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi16ger2spp, |
| MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi4ger8_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi4ger8, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi4ger8pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi4ger8pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi8ger4_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi8ger4, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi8ger4pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi8ger4pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_pmxvi8ger4spp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Pmxvi8ger4spp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, |
| {"a", asValue}, |
| {"b", asValue}, |
| {"xmask", asValue}, |
| {"ymask", asValue}, |
| {"pmask", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvbf16ger2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvbf16ger2, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvbf16ger2nn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvbf16ger2nn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvbf16ger2np", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvbf16ger2np, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvbf16ger2pn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvbf16ger2pn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvbf16ger2pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvbf16ger2pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf16ger2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf16ger2, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf16ger2nn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf16ger2nn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf16ger2np", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf16ger2np, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf16ger2pn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf16ger2pn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf16ger2pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf16ger2pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf32ger", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf32ger, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf32gernn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf32gernn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf32gernp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf32gernp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf32gerpn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf32gerpn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf32gerpp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf32gerpp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf64ger", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf64ger, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf64gernn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf64gernn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf64gernp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf64gernp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf64gerpn", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf64gerpn, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvf64gerpp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvf64gerpp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi16ger2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi16ger2, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi16ger2pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi16ger2pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi16ger2s", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi16ger2s, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi16ger2spp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi16ger2spp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi4ger8_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi4ger8, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi4ger8pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi4ger8pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi8ger4_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi8ger4, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi8ger4pp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi8ger4pp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xvi8ger4spp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xvi8ger4spp, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xxmfacc", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xxmfacc, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xxmtacc", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xxmtacc, MMAHandlerOp::FirstArgIsResult>), |
| {{{"acc", asAddr}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mma_xxsetaccz", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genMmaIntr<MMAOp::Xxsetaccz, MMAHandlerOp::SubToFunc>), |
| {{{"acc", asAddr}}}, |
| /*isElemental=*/true}, |
| {"__ppc_mtfsf", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<false>), |
| {{{"mask", asValue}, {"r", asValue}}}, |
| /*isElemental=*/false}, |
| {"__ppc_mtfsfi", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<true>), |
| {{{"bf", asValue}, {"i", asValue}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_abs", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecAbs), |
| {{{"arg1", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_add", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecAddAndMulSubXor<VecOp::Add>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_and", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecAddAndMulSubXor<VecOp::And>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_any_ge", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecAnyCompare<VecOp::Anyge>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_cmpge", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecCmp<VecOp::Cmpge>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_cmpgt", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecCmp<VecOp::Cmpgt>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_cmple", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecCmp<VecOp::Cmple>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_cmplt", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecCmp<VecOp::Cmplt>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_convert", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecConvert<VecOp::Convert>), |
| {{{"v", asValue}, {"mold", asValue}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_ctf", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecConvert<VecOp::Ctf>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_cvf", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecConvert<VecOp::Cvf>), |
| {{{"arg1", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_extract", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecExtract), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_insert", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecInsert), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_ld", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdCallGrp<VecOp::Ld>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_lde", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdCallGrp<VecOp::Lde>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_ldl", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdCallGrp<VecOp::Ldl>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_lvsl", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLvsGrp<VecOp::Lvsl>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_lvsr", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLvsGrp<VecOp::Lvsr>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_lxv", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdNoCallGrp<VecOp::Lxv>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_lxvp", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdCallGrp<VecOp::Lxvp>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_mergeh", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecMerge<VecOp::Mergeh>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_mergel", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecMerge<VecOp::Mergel>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_msub", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecNmaddMsub<VecOp::Msub>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_mul", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecAddAndMulSubXor<VecOp::Mul>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_nmadd", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecNmaddMsub<VecOp::Nmadd>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_perm", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecPerm<VecOp::Perm>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_permi", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecPerm<VecOp::Permi>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sel", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecSel), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sl", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Sl>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sld", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Sld>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sldw", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Sldw>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sll", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Sll>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_slo", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Slo>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_splat", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecSplat<VecOp::Splat>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_splat_s32_", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecSplat<VecOp::Splat_s32>), |
| {{{"arg1", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_splats", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecSplat<VecOp::Splats>), |
| {{{"arg1", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sr", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Sr>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_srl", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Srl>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_sro", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecShift<VecOp::Sro>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_st", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecStore<VecOp::St>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_ste", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecStore<VecOp::Ste>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_stxv", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecXStore<VecOp::Stxv>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_stxvp", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecStore<VecOp::Stxvp>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_sub", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecAddAndMulSubXor<VecOp::Sub>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_xl", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecXlGrp), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xl_be", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdNoCallGrp<VecOp::Xlbe>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xld2_", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdCallGrp<VecOp::Xld2>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xlds", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecXlds), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xlw4_", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecLdCallGrp<VecOp::Xlw4>), |
| {{{"arg1", asValue}, {"arg2", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xor", |
| static_cast<IntrinsicLibrary::ExtendedGenerator>( |
| &PI::genVecAddAndMulSubXor<VecOp::Xor>), |
| {{{"arg1", asValue}, {"arg2", asValue}}}, |
| /*isElemental=*/true}, |
| {"__ppc_vec_xst", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecXStore<VecOp::Xst>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xst_be", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecXStore<VecOp::Xst_be>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xstd2_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecXStore<VecOp::Xstd2>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| {"__ppc_vec_xstw4_", |
| static_cast<IntrinsicLibrary::SubroutineGenerator>( |
| &PI::genVecXStore<VecOp::Xstw4>), |
| {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}}, |
| /*isElemental=*/false}, |
| }; |
| |
| static constexpr MathOperation ppcMathOperations[] = { |
| // fcfi is just another name for fcfid, there is no llvm.ppc.fcfi. |
| {"__ppc_fcfi", "llvm.ppc.fcfid", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fcfid", "llvm.ppc.fcfid", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fcfud", "llvm.ppc.fcfud", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fctid", "llvm.ppc.fctid", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fctidz", "llvm.ppc.fctidz", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fctiw", "llvm.ppc.fctiw", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fctiwz", "llvm.ppc.fctiwz", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fctudz", "llvm.ppc.fctudz", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fctuwz", "llvm.ppc.fctuwz", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fmadd", "llvm.fma.f32", |
| genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>, |
| genMathOp<mlir::math::FmaOp>}, |
| {"__ppc_fmadd", "llvm.fma.f64", |
| genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>, |
| genMathOp<mlir::math::FmaOp>}, |
| {"__ppc_fmsub", "llvm.ppc.fmsubs", |
| genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>, |
| genLibCall}, |
| {"__ppc_fmsub", "llvm.ppc.fmsub", |
| genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fnabs", "llvm.ppc.fnabss", genFuncType<Ty::Real<4>, Ty::Real<4>>, |
| genLibCall}, |
| {"__ppc_fnabs", "llvm.ppc.fnabs", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fnmadd", "llvm.ppc.fnmadds", |
| genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>, |
| genLibCall}, |
| {"__ppc_fnmadd", "llvm.ppc.fnmadd", |
| genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fnmsub", "llvm.ppc.fnmsub.f32", |
| genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>, |
| genLibCall}, |
| {"__ppc_fnmsub", "llvm.ppc.fnmsub.f64", |
| genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fre", "llvm.ppc.fre", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_fres", "llvm.ppc.fres", genFuncType<Ty::Real<4>, Ty::Real<4>>, |
| genLibCall}, |
| {"__ppc_frsqrte", "llvm.ppc.frsqrte", genFuncType<Ty::Real<8>, Ty::Real<8>>, |
| genLibCall}, |
| {"__ppc_frsqrtes", "llvm.ppc.frsqrtes", |
| genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall}, |
| {"__ppc_vec_cvbf16spn", "llvm.ppc.vsx.xvcvbf16spn", |
| genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>, genLibCall}, |
| {"__ppc_vec_cvspbf16_", "llvm.ppc.vsx.xvcvspbf16", |
| genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>, genLibCall}, |
| {"__ppc_vec_madd", "llvm.fma.v4f32", |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>, |
| Ty::RealVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_madd", "llvm.fma.v2f64", |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>, |
| Ty::RealVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsb", |
| genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>, |
| Ty::IntegerVector<1>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsh", |
| genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>, |
| Ty::IntegerVector<2>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsw", |
| genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsd", |
| genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>, |
| Ty::IntegerVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxub", |
| genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>, |
| Ty::UnsignedVector<1>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxuh", |
| genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>, |
| Ty::UnsignedVector<2>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxuw", |
| genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>, |
| Ty::UnsignedVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.altivec.vmaxud", |
| genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>, |
| Ty::UnsignedVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.vsx.xvmaxsp", |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_max", "llvm.ppc.vsx.xvmaxdp", |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminsb", |
| genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>, |
| Ty::IntegerVector<1>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminsh", |
| genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>, |
| Ty::IntegerVector<2>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminsw", |
| genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminsd", |
| genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>, |
| Ty::IntegerVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminub", |
| genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>, |
| Ty::UnsignedVector<1>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminuh", |
| genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>, |
| Ty::UnsignedVector<2>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminuw", |
| genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>, |
| Ty::UnsignedVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.altivec.vminud", |
| genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>, |
| Ty::UnsignedVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.vsx.xvminsp", |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_min", "llvm.ppc.vsx.xvmindp", |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>, |
| genLibCall}, |
| {"__ppc_vec_nmsub", "llvm.ppc.fnmsub.v4f32", |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>, |
| Ty::RealVector<4>>, |
| genLibCall}, |
| {"__ppc_vec_nmsub", "llvm.ppc.fnmsub.v2f64", |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>, |
| Ty::RealVector<8>>, |
| genLibCall}, |
| }; |
| |
| const IntrinsicHandler *findPPCIntrinsicHandler(llvm::StringRef name) { |
| auto compare = [](const IntrinsicHandler &ppcHandler, llvm::StringRef name) { |
| return name.compare(ppcHandler.name) > 0; |
| }; |
| auto result = llvm::lower_bound(ppcHandlers, name, compare); |
| return result != std::end(ppcHandlers) && result->name == name ? result |
| : nullptr; |
| } |
| |
| using RtMap = Fortran::common::StaticMultimapView<MathOperation>; |
| static constexpr RtMap ppcMathOps(ppcMathOperations); |
| static_assert(ppcMathOps.Verify() && "map must be sorted"); |
| |
| std::pair<const MathOperation *, const MathOperation *> |
| checkPPCMathOperationsRange(llvm::StringRef name) { |
| return ppcMathOps.equal_range(name); |
| } |
| |
| // Helper functions for vector element ordering. |
| bool PPCIntrinsicLibrary::isBEVecElemOrderOnLE() { |
| const auto triple{fir::getTargetTriple(builder.getModule())}; |
| return (triple.isLittleEndian() && |
| converter->getLoweringOptions().getNoPPCNativeVecElemOrder()); |
| } |
| bool PPCIntrinsicLibrary::isNativeVecElemOrderOnLE() { |
| const auto triple{fir::getTargetTriple(builder.getModule())}; |
| return (triple.isLittleEndian() && |
| !converter->getLoweringOptions().getNoPPCNativeVecElemOrder()); |
| } |
| bool PPCIntrinsicLibrary::changeVecElemOrder() { |
| const auto triple{fir::getTargetTriple(builder.getModule())}; |
| return (triple.isLittleEndian() != |
| converter->getLoweringOptions().getNoPPCNativeVecElemOrder()); |
| } |
| |
| static mlir::FunctionType genMmaVpFuncType(mlir::MLIRContext *context, |
| int quadCnt, int pairCnt, int vecCnt, |
| int intCnt = 0, |
| int vecElemBitSize = 8, |
| int intBitSize = 32) { |
| // Constructs a function type with the following signature: |
| // Result type: __vector_pair |
| // Arguments: |
| // quadCnt: number of arguments that has __vector_quad type, followed by |
| // pairCnt: number of arguments that has __vector_pair type, followed by |
| // vecCnt: number of arguments that has vector(integer) type, followed by |
| // intCnt: number of arguments that has integer type |
| // vecElemBitSize: specifies the size of vector elements in bits |
| // intBitSize: specifies the size of integer arguments in bits |
| auto vType{mlir::VectorType::get( |
| 128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))}; |
| auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))}; |
| auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))}; |
| auto iType{mlir::IntegerType::get(context, intBitSize)}; |
| llvm::SmallVector<mlir::Type> argTypes; |
| for (int i = 0; i < quadCnt; ++i) { |
| argTypes.push_back(vqType); |
| } |
| for (int i = 0; i < pairCnt; ++i) { |
| argTypes.push_back(vpType); |
| } |
| for (int i = 0; i < vecCnt; ++i) { |
| argTypes.push_back(vType); |
| } |
| for (int i = 0; i < intCnt; ++i) { |
| argTypes.push_back(iType); |
| } |
| |
| return mlir::FunctionType::get(context, argTypes, {vpType}); |
| } |
| |
| static mlir::FunctionType genMmaVqFuncType(mlir::MLIRContext *context, |
| int quadCnt, int pairCnt, int vecCnt, |
| int intCnt = 0, |
| int vecElemBitSize = 8, |
| int intBitSize = 32) { |
| // Constructs a function type with the following signature: |
| // Result type: __vector_quad |
| // Arguments: |
| // quadCnt: number of arguments that has __vector_quad type, followed by |
| // pairCnt: number of arguments that has __vector_pair type, followed by |
| // vecCnt: number of arguments that has vector(integer) type, followed by |
| // intCnt: number of arguments that has integer type |
| // vecElemBitSize: specifies the size of vector elements in bits |
| // intBitSize: specifies the size of integer arguments in bits |
| auto vType{mlir::VectorType::get( |
| 128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))}; |
| auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))}; |
| auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))}; |
| auto iType{mlir::IntegerType::get(context, intBitSize)}; |
| llvm::SmallVector<mlir::Type> argTypes; |
| for (int i = 0; i < quadCnt; ++i) { |
| argTypes.push_back(vqType); |
| } |
| for (int i = 0; i < pairCnt; ++i) { |
| argTypes.push_back(vpType); |
| } |
| for (int i = 0; i < vecCnt; ++i) { |
| argTypes.push_back(vType); |
| } |
| for (int i = 0; i < intCnt; ++i) { |
| argTypes.push_back(iType); |
| } |
| |
| return mlir::FunctionType::get(context, argTypes, {vqType}); |
| } |
| |
| mlir::FunctionType genMmaDisassembleFuncType(mlir::MLIRContext *context, |
| MMAOp mmaOp) { |
| auto vType{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))}; |
| llvm::SmallVector<mlir::Type> members; |
| |
| if (mmaOp == MMAOp::DisassembleAcc) { |
| auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))}; |
| members.push_back(vType); |
| members.push_back(vType); |
| members.push_back(vType); |
| members.push_back(vType); |
| auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)}; |
| return mlir::FunctionType::get(context, {vqType}, {resType}); |
| } else if (mmaOp == MMAOp::DisassemblePair) { |
| auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))}; |
| members.push_back(vType); |
| members.push_back(vType); |
| auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)}; |
| return mlir::FunctionType::get(context, {vpType}, {resType}); |
| } else { |
| llvm_unreachable( |
| "Unsupported intrinsic code for function signature generator"); |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // PowerPC specific intrinsic handlers. |
| //===----------------------------------------------------------------------===// |
| |
| // MTFSF, MTFSFI |
| template <bool isImm> |
| void PPCIntrinsicLibrary::genMtfsf(llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| llvm::SmallVector<mlir::Value> scalarArgs; |
| for (const fir::ExtendedValue &arg : args) |
| if (arg.getUnboxed()) |
| scalarArgs.emplace_back(fir::getBase(arg)); |
| else |
| mlir::emitError(loc, "nonscalar intrinsic argument"); |
| |
| mlir::FunctionType libFuncType; |
| mlir::func::FuncOp funcOp; |
| if (isImm) { |
| libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Integer<4>>( |
| builder.getContext(), builder); |
| funcOp = builder.createFunction(loc, "llvm.ppc.mtfsfi", libFuncType); |
| } else { |
| libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Real<8>>( |
| builder.getContext(), builder); |
| funcOp = builder.createFunction(loc, "llvm.ppc.mtfsf", libFuncType); |
| } |
| builder.create<fir::CallOp>(loc, funcOp, scalarArgs); |
| } |
| |
| // VEC_ABS |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 1); |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| auto vTypeInfo{getVecTypeFromFir(argBases[0])}; |
| |
| mlir::func::FuncOp funcOp{nullptr}; |
| mlir::FunctionType ftype; |
| llvm::StringRef fname{}; |
| if (vTypeInfo.isFloat()) { |
| if (vTypeInfo.isFloat32()) { |
| fname = "llvm.fabs.v4f32"; |
| ftype = |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<4>>(context, builder); |
| } else if (vTypeInfo.isFloat64()) { |
| fname = "llvm.fabs.v2f64"; |
| ftype = |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<8>>(context, builder); |
| } |
| |
| funcOp = builder.createFunction(loc, fname, ftype); |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, argBases[0])}; |
| return callOp.getResult(0); |
| } else if (auto eleTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) { |
| // vec_abs(arg1) = max(0 - arg1, arg1) |
| |
| auto newVecTy{mlir::VectorType::get(vTypeInfo.len, eleTy)}; |
| auto varg1{builder.createConvert(loc, newVecTy, argBases[0])}; |
| // construct vector(0,..) |
| auto zeroVal{builder.createIntegerConstant(loc, eleTy, 0)}; |
| auto vZero{ |
| builder.create<mlir::vector::BroadcastOp>(loc, newVecTy, zeroVal)}; |
| auto zeroSubVarg1{builder.create<mlir::arith::SubIOp>(loc, vZero, varg1)}; |
| |
| mlir::func::FuncOp funcOp{nullptr}; |
| switch (eleTy.getWidth()) { |
| case 8: |
| fname = "llvm.ppc.altivec.vmaxsb"; |
| ftype = genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>, |
| Ty::IntegerVector<1>>(context, builder); |
| break; |
| case 16: |
| fname = "llvm.ppc.altivec.vmaxsh"; |
| ftype = genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>, |
| Ty::IntegerVector<2>>(context, builder); |
| break; |
| case 32: |
| fname = "llvm.ppc.altivec.vmaxsw"; |
| ftype = genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>>(context, builder); |
| break; |
| case 64: |
| fname = "llvm.ppc.altivec.vmaxsd"; |
| ftype = genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>, |
| Ty::IntegerVector<8>>(context, builder); |
| break; |
| default: |
| llvm_unreachable("invalid integer size"); |
| } |
| funcOp = builder.createFunction(loc, fname, ftype); |
| |
| mlir::Value args[] = {zeroSubVarg1, varg1}; |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, args)}; |
| return builder.createConvert(loc, argBases[0].getType(), |
| callOp.getResult(0)); |
| } |
| |
| llvm_unreachable("unknown vector type"); |
| } |
| |
| // VEC_ADD, VEC_AND, VEC_SUB, VEC_MUL, VEC_XOR |
| template <VecOp vop> |
| fir::ExtendedValue PPCIntrinsicLibrary::genVecAddAndMulSubXor( |
| mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto argBases{getBasesForArgs(args)}; |
| auto argsTy{getTypesForArgs(argBases)}; |
| assert(mlir::isa<fir::VectorType>(argsTy[0]) && |
| mlir::isa<fir::VectorType>(argsTy[1])); |
| |
| auto vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| |
| const auto isInteger{mlir::isa<mlir::IntegerType>(vecTyInfo.eleTy)}; |
| const auto isFloat{mlir::isa<mlir::FloatType>(vecTyInfo.eleTy)}; |
| assert((isInteger || isFloat) && "unknown vector type"); |
| |
| auto vargs{convertVecArgs(builder, loc, vecTyInfo, argBases)}; |
| |
| mlir::Value r{nullptr}; |
| switch (vop) { |
| case VecOp::Add: |
| if (isInteger) |
| r = builder.create<mlir::arith::AddIOp>(loc, vargs[0], vargs[1]); |
| else if (isFloat) |
| r = builder.create<mlir::arith::AddFOp>(loc, vargs[0], vargs[1]); |
| break; |
| case VecOp::Mul: |
| if (isInteger) |
| r = builder.create<mlir::arith::MulIOp>(loc, vargs[0], vargs[1]); |
| else if (isFloat) |
| r = builder.create<mlir::arith::MulFOp>(loc, vargs[0], vargs[1]); |
| break; |
| case VecOp::Sub: |
| if (isInteger) |
| r = builder.create<mlir::arith::SubIOp>(loc, vargs[0], vargs[1]); |
| else if (isFloat) |
| r = builder.create<mlir::arith::SubFOp>(loc, vargs[0], vargs[1]); |
| break; |
| case VecOp::And: |
| case VecOp::Xor: { |
| mlir::Value arg1{nullptr}; |
| mlir::Value arg2{nullptr}; |
| if (isInteger) { |
| arg1 = vargs[0]; |
| arg2 = vargs[1]; |
| } else if (isFloat) { |
| // bitcast the arguments to integer |
| auto wd{mlir::dyn_cast<mlir::FloatType>(vecTyInfo.eleTy).getWidth()}; |
| auto ftype{builder.getIntegerType(wd)}; |
| auto bcVecTy{mlir::VectorType::get(vecTyInfo.len, ftype)}; |
| arg1 = builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[0]); |
| arg2 = builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[1]); |
| } |
| if (vop == VecOp::And) |
| r = builder.create<mlir::arith::AndIOp>(loc, arg1, arg2); |
| else if (vop == VecOp::Xor) |
| r = builder.create<mlir::arith::XOrIOp>(loc, arg1, arg2); |
| |
| if (isFloat) |
| r = builder.create<mlir::vector::BitCastOp>(loc, vargs[0].getType(), r); |
| |
| break; |
| } |
| } |
| |
| return builder.createConvert(loc, argsTy[0], r); |
| } |
| |
| // VEC_ANY_GE |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecAnyCompare(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| assert(vop == VecOp::Anyge && "unknown vector compare operation"); |
| auto argBases{getBasesForArgs(args)}; |
| VecTypeInfo vTypeInfo{getVecTypeFromFir(argBases[0])}; |
| [[maybe_unused]] const auto isSupportedTy{ |
| mlir::isa<mlir::Float32Type, mlir::Float64Type, mlir::IntegerType>( |
| vTypeInfo.eleTy)}; |
| assert(isSupportedTy && "unsupported vector type"); |
| |
| // Constants for mapping CR6 bits to predicate result |
| enum { CR6_EQ_REV = 1, CR6_LT_REV = 3 }; |
| |
| auto context{builder.getContext()}; |
| |
| static std::map<std::pair<ParamTypeId, unsigned>, |
| std::pair<llvm::StringRef, mlir::FunctionType>> |
| uiBuiltin{ |
| {std::make_pair(ParamTypeId::IntegerVector, 8), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsb.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<1>, |
| Ty::IntegerVector<1>>(context, builder))}, |
| {std::make_pair(ParamTypeId::IntegerVector, 16), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsh.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<2>, |
| Ty::IntegerVector<2>>(context, builder))}, |
| {std::make_pair(ParamTypeId::IntegerVector, 32), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsw.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>>(context, builder))}, |
| {std::make_pair(ParamTypeId::IntegerVector, 64), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsd.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<8>, |
| Ty::IntegerVector<8>>(context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 8), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtub.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, |
| Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>( |
| context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 16), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtuh.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, |
| Ty::UnsignedVector<2>, Ty::UnsignedVector<2>>( |
| context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 32), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtuw.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, |
| Ty::UnsignedVector<4>, Ty::UnsignedVector<4>>( |
| context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 64), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtud.p", |
| genFuncType<Ty::Integer<4>, Ty::Integer<4>, |
| Ty::UnsignedVector<8>, Ty::UnsignedVector<8>>( |
| context, builder))}, |
| }; |
| |
| mlir::FunctionType ftype{nullptr}; |
| llvm::StringRef fname; |
| const auto i32Ty{mlir::IntegerType::get(context, 32)}; |
| llvm::SmallVector<mlir::Value> cmpArgs; |
| mlir::Value op{nullptr}; |
| const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()}; |
| |
| if (auto elementTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) { |
| std::pair<llvm::StringRef, mlir::FunctionType> bi; |
| bi = (elementTy.isUnsignedInteger()) |
| ? uiBuiltin[std::pair(ParamTypeId::UnsignedVector, width)] |
| : uiBuiltin[std::pair(ParamTypeId::IntegerVector, width)]; |
| |
| fname = std::get<0>(bi); |
| ftype = std::get<1>(bi); |
| |
| op = builder.createIntegerConstant(loc, i32Ty, CR6_LT_REV); |
| cmpArgs.emplace_back(op); |
| // reverse the argument order |
| cmpArgs.emplace_back(argBases[1]); |
| cmpArgs.emplace_back(argBases[0]); |
| } else if (vTypeInfo.isFloat()) { |
| if (vTypeInfo.isFloat32()) { |
| fname = "llvm.ppc.vsx.xvcmpgesp.p"; |
| ftype = genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::RealVector<4>, |
| Ty::RealVector<4>>(context, builder); |
| } else { |
| fname = "llvm.ppc.vsx.xvcmpgedp.p"; |
| ftype = genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::RealVector<8>, |
| Ty::RealVector<8>>(context, builder); |
| } |
| op = builder.createIntegerConstant(loc, i32Ty, CR6_EQ_REV); |
| cmpArgs.emplace_back(op); |
| cmpArgs.emplace_back(argBases[0]); |
| cmpArgs.emplace_back(argBases[1]); |
| } |
| assert((!fname.empty() && ftype) && "invalid type"); |
| |
| mlir::func::FuncOp funcOp{builder.createFunction(loc, fname, ftype)}; |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, cmpArgs)}; |
| return callOp.getResult(0); |
| } |
| |
| static std::pair<llvm::StringRef, mlir::FunctionType> |
| getVecCmpFuncTypeAndName(VecTypeInfo &vTypeInfo, VecOp vop, |
| fir::FirOpBuilder &builder) { |
| auto context{builder.getContext()}; |
| static std::map<std::pair<ParamTypeId, unsigned>, |
| std::pair<llvm::StringRef, mlir::FunctionType>> |
| iuBuiltinName{ |
| {std::make_pair(ParamTypeId::IntegerVector, 8), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsb", |
| genFuncType<Ty::UnsignedVector<1>, Ty::IntegerVector<1>, |
| Ty::IntegerVector<1>>(context, builder))}, |
| {std::make_pair(ParamTypeId::IntegerVector, 16), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsh", |
| genFuncType<Ty::UnsignedVector<2>, Ty::IntegerVector<2>, |
| Ty::IntegerVector<2>>(context, builder))}, |
| {std::make_pair(ParamTypeId::IntegerVector, 32), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsw", |
| genFuncType<Ty::UnsignedVector<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>>(context, builder))}, |
| {std::make_pair(ParamTypeId::IntegerVector, 64), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtsd", |
| genFuncType<Ty::UnsignedVector<8>, Ty::IntegerVector<8>, |
| Ty::IntegerVector<8>>(context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 8), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtub", |
| genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>, |
| Ty::UnsignedVector<1>>(context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 16), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtuh", |
| genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>, |
| Ty::UnsignedVector<2>>(context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 32), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtuw", |
| genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>, |
| Ty::UnsignedVector<4>>(context, builder))}, |
| {std::make_pair(ParamTypeId::UnsignedVector, 64), |
| std::make_pair( |
| "llvm.ppc.altivec.vcmpgtud", |
| genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>, |
| Ty::UnsignedVector<8>>(context, builder))}}; |
| |
| // VSX only defines GE and GT builtins. Cmple and Cmplt use GE and GT with |
| // arguments revsered. |
| enum class Cmp { gtOrLt, geOrLe }; |
| static std::map<std::pair<Cmp, int>, |
| std::pair<llvm::StringRef, mlir::FunctionType>> |
| rGBI{{std::make_pair(Cmp::geOrLe, 32), |
| std::make_pair("llvm.ppc.vsx.xvcmpgesp", |
| genFuncType<Ty::UnsignedVector<4>, Ty::RealVector<4>, |
| Ty::RealVector<4>>(context, builder))}, |
| {std::make_pair(Cmp::geOrLe, 64), |
| std::make_pair("llvm.ppc.vsx.xvcmpgedp", |
| genFuncType<Ty::UnsignedVector<8>, Ty::RealVector<8>, |
| Ty::RealVector<8>>(context, builder))}, |
| {std::make_pair(Cmp::gtOrLt, 32), |
| std::make_pair("llvm.ppc.vsx.xvcmpgtsp", |
| genFuncType<Ty::UnsignedVector<4>, Ty::RealVector<4>, |
| Ty::RealVector<4>>(context, builder))}, |
| {std::make_pair(Cmp::gtOrLt, 64), |
| std::make_pair("llvm.ppc.vsx.xvcmpgtdp", |
| genFuncType<Ty::UnsignedVector<8>, Ty::RealVector<8>, |
| Ty::RealVector<8>>(context, builder))}}; |
| |
| const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()}; |
| std::pair<llvm::StringRef, mlir::FunctionType> specFunc; |
| if (auto elementTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) |
| specFunc = |
| (elementTy.isUnsignedInteger()) |
| ? iuBuiltinName[std::make_pair(ParamTypeId::UnsignedVector, width)] |
| : iuBuiltinName[std::make_pair(ParamTypeId::IntegerVector, width)]; |
| else if (vTypeInfo.isFloat()) |
| specFunc = (vop == VecOp::Cmpge || vop == VecOp::Cmple) |
| ? rGBI[std::make_pair(Cmp::geOrLe, width)] |
| : rGBI[std::make_pair(Cmp::gtOrLt, width)]; |
| |
| assert(!std::get<0>(specFunc).empty() && "unknown builtin name"); |
| assert(std::get<1>(specFunc) && "unknown function type"); |
| return specFunc; |
| } |
| |
| // VEC_CMPGE, VEC_CMPGT, VEC_CMPLE, VEC_CMPLT |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| VecTypeInfo vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| auto varg{convertVecArgs(builder, loc, vecTyInfo, argBases)}; |
| |
| std::pair<llvm::StringRef, mlir::FunctionType> funcTyNam{ |
| getVecCmpFuncTypeAndName(vecTyInfo, vop, builder)}; |
| |
| mlir::func::FuncOp funcOp = builder.createFunction( |
| loc, std::get<0>(funcTyNam), std::get<1>(funcTyNam)); |
| |
| mlir::Value res{nullptr}; |
| |
| if (auto eTy = mlir::dyn_cast<mlir::IntegerType>(vecTyInfo.eleTy)) { |
| constexpr int firstArg{0}; |
| constexpr int secondArg{1}; |
| std::map<VecOp, std::array<int, 2>> argOrder{ |
| {VecOp::Cmpge, {secondArg, firstArg}}, |
| {VecOp::Cmple, {firstArg, secondArg}}, |
| {VecOp::Cmpgt, {firstArg, secondArg}}, |
| {VecOp::Cmplt, {secondArg, firstArg}}}; |
| |
| // Construct the function return type, unsigned vector, for conversion. |
| auto itype = mlir::IntegerType::get(context, eTy.getWidth(), |
| mlir::IntegerType::Unsigned); |
| auto returnType = fir::VectorType::get(vecTyInfo.len, itype); |
| |
| switch (vop) { |
| case VecOp::Cmpgt: |
| case VecOp::Cmplt: { |
| // arg1 > arg2 --> vcmpgt(arg1, arg2) |
| // arg1 < arg2 --> vcmpgt(arg2, arg1) |
| mlir::Value vargs[]{argBases[argOrder[vop][0]], |
| argBases[argOrder[vop][1]]}; |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)}; |
| res = callOp.getResult(0); |
| break; |
| } |
| case VecOp::Cmpge: |
| case VecOp::Cmple: { |
| // arg1 >= arg2 --> vcmpge(arg2, arg1) xor vector(-1) |
| // arg1 <= arg2 --> vcmpge(arg1, arg2) xor vector(-1) |
| mlir::Value vargs[]{argBases[argOrder[vop][0]], |
| argBases[argOrder[vop][1]]}; |
| |
| // Construct a constant vector(-1) |
| auto negOneVal{builder.createIntegerConstant( |
| loc, getConvertedElementType(context, eTy), -1)}; |
| auto vNegOne{builder.create<mlir::vector::BroadcastOp>( |
| loc, vecTyInfo.toMlirVectorType(context), negOneVal)}; |
| |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)}; |
| mlir::Value callRes{callOp.getResult(0)}; |
| auto vargs2{ |
| convertVecArgs(builder, loc, vecTyInfo, mlir::ValueRange{callRes})}; |
| auto xorRes{builder.create<mlir::arith::XOrIOp>(loc, vargs2[0], vNegOne)}; |
| |
| res = builder.createConvert(loc, returnType, xorRes); |
| break; |
| } |
| default: |
| llvm_unreachable("Invalid vector operation for generator"); |
| } |
| } else if (vecTyInfo.isFloat()) { |
| mlir::Value vargs[2]; |
| switch (vop) { |
| case VecOp::Cmpge: |
| case VecOp::Cmpgt: |
| vargs[0] = argBases[0]; |
| vargs[1] = argBases[1]; |
| break; |
| case VecOp::Cmple: |
| case VecOp::Cmplt: |
| // Swap the arguments as xvcmpg[et] is used |
| vargs[0] = argBases[1]; |
| vargs[1] = argBases[0]; |
| break; |
| default: |
| llvm_unreachable("Invalid vector operation for generator"); |
| } |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)}; |
| res = callOp.getResult(0); |
| } else |
| llvm_unreachable("invalid vector type"); |
| |
| return res; |
| } |
| |
| static inline mlir::Value swapVectorWordPairs(fir::FirOpBuilder &builder, |
| mlir::Location loc, |
| mlir::Value arg) { |
| auto ty = arg.getType(); |
| auto context{builder.getContext()}; |
| auto vtype{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))}; |
| |
| if (ty != vtype) |
| arg = builder.create<mlir::LLVM::BitcastOp>(loc, vtype, arg).getResult(); |
| |
| llvm::SmallVector<int64_t, 16> mask{4, 5, 6, 7, 0, 1, 2, 3, |
| 12, 13, 14, 15, 8, 9, 10, 11}; |
| arg = builder.create<mlir::vector::ShuffleOp>(loc, arg, arg, mask); |
| if (ty != vtype) |
| arg = builder.create<mlir::LLVM::BitcastOp>(loc, ty, arg); |
| return arg; |
| } |
| |
| // VEC_CONVERT, VEC_CTF, VEC_CVF |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| auto vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| auto mlirTy{vecTyInfo.toMlirVectorType(context)}; |
| auto vArg1{builder.createConvert(loc, mlirTy, argBases[0])}; |
| const auto i32Ty{mlir::IntegerType::get(context, 32)}; |
| |
| switch (vop) { |
| case VecOp::Ctf: { |
| assert(args.size() == 2); |
| auto convArg{builder.createConvert(loc, i32Ty, argBases[1])}; |
| auto eTy{mlir::dyn_cast<mlir::IntegerType>(vecTyInfo.eleTy)}; |
| assert(eTy && "Unsupported vector type"); |
| const auto isUnsigned{eTy.isUnsignedInteger()}; |
| const auto width{eTy.getWidth()}; |
| |
| if (width == 32) { |
| auto ftype{(isUnsigned) |
| ? genFuncType<Ty::RealVector<4>, Ty::UnsignedVector<4>, |
| Ty::Integer<4>>(context, builder) |
| : genFuncType<Ty::RealVector<4>, Ty::IntegerVector<4>, |
| Ty::Integer<4>>(context, builder)}; |
| const llvm::StringRef fname{(isUnsigned) ? "llvm.ppc.altivec.vcfux" |
| : "llvm.ppc.altivec.vcfsx"}; |
| auto funcOp{builder.createFunction(loc, fname, ftype)}; |
| mlir::Value newArgs[] = {argBases[0], convArg}; |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)}; |
| |
| return callOp.getResult(0); |
| } else if (width == 64) { |
| auto fTy{mlir::FloatType::getF64(context)}; |
| auto ty{mlir::VectorType::get(2, fTy)}; |
| |
| // vec_vtf(arg1, arg2) = fmul(1.0 / (1 << arg2), llvm.sitofp(arg1)) |
| auto convOp{(isUnsigned) |
| ? builder.create<mlir::LLVM::UIToFPOp>(loc, ty, vArg1) |
| : builder.create<mlir::LLVM::SIToFPOp>(loc, ty, vArg1)}; |
| |
| // construct vector<1./(1<<arg1), 1.0/(1<<arg1)> |
| auto constInt{mlir::dyn_cast_or_null<mlir::IntegerAttr>( |
| mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[1].getDefiningOp()) |
| .getValue())}; |
| assert(constInt && "expected integer constant argument"); |
| double f{1.0 / (1 << constInt.getInt())}; |
| llvm::SmallVector<double> vals{f, f}; |
| auto constOp{builder.create<mlir::arith::ConstantOp>( |
| loc, ty, builder.getF64VectorAttr(vals))}; |
| |
| auto mulOp{builder.create<mlir::LLVM::FMulOp>( |
| loc, ty, convOp->getResult(0), constOp)}; |
| |
| return builder.createConvert(loc, fir::VectorType::get(2, fTy), mulOp); |
| } |
| llvm_unreachable("invalid element integer kind"); |
| } |
| case VecOp::Convert: { |
| assert(args.size() == 2); |
| // resultType has mold type (if scalar) or element type (if array) |
| auto resTyInfo{getVecTypeFromFirType(resultType)}; |
| auto moldTy{resTyInfo.toMlirVectorType(context)}; |
| auto firTy{resTyInfo.toFirVectorType()}; |
| |
| // vec_convert(v, mold) = bitcast v to "type of mold" |
| auto conv{builder.create<mlir::LLVM::BitcastOp>(loc, moldTy, vArg1)}; |
| |
| return builder.createConvert(loc, firTy, conv); |
| } |
| case VecOp::Cvf: { |
| assert(args.size() == 1); |
| |
| mlir::Value newArgs[]{vArg1}; |
| if (vecTyInfo.isFloat32()) { |
| if (changeVecElemOrder()) |
| newArgs[0] = swapVectorWordPairs(builder, loc, newArgs[0]); |
| |
| const llvm::StringRef fname{"llvm.ppc.vsx.xvcvspdp"}; |
| auto ftype{ |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<4>>(context, builder)}; |
| auto funcOp{builder.createFunction(loc, fname, ftype)}; |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)}; |
| |
| return callOp.getResult(0); |
| } else if (vecTyInfo.isFloat64()) { |
| const llvm::StringRef fname{"llvm.ppc.vsx.xvcvdpsp"}; |
| auto ftype{ |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<8>>(context, builder)}; |
| auto funcOp{builder.createFunction(loc, fname, ftype)}; |
| newArgs[0] = |
| builder.create<fir::CallOp>(loc, funcOp, newArgs).getResult(0); |
| auto fvf32Ty{newArgs[0].getType()}; |
| auto f32type{mlir::FloatType::getF32(context)}; |
| auto mvf32Ty{mlir::VectorType::get(4, f32type)}; |
| newArgs[0] = builder.createConvert(loc, mvf32Ty, newArgs[0]); |
| |
| if (changeVecElemOrder()) |
| newArgs[0] = swapVectorWordPairs(builder, loc, newArgs[0]); |
| |
| return builder.createConvert(loc, fvf32Ty, newArgs[0]); |
| } |
| llvm_unreachable("invalid element integer kind"); |
| } |
| default: |
| llvm_unreachable("Invalid vector operation for generator"); |
| } |
| } |
| |
| static mlir::Value convertVectorElementOrder(fir::FirOpBuilder &builder, |
| mlir::Location loc, |
| VecTypeInfo vecInfo, |
| mlir::Value idx) { |
| mlir::Value numSub1{ |
| builder.createIntegerConstant(loc, idx.getType(), vecInfo.len - 1)}; |
| return builder.create<mlir::LLVM::SubOp>(loc, idx.getType(), numSub1, idx); |
| } |
| |
| // VEC_EXTRACT |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecExtract(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto argBases{getBasesForArgs(args)}; |
| auto argTypes{getTypesForArgs(argBases)}; |
| auto vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| |
| auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())}; |
| auto varg0{builder.createConvert(loc, mlirTy, argBases[0])}; |
| |
| // arg2 modulo the number of elements in arg1 to determine the element |
| // position |
| auto numEle{builder.createIntegerConstant(loc, argTypes[1], vecTyInfo.len)}; |
| mlir::Value uremOp{ |
| builder.create<mlir::LLVM::URemOp>(loc, argBases[1], numEle)}; |
| |
| if (!isNativeVecElemOrderOnLE()) |
| uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp); |
| |
| return builder.create<mlir::vector::ExtractElementOp>(loc, varg0, uremOp); |
| } |
| |
| // VEC_INSERT |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecInsert(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 3); |
| auto argBases{getBasesForArgs(args)}; |
| auto argTypes{getTypesForArgs(argBases)}; |
| auto vecTyInfo{getVecTypeFromFir(argBases[1])}; |
| auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())}; |
| auto varg1{builder.createConvert(loc, mlirTy, argBases[1])}; |
| |
| auto numEle{builder.createIntegerConstant(loc, argTypes[2], vecTyInfo.len)}; |
| mlir::Value uremOp{ |
| builder.create<mlir::LLVM::URemOp>(loc, argBases[2], numEle)}; |
| |
| if (!isNativeVecElemOrderOnLE()) |
| uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp); |
| |
| auto res{builder.create<mlir::vector::InsertElementOp>(loc, argBases[0], |
| varg1, uremOp)}; |
| return builder.create<fir::ConvertOp>(loc, vecTyInfo.toFirVectorType(), res); |
| } |
| |
| // VEC_MERGEH, VEC_MERGEL |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecMerge(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto argBases{getBasesForArgs(args)}; |
| auto vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| llvm::SmallVector<int64_t, 16> mMask; // native vector element order mask |
| llvm::SmallVector<int64_t, 16> rMask; // non-native vector element order mask |
| |
| switch (vop) { |
| case VecOp::Mergeh: { |
| switch (vecTyInfo.len) { |
| case 2: { |
| enum { V1 = 0, V2 = 2 }; |
| mMask = {V1 + 0, V2 + 0}; |
| rMask = {V2 + 1, V1 + 1}; |
| break; |
| } |
| case 4: { |
| enum { V1 = 0, V2 = 4 }; |
| mMask = {V1 + 0, V2 + 0, V1 + 1, V2 + 1}; |
| rMask = {V2 + 2, V1 + 2, V2 + 3, V1 + 3}; |
| break; |
| } |
| case 8: { |
| enum { V1 = 0, V2 = 8 }; |
| mMask = {V1 + 0, V2 + 0, V1 + 1, V2 + 1, V1 + 2, V2 + 2, V1 + 3, V2 + 3}; |
| rMask = {V2 + 4, V1 + 4, V2 + 5, V1 + 5, V2 + 6, V1 + 6, V2 + 7, V1 + 7}; |
| break; |
| } |
| case 16: |
| mMask = {0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13, |
| 0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17}; |
| rMask = {0x18, 0x08, 0x19, 0x09, 0x1A, 0x0A, 0x1B, 0x0B, |
| 0x1C, 0x0C, 0x1D, 0x0D, 0x1E, 0x0E, 0x1F, 0x0F}; |
| break; |
| default: |
| llvm_unreachable("unexpected vector length"); |
| } |
| break; |
| } |
| case VecOp::Mergel: { |
| switch (vecTyInfo.len) { |
| case 2: { |
| enum { V1 = 0, V2 = 2 }; |
| mMask = {V1 + 1, V2 + 1}; |
| rMask = {V2 + 0, V1 + 0}; |
| break; |
| } |
| case 4: { |
| enum { V1 = 0, V2 = 4 }; |
| mMask = {V1 + 2, V2 + 2, V1 + 3, V2 + 3}; |
| rMask = {V2 + 0, V1 + 0, V2 + 1, V1 + 1}; |
| break; |
| } |
| case 8: { |
| enum { V1 = 0, V2 = 8 }; |
| mMask = {V1 + 4, V2 + 4, V1 + 5, V2 + 5, V1 + 6, V2 + 6, V1 + 7, V2 + 7}; |
| rMask = {V2 + 0, V1 + 0, V2 + 1, V1 + 1, V2 + 2, V1 + 2, V2 + 3, V1 + 3}; |
| break; |
| } |
| case 16: |
| mMask = {0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B, |
| 0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F}; |
| rMask = {0x10, 0x00, 0x11, 0x01, 0x12, 0x02, 0x13, 0x03, |
| 0x14, 0x04, 0x15, 0x05, 0x16, 0x06, 0x17, 0x07}; |
| break; |
| default: |
| llvm_unreachable("unexpected vector length"); |
| } |
| break; |
| } |
| default: |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| |
| auto vargs{convertVecArgs(builder, loc, vecTyInfo, argBases)}; |
| |
| llvm::SmallVector<int64_t, 16> &mergeMask = |
| (isBEVecElemOrderOnLE()) ? rMask : mMask; |
| |
| auto callOp{builder.create<mlir::vector::ShuffleOp>(loc, vargs[0], vargs[1], |
| mergeMask)}; |
| return builder.createConvert(loc, resultType, callOp); |
| } |
| |
| static mlir::Value addOffsetToAddress(fir::FirOpBuilder &builder, |
| mlir::Location loc, mlir::Value baseAddr, |
| mlir::Value offset) { |
| auto typeExtent{fir::SequenceType::getUnknownExtent()}; |
| // Construct an !fir.ref<!ref.array<?xi8>> type |
| auto arrRefTy{builder.getRefType(fir::SequenceType::get( |
| {typeExtent}, mlir::IntegerType::get(builder.getContext(), 8)))}; |
| // Convert arg to !fir.ref<!ref.array<?xi8>> |
| auto resAddr{builder.create<fir::ConvertOp>(loc, arrRefTy, baseAddr)}; |
| |
| return builder.create<fir::CoordinateOp>(loc, arrRefTy, resAddr, offset); |
| } |
| |
| static mlir::Value reverseVectorElements(fir::FirOpBuilder &builder, |
| mlir::Location loc, mlir::Value v, |
| int64_t len) { |
| assert(mlir::isa<mlir::VectorType>(v.getType())); |
| assert(len > 0); |
| llvm::SmallVector<int64_t, 16> mask; |
| for (int64_t i = 0; i < len; ++i) { |
| mask.push_back(len - 1 - i); |
| } |
| auto undefVec{builder.create<fir::UndefOp>(loc, v.getType())}; |
| return builder.create<mlir::vector::ShuffleOp>(loc, v, undefVec, mask); |
| } |
| |
| static mlir::NamedAttribute getAlignmentAttr(fir::FirOpBuilder &builder, |
| const int val) { |
| auto i64ty{mlir::IntegerType::get(builder.getContext(), 64)}; |
| auto alignAttr{mlir::IntegerAttr::get(i64ty, val)}; |
| return builder.getNamedAttr("alignment", alignAttr); |
| } |
| |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecXlGrp(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| VecTypeInfo vecTyInfo{getVecTypeFromFirType(resultType)}; |
| switch (vecTyInfo.eleTy.getIntOrFloatBitWidth()) { |
| case 8: |
| // vec_xlb1 |
| return genVecLdNoCallGrp<VecOp::Xl>(resultType, args); |
| case 16: |
| // vec_xlh8 |
| return genVecLdNoCallGrp<VecOp::Xl>(resultType, args); |
| case 32: |
| // vec_xlw4 |
| return genVecLdCallGrp<VecOp::Xlw4>(resultType, args); |
| case 64: |
| // vec_xld2 |
| return genVecLdCallGrp<VecOp::Xld2>(resultType, args); |
| default: |
| llvm_unreachable("invalid kind"); |
| } |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| |
| template <VecOp vop> |
| fir::ExtendedValue PPCIntrinsicLibrary::genVecLdNoCallGrp( |
| mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto arg0{getBase(args[0])}; |
| auto arg1{getBase(args[1])}; |
| |
| auto vecTyInfo{getVecTypeFromFirType(resultType)}; |
| auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())}; |
| auto firTy{vecTyInfo.toFirVectorType()}; |
| |
| // Add the %val of arg0 to %addr of arg1 |
| auto addr{addOffsetToAddress(builder, loc, arg1, arg0)}; |
| |
| const auto triple{fir::getTargetTriple(builder.getModule())}; |
| // Need to get align 1. |
| auto result{builder.create<fir::LoadOp>(loc, mlirTy, addr, |
| getAlignmentAttr(builder, 1))}; |
| if ((vop == VecOp::Xl && isBEVecElemOrderOnLE()) || |
| (vop == VecOp::Xlbe && triple.isLittleEndian())) |
| return builder.createConvert( |
| loc, firTy, reverseVectorElements(builder, loc, result, vecTyInfo.len)); |
| |
| return builder.createConvert(loc, firTy, result); |
| } |
| |
| // VEC_LD, VEC_LDE, VEC_LDL, VEC_LXVP, VEC_XLD2, VEC_XLW4 |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecLdCallGrp(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto context{builder.getContext()}; |
| auto arg0{getBase(args[0])}; |
| auto arg1{getBase(args[1])}; |
| |
| // Prepare the return type in FIR. |
| auto vecResTyInfo{getVecTypeFromFirType(resultType)}; |
| auto mlirTy{vecResTyInfo.toMlirVectorType(context)}; |
| auto firTy{vecResTyInfo.toFirVectorType()}; |
| |
| // llvm.ppc.altivec.lvx* returns <4xi32> |
| // Others, like "llvm.ppc.altivec.lvebx" too if arg2 is not of Integer type |
| const auto i32Ty{mlir::IntegerType::get(builder.getContext(), 32)}; |
| const auto mVecI32Ty{mlir::VectorType::get(4, i32Ty)}; |
| |
| // For vec_ld, need to convert arg0 from i64 to i32 |
| if (vop == VecOp::Ld && arg0.getType().getIntOrFloatBitWidth() == 64) |
| arg0 = builder.createConvert(loc, i32Ty, arg0); |
| |
| // Add the %val of arg0 to %addr of arg1 |
| auto addr{addOffsetToAddress(builder, loc, arg1, arg0)}; |
| llvm::SmallVector<mlir::Value, 4> parsedArgs{addr}; |
| |
| mlir::Type intrinResTy{nullptr}; |
| llvm::StringRef fname{}; |
| switch (vop) { |
| case VecOp::Ld: |
| fname = "llvm.ppc.altivec.lvx"; |
| intrinResTy = mVecI32Ty; |
| break; |
| case VecOp::Lde: |
| switch (vecResTyInfo.eleTy.getIntOrFloatBitWidth()) { |
| case 8: |
| fname = "llvm.ppc.altivec.lvebx"; |
| intrinResTy = mlirTy; |
| break; |
| case 16: |
| fname = "llvm.ppc.altivec.lvehx"; |
| intrinResTy = mlirTy; |
| break; |
| case 32: |
| fname = "llvm.ppc.altivec.lvewx"; |
| if (mlir::isa<mlir::IntegerType>(vecResTyInfo.eleTy)) |
| intrinResTy = mlirTy; |
| else |
| intrinResTy = mVecI32Ty; |
| break; |
| default: |
| llvm_unreachable("invalid vector for vec_lde"); |
| } |
| break; |
| case VecOp::Ldl: |
| fname = "llvm.ppc.altivec.lvxl"; |
| intrinResTy = mVecI32Ty; |
| break; |
| case VecOp::Lxvp: |
| fname = "llvm.ppc.vsx.lxvp"; |
| intrinResTy = fir::VectorType::get(256, mlir::IntegerType::get(context, 1)); |
| break; |
| case VecOp::Xld2: { |
| fname = isBEVecElemOrderOnLE() ? "llvm.ppc.vsx.lxvd2x.be" |
| : "llvm.ppc.vsx.lxvd2x"; |
| // llvm.ppc.altivec.lxvd2x* returns <2 x double> |
| intrinResTy = mlir::VectorType::get(2, mlir::FloatType::getF64(context)); |
| } break; |
| case VecOp::Xlw4: |
| fname = isBEVecElemOrderOnLE() ? "llvm.ppc.vsx.lxvw4x.be" |
| : "llvm.ppc.vsx.lxvw4x"; |
| // llvm.ppc.altivec.lxvw4x* returns <4xi32> |
| intrinResTy = mVecI32Ty; |
| break; |
| default: |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| |
| auto funcType{ |
| mlir::FunctionType::get(context, {addr.getType()}, {intrinResTy})}; |
| auto funcOp{builder.createFunction(loc, fname, funcType)}; |
| auto result{ |
| builder.create<fir::CallOp>(loc, funcOp, parsedArgs).getResult(0)}; |
| |
| if (vop == VecOp::Lxvp) |
| return result; |
| |
| if (intrinResTy != mlirTy) |
| result = builder.create<mlir::vector::BitCastOp>(loc, mlirTy, result); |
| |
| if (vop != VecOp::Xld2 && vop != VecOp::Xlw4 && isBEVecElemOrderOnLE()) |
| return builder.createConvert( |
| loc, firTy, |
| reverseVectorElements(builder, loc, result, vecResTyInfo.len)); |
| |
| return builder.createConvert(loc, firTy, result); |
| } |
| |
| // VEC_LVSL, VEC_LVSR |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecLvsGrp(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto context{builder.getContext()}; |
| auto arg0{getBase(args[0])}; |
| auto arg1{getBase(args[1])}; |
| |
| auto vecTyInfo{getVecTypeFromFirType(resultType)}; |
| auto mlirTy{vecTyInfo.toMlirVectorType(context)}; |
| auto firTy{vecTyInfo.toFirVectorType()}; |
| |
| // Convert arg0 to i64 type if needed |
| auto i64ty{mlir::IntegerType::get(context, 64)}; |
| if (arg0.getType() != i64ty) |
| arg0 = builder.create<fir::ConvertOp>(loc, i64ty, arg0); |
| |
| // offset is modulo 16, so shift left 56 bits and then right 56 bits to clear |
| // upper 56 bit while preserving sign |
| auto shiftVal{builder.createIntegerConstant(loc, i64ty, 56)}; |
| auto offset{builder.create<mlir::arith::ShLIOp>(loc, arg0, shiftVal)}; |
| auto offset2{builder.create<mlir::arith::ShRSIOp>(loc, offset, shiftVal)}; |
| |
| // Add the offsetArg to %addr of arg1 |
| auto addr{addOffsetToAddress(builder, loc, arg1, offset2)}; |
| llvm::SmallVector<mlir::Value, 4> parsedArgs{addr}; |
| |
| llvm::StringRef fname{}; |
| switch (vop) { |
| case VecOp::Lvsl: |
| fname = "llvm.ppc.altivec.lvsl"; |
| break; |
| case VecOp::Lvsr: |
| fname = "llvm.ppc.altivec.lvsr"; |
| break; |
| default: |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| auto funcType{mlir::FunctionType::get(context, {addr.getType()}, {mlirTy})}; |
| auto funcOp{builder.createFunction(loc, fname, funcType)}; |
| auto result{ |
| builder.create<fir::CallOp>(loc, funcOp, parsedArgs).getResult(0)}; |
| |
| if (isNativeVecElemOrderOnLE()) |
| return builder.createConvert( |
| loc, firTy, reverseVectorElements(builder, loc, result, vecTyInfo.len)); |
| |
| return builder.createConvert(loc, firTy, result); |
| } |
| |
| // VEC_NMADD, VEC_MSUB |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecNmaddMsub(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 3); |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| auto vTypeInfo{getVecTypeFromFir(argBases[0])}; |
| auto newArgs{convertVecArgs(builder, loc, vTypeInfo, argBases)}; |
| const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()}; |
| |
| static std::map<int, std::pair<llvm::StringRef, mlir::FunctionType>> fmaMap{ |
| {32, |
| std::make_pair( |
| "llvm.fma.v4f32", |
| genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>( |
| context, builder))}, |
| {64, |
| std::make_pair( |
| "llvm.fma.v2f64", |
| genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>( |
| context, builder))}}; |
| |
| auto funcOp{builder.createFunction(loc, std::get<0>(fmaMap[width]), |
| std::get<1>(fmaMap[width]))}; |
| if (vop == VecOp::Nmadd) { |
| // vec_nmadd(arg1, arg2, arg3) = -fma(arg1, arg2, arg3) |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)}; |
| |
| // We need to convert fir.vector to MLIR vector to use fneg and then back |
| // to fir.vector to store. |
| auto vCall{builder.createConvert(loc, vTypeInfo.toMlirVectorType(context), |
| callOp.getResult(0))}; |
| auto neg{builder.create<mlir::arith::NegFOp>(loc, vCall)}; |
| return builder.createConvert(loc, vTypeInfo.toFirVectorType(), neg); |
| } else if (vop == VecOp::Msub) { |
| // vec_msub(arg1, arg2, arg3) = fma(arg1, arg2, -arg3) |
| newArgs[2] = builder.create<mlir::arith::NegFOp>(loc, newArgs[2]); |
| |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)}; |
| return callOp.getResult(0); |
| } |
| llvm_unreachable("Invalid vector operation for generator"); |
| } |
| |
| // VEC_PERM, VEC_PERMI |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 3); |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| auto argTypes{getTypesForArgs(argBases)}; |
| auto vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| auto mlirTy{vecTyInfo.toMlirVectorType(context)}; |
| |
| auto vi32Ty{mlir::VectorType::get(4, mlir::IntegerType::get(context, 32))}; |
| auto vf64Ty{mlir::VectorType::get(2, mlir::FloatType::getF64(context))}; |
| |
| auto mArg0{builder.createConvert(loc, mlirTy, argBases[0])}; |
| auto mArg1{builder.createConvert(loc, mlirTy, argBases[1])}; |
| |
| switch (vop) { |
| case VecOp::Perm: { |
| VecTypeInfo maskVecTyInfo{getVecTypeFromFir(argBases[2])}; |
| auto mlirMaskTy{maskVecTyInfo.toMlirVectorType(context)}; |
| auto mMask{builder.createConvert(loc, mlirMaskTy, argBases[2])}; |
| |
| if (mlirTy != vi32Ty) { |
| mArg0 = |
| builder.create<mlir::LLVM::BitcastOp>(loc, vi32Ty, mArg0).getResult(); |
| mArg1 = |
| builder.create<mlir::LLVM::BitcastOp>(loc, vi32Ty, mArg1).getResult(); |
| } |
| |
| auto funcOp{builder.createFunction( |
| loc, "llvm.ppc.altivec.vperm", |
| genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>, Ty::IntegerVector<1>>(context, |
| builder))}; |
| |
| llvm::SmallVector<mlir::Value> newArgs; |
| if (isNativeVecElemOrderOnLE()) { |
| auto i8Ty{mlir::IntegerType::get(context, 8)}; |
| auto v8Ty{mlir::VectorType::get(16, i8Ty)}; |
| auto negOne{builder.createMinusOneInteger(loc, i8Ty)}; |
| auto vNegOne{ |
| builder.create<mlir::vector::BroadcastOp>(loc, v8Ty, negOne)}; |
| |
| mMask = builder.create<mlir::arith::XOrIOp>(loc, mMask, vNegOne); |
| newArgs = {mArg1, mArg0, mMask}; |
| } else { |
| newArgs = {mArg0, mArg1, mMask}; |
| } |
| |
| auto res{builder.create<fir::CallOp>(loc, funcOp, newArgs).getResult(0)}; |
| |
| if (res.getType() != argTypes[0]) { |
| // fir.call llvm.ppc.altivec.vperm returns !fir.vector<i4:32> |
| // convert the result back to the original type |
| res = builder.createConvert(loc, vi32Ty, res); |
| if (mlirTy != vi32Ty) |
| res = |
| builder.create<mlir::LLVM::BitcastOp>(loc, mlirTy, res).getResult(); |
| } |
| return builder.createConvert(loc, resultType, res); |
| } |
| case VecOp::Permi: { |
| // arg3 is a constant |
| auto constIntOp{mlir::dyn_cast_or_null<mlir::IntegerAttr>( |
| mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[2].getDefiningOp()) |
| .getValue())}; |
| assert(constIntOp && "expected integer constant argument"); |
| auto constInt{constIntOp.getInt()}; |
| // arg1, arg2, and result type share same VecTypeInfo |
| if (vecTyInfo.isFloat()) { |
| mArg0 = |
| builder.create<mlir::LLVM::BitcastOp>(loc, vf64Ty, mArg0).getResult(); |
| mArg1 = |
| builder.create<mlir::LLVM::BitcastOp>(loc, vf64Ty, mArg1).getResult(); |
| } |
| |
| llvm::SmallVector<int64_t, 2> nMask; // native vector element order mask |
| llvm::SmallVector<int64_t, 2> rMask; // non-native vector element order mask |
| enum { V1 = 0, V2 = 2 }; |
| switch (constInt) { |
| case 0: |
| nMask = {V1 + 0, V2 + 0}; |
| rMask = {V2 + 1, V1 + 1}; |
| break; |
| case 1: |
| nMask = {V1 + 0, V2 + 1}; |
| rMask = {V2 + 0, V1 + 1}; |
| break; |
| case 2: |
| nMask = {V1 + 1, V2 + 0}; |
| rMask = {V2 + 1, V1 + 0}; |
| break; |
| case 3: |
| nMask = {V1 + 1, V2 + 1}; |
| rMask = {V2 + 0, V1 + 0}; |
| break; |
| default: |
| llvm_unreachable("unexpected arg3 value for vec_permi"); |
| } |
| |
| llvm::SmallVector<int64_t, 2> mask = |
| (isBEVecElemOrderOnLE()) ? rMask : nMask; |
| auto res{builder.create<mlir::vector::ShuffleOp>(loc, mArg0, mArg1, mask)}; |
| if (res.getType() != mlirTy) { |
| auto cast{builder.create<mlir::LLVM::BitcastOp>(loc, mlirTy, res)}; |
| return builder.createConvert(loc, resultType, cast); |
| } |
| return builder.createConvert(loc, resultType, res); |
| } |
| default: |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| } |
| |
| // VEC_SEL |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecSel(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 3); |
| auto argBases{getBasesForArgs(args)}; |
| llvm::SmallVector<VecTypeInfo, 4> vecTyInfos; |
| for (size_t i = 0; i < argBases.size(); i++) { |
| vecTyInfos.push_back(getVecTypeFromFir(argBases[i])); |
| } |
| auto vargs{convertVecArgs(builder, loc, vecTyInfos, argBases)}; |
| |
| auto i8Ty{mlir::IntegerType::get(builder.getContext(), 8)}; |
| auto negOne{builder.createMinusOneInteger(loc, i8Ty)}; |
| |
| // construct a constant <16 x i8> vector with value -1 for bitcast |
| auto bcVecTy{mlir::VectorType::get(16, i8Ty)}; |
| auto vNegOne{builder.create<mlir::vector::BroadcastOp>(loc, bcVecTy, negOne)}; |
| |
| // bitcast arguments to bcVecTy |
| auto arg1{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[0])}; |
| auto arg2{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[1])}; |
| auto arg3{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[2])}; |
| |
| // vec_sel(arg1, arg2, arg3) = |
| // (arg2 and arg3) or (arg1 and (arg3 xor vector(-1,...))) |
| auto comp{builder.create<mlir::arith::XOrIOp>(loc, arg3, vNegOne)}; |
| auto a1AndComp{builder.create<mlir::arith::AndIOp>(loc, arg1, comp)}; |
| auto a1OrA2{builder.create<mlir::arith::AndIOp>(loc, arg2, arg3)}; |
| auto res{builder.create<mlir::arith::OrIOp>(loc, a1AndComp, a1OrA2)}; |
| |
| auto bcRes{ |
| builder.create<mlir::vector::BitCastOp>(loc, vargs[0].getType(), res)}; |
| |
| return builder.createConvert(loc, vecTyInfos[0].toFirVectorType(), bcRes); |
| } |
| |
| // VEC_SL, VEC_SLD, VEC_SLDW, VEC_SLL, VEC_SLO, VEC_SR, VEC_SRL, VEC_SRO |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecShift(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| auto argTypes{getTypesForArgs(argBases)}; |
| |
| llvm::SmallVector<VecTypeInfo, 2> vecTyInfoArgs; |
| vecTyInfoArgs.push_back(getVecTypeFromFir(argBases[0])); |
| vecTyInfoArgs.push_back(getVecTypeFromFir(argBases[1])); |
| |
| // Convert the first two arguments to MLIR vectors |
| llvm::SmallVector<mlir::Type, 2> mlirTyArgs; |
| mlirTyArgs.push_back(vecTyInfoArgs[0].toMlirVectorType(context)); |
| mlirTyArgs.push_back(vecTyInfoArgs[1].toMlirVectorType(context)); |
| |
| llvm::SmallVector<mlir::Value, 2> mlirVecArgs; |
| mlirVecArgs.push_back(builder.createConvert(loc, mlirTyArgs[0], argBases[0])); |
| mlirVecArgs.push_back(builder.createConvert(loc, mlirTyArgs[1], argBases[1])); |
| |
| mlir::Value shftRes{nullptr}; |
| |
| if (vop == VecOp::Sl || vop == VecOp::Sr) { |
| assert(args.size() == 2); |
| // Construct the mask |
| auto width{ |
| mlir::dyn_cast<mlir::IntegerType>(vecTyInfoArgs[1].eleTy).getWidth()}; |
| auto vecVal{builder.createIntegerConstant( |
| loc, getConvertedElementType(context, vecTyInfoArgs[0].eleTy), width)}; |
| auto mask{ |
| builder.create<mlir::vector::BroadcastOp>(loc, mlirTyArgs[1], vecVal)}; |
| auto shft{builder.create<mlir::arith::RemUIOp>(loc, mlirVecArgs[1], mask)}; |
| |
| mlir::Value res{nullptr}; |
| if (vop == VecOp::Sr) |
| res = builder.create<mlir::arith::ShRUIOp>(loc, mlirVecArgs[0], shft); |
| else if (vop == VecOp::Sl) |
| res = builder.create<mlir::arith::ShLIOp>(loc, mlirVecArgs[0], shft); |
| |
| shftRes = builder.createConvert(loc, argTypes[0], res); |
| } else if (vop == VecOp::Sll || vop == VecOp::Slo || vop == VecOp::Srl || |
| vop == VecOp::Sro) { |
| assert(args.size() == 2); |
| |
| // Bitcast to vector<4xi32> |
| auto bcVecTy{mlir::VectorType::get(4, builder.getIntegerType(32))}; |
| if (mlirTyArgs[0] != bcVecTy) |
| mlirVecArgs[0] = |
| builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, mlirVecArgs[0]); |
| if (mlirTyArgs[1] != bcVecTy) |
| mlirVecArgs[1] = |
| builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, mlirVecArgs[1]); |
| |
| llvm::StringRef funcName; |
| switch (vop) { |
| case VecOp::Srl: |
| funcName = "llvm.ppc.altivec.vsr"; |
| break; |
| case VecOp::Sro: |
| funcName = "llvm.ppc.altivec.vsro"; |
| break; |
| case VecOp::Sll: |
| funcName = "llvm.ppc.altivec.vsl"; |
| break; |
| case VecOp::Slo: |
| funcName = "llvm.ppc.altivec.vslo"; |
| break; |
| default: |
| llvm_unreachable("unknown vector shift operation"); |
| } |
| auto funcTy{genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>, |
| Ty::IntegerVector<4>>(context, builder)}; |
| mlir::func::FuncOp funcOp{builder.createFunction(loc, funcName, funcTy)}; |
| auto callOp{builder.create<fir::CallOp>(loc, funcOp, mlirVecArgs)}; |
| |
| // If the result vector type is different from the original type, need |
| // to convert to mlir vector, bitcast and then convert back to fir vector. |
| if (callOp.getResult(0).getType() != argTypes[0]) { |
| auto res = builder.createConvert(loc, bcVecTy, callOp.getResult(0)); |
| res = builder.create<mlir::vector::BitCastOp>(loc, mlirTyArgs[0], res); |
| shftRes = builder.createConvert(loc, argTypes[0], res); |
| } else { |
| shftRes = callOp.getResult(0); |
| } |
| } else if (vop == VecOp::Sld || vop == VecOp::Sldw) { |
| assert(args.size() == 3); |
| auto constIntOp = mlir::dyn_cast_or_null<mlir::IntegerAttr>( |
| mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[2].getDefiningOp()) |
| .getValue()); |
| assert(constIntOp && "expected integer constant argument"); |
| |
| // Bitcast to vector<16xi8> |
| auto vi8Ty{mlir::VectorType::get(16, builder.getIntegerType(8))}; |
| if (mlirTyArgs[0] != vi8Ty) { |
| mlirVecArgs[0] = |
| builder.create<mlir::LLVM::BitcastOp>(loc, vi8Ty, mlirVecArgs[0]) |
| .getResult(); |
| mlirVecArgs[1] = |
| builder.create<mlir::LLVM::BitcastOp>(loc, vi8Ty, mlirVecArgs[1]) |
| .getResult(); |
| } |
| |
| // Construct the mask for shuffling |
| auto shiftVal{constIntOp.getInt()}; |
| if (vop == VecOp::Sldw) |
| shiftVal = shiftVal << 2; |
| shiftVal &= 0xF; |
| llvm::SmallVector<int64_t, 16> mask; |
| // Shuffle with mask based on the endianness |
| const auto triple{fir::getTargetTriple(builder.getModule())}; |
| if (triple.isLittleEndian()) { |
| for (int i = 16; i < 32; ++i) |
| mask.push_back(i - shiftVal); |
| shftRes = builder.create<mlir::vector::ShuffleOp>(loc, mlirVecArgs[1], |
| mlirVecArgs[0], mask); |
| } else { |
| for (int i = 0; i < 16; ++i) |
| mask.push_back(i + shiftVal); |
| shftRes = builder.create<mlir::vector::ShuffleOp>(loc, mlirVecArgs[0], |
| mlirVecArgs[1], mask); |
| } |
| |
| // Bitcast to the original type |
| if (shftRes.getType() != mlirTyArgs[0]) |
| shftRes = |
| builder.create<mlir::LLVM::BitcastOp>(loc, mlirTyArgs[0], shftRes); |
| |
| return builder.createConvert(loc, resultType, shftRes); |
| } else |
| llvm_unreachable("Invalid vector operation for generator"); |
| |
| return shftRes; |
| } |
| |
| // VEC_SPLAT, VEC_SPLATS, VEC_SPLAT_S32 |
| template <VecOp vop> |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecSplat(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| |
| mlir::vector::SplatOp splatOp{nullptr}; |
| mlir::Type retTy{nullptr}; |
| switch (vop) { |
| case VecOp::Splat: { |
| assert(args.size() == 2); |
| auto vecTyInfo{getVecTypeFromFir(argBases[0])}; |
| |
| auto extractOp{genVecExtract(resultType, args)}; |
| splatOp = builder.create<mlir::vector::SplatOp>( |
| loc, *(extractOp.getUnboxed()), vecTyInfo.toMlirVectorType(context)); |
| retTy = vecTyInfo.toFirVectorType(); |
| break; |
| } |
| case VecOp::Splats: { |
| assert(args.size() == 1); |
| auto vecTyInfo{getVecTypeFromEle(argBases[0])}; |
| |
| splatOp = builder.create<mlir::vector::SplatOp>( |
| loc, argBases[0], vecTyInfo.toMlirVectorType(context)); |
| retTy = vecTyInfo.toFirVectorType(); |
| break; |
| } |
| case VecOp::Splat_s32: { |
| assert(args.size() == 1); |
| auto eleTy{builder.getIntegerType(32)}; |
| auto intOp{builder.createConvert(loc, eleTy, argBases[0])}; |
| |
| // the intrinsic always returns vector(integer(4)) |
| splatOp = builder.create<mlir::vector::SplatOp>( |
| loc, intOp, mlir::VectorType::get(4, eleTy)); |
| retTy = fir::VectorType::get(4, eleTy); |
| break; |
| } |
| default: |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| return builder.createConvert(loc, retTy, splatOp); |
| } |
| |
| fir::ExtendedValue |
| PPCIntrinsicLibrary::genVecXlds(mlir::Type resultType, |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 2); |
| auto arg0{getBase(args[0])}; |
| auto arg1{getBase(args[1])}; |
| |
| // Prepare the return type in FIR. |
| auto vecTyInfo{getVecTypeFromFirType(resultType)}; |
| auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())}; |
| auto firTy{vecTyInfo.toFirVectorType()}; |
| |
| // Add the %val of arg0 to %addr of arg1 |
| auto addr{addOffsetToAddress(builder, loc, arg1, arg0)}; |
| |
| auto i64Ty{mlir::IntegerType::get(builder.getContext(), 64)}; |
| auto i64VecTy{mlir::VectorType::get(2, i64Ty)}; |
| auto i64RefTy{builder.getRefType(i64Ty)}; |
| auto addrConv{builder.create<fir::ConvertOp>(loc, i64RefTy, addr)}; |
| |
| auto addrVal{builder.create<fir::LoadOp>(loc, addrConv)}; |
| auto splatRes{builder.create<mlir::vector::SplatOp>(loc, addrVal, i64VecTy)}; |
| |
| mlir::Value result{nullptr}; |
| if (mlirTy != splatRes.getType()) { |
| result = builder.create<mlir::vector::BitCastOp>(loc, mlirTy, splatRes); |
| } else |
| result = splatRes; |
| |
| return builder.createConvert(loc, firTy, result); |
| } |
| |
| const char *getMmaIrIntrName(MMAOp mmaOp) { |
| switch (mmaOp) { |
| case MMAOp::AssembleAcc: |
| return "llvm.ppc.mma.assemble.acc"; |
| case MMAOp::AssemblePair: |
| return "llvm.ppc.vsx.assemble.pair"; |
| case MMAOp::DisassembleAcc: |
| return "llvm.ppc.mma.disassemble.acc"; |
| case MMAOp::DisassemblePair: |
| return "llvm.ppc.vsx.disassemble.pair"; |
| case MMAOp::Xxmfacc: |
| return "llvm.ppc.mma.xxmfacc"; |
| case MMAOp::Xxmtacc: |
| return "llvm.ppc.mma.xxmtacc"; |
| case MMAOp::Xxsetaccz: |
| return "llvm.ppc.mma.xxsetaccz"; |
| case MMAOp::Pmxvbf16ger2: |
| return "llvm.ppc.mma.pmxvbf16ger2"; |
| case MMAOp::Pmxvbf16ger2nn: |
| return "llvm.ppc.mma.pmxvbf16ger2nn"; |
| case MMAOp::Pmxvbf16ger2np: |
| return "llvm.ppc.mma.pmxvbf16ger2np"; |
| case MMAOp::Pmxvbf16ger2pn: |
| return "llvm.ppc.mma.pmxvbf16ger2pn"; |
| case MMAOp::Pmxvbf16ger2pp: |
| return "llvm.ppc.mma.pmxvbf16ger2pp"; |
| case MMAOp::Pmxvf16ger2: |
| return "llvm.ppc.mma.pmxvf16ger2"; |
| case MMAOp::Pmxvf16ger2nn: |
| return "llvm.ppc.mma.pmxvf16ger2nn"; |
| case MMAOp::Pmxvf16ger2np: |
| return "llvm.ppc.mma.pmxvf16ger2np"; |
| case MMAOp::Pmxvf16ger2pn: |
| return "llvm.ppc.mma.pmxvf16ger2pn"; |
| case MMAOp::Pmxvf16ger2pp: |
| return "llvm.ppc.mma.pmxvf16ger2pp"; |
| case MMAOp::Pmxvf32ger: |
| return "llvm.ppc.mma.pmxvf32ger"; |
| case MMAOp::Pmxvf32gernn: |
| return "llvm.ppc.mma.pmxvf32gernn"; |
| case MMAOp::Pmxvf32gernp: |
| return "llvm.ppc.mma.pmxvf32gernp"; |
| case MMAOp::Pmxvf32gerpn: |
| return "llvm.ppc.mma.pmxvf32gerpn"; |
| case MMAOp::Pmxvf32gerpp: |
| return "llvm.ppc.mma.pmxvf32gerpp"; |
| case MMAOp::Pmxvf64ger: |
| return "llvm.ppc.mma.pmxvf64ger"; |
| case MMAOp::Pmxvf64gernn: |
| return "llvm.ppc.mma.pmxvf64gernn"; |
| case MMAOp::Pmxvf64gernp: |
| return "llvm.ppc.mma.pmxvf64gernp"; |
| case MMAOp::Pmxvf64gerpn: |
| return "llvm.ppc.mma.pmxvf64gerpn"; |
| case MMAOp::Pmxvf64gerpp: |
| return "llvm.ppc.mma.pmxvf64gerpp"; |
| case MMAOp::Pmxvi16ger2: |
| return "llvm.ppc.mma.pmxvi16ger2"; |
| case MMAOp::Pmxvi16ger2pp: |
| return "llvm.ppc.mma.pmxvi16ger2pp"; |
| case MMAOp::Pmxvi16ger2s: |
| return "llvm.ppc.mma.pmxvi16ger2s"; |
| case MMAOp::Pmxvi16ger2spp: |
| return "llvm.ppc.mma.pmxvi16ger2spp"; |
| case MMAOp::Pmxvi4ger8: |
| return "llvm.ppc.mma.pmxvi4ger8"; |
| case MMAOp::Pmxvi4ger8pp: |
| return "llvm.ppc.mma.pmxvi4ger8pp"; |
| case MMAOp::Pmxvi8ger4: |
| return "llvm.ppc.mma.pmxvi8ger4"; |
| case MMAOp::Pmxvi8ger4pp: |
| return "llvm.ppc.mma.pmxvi8ger4pp"; |
| case MMAOp::Pmxvi8ger4spp: |
| return "llvm.ppc.mma.pmxvi8ger4spp"; |
| case MMAOp::Xvbf16ger2: |
| return "llvm.ppc.mma.xvbf16ger2"; |
| case MMAOp::Xvbf16ger2nn: |
| return "llvm.ppc.mma.xvbf16ger2nn"; |
| case MMAOp::Xvbf16ger2np: |
| return "llvm.ppc.mma.xvbf16ger2np"; |
| case MMAOp::Xvbf16ger2pn: |
| return "llvm.ppc.mma.xvbf16ger2pn"; |
| case MMAOp::Xvbf16ger2pp: |
| return "llvm.ppc.mma.xvbf16ger2pp"; |
| case MMAOp::Xvf16ger2: |
| return "llvm.ppc.mma.xvf16ger2"; |
| case MMAOp::Xvf16ger2nn: |
| return "llvm.ppc.mma.xvf16ger2nn"; |
| case MMAOp::Xvf16ger2np: |
| return "llvm.ppc.mma.xvf16ger2np"; |
| case MMAOp::Xvf16ger2pn: |
| return "llvm.ppc.mma.xvf16ger2pn"; |
| case MMAOp::Xvf16ger2pp: |
| return "llvm.ppc.mma.xvf16ger2pp"; |
| case MMAOp::Xvf32ger: |
| return "llvm.ppc.mma.xvf32ger"; |
| case MMAOp::Xvf32gernn: |
| return "llvm.ppc.mma.xvf32gernn"; |
| case MMAOp::Xvf32gernp: |
| return "llvm.ppc.mma.xvf32gernp"; |
| case MMAOp::Xvf32gerpn: |
| return "llvm.ppc.mma.xvf32gerpn"; |
| case MMAOp::Xvf32gerpp: |
| return "llvm.ppc.mma.xvf32gerpp"; |
| case MMAOp::Xvf64ger: |
| return "llvm.ppc.mma.xvf64ger"; |
| case MMAOp::Xvf64gernn: |
| return "llvm.ppc.mma.xvf64gernn"; |
| case MMAOp::Xvf64gernp: |
| return "llvm.ppc.mma.xvf64gernp"; |
| case MMAOp::Xvf64gerpn: |
| return "llvm.ppc.mma.xvf64gerpn"; |
| case MMAOp::Xvf64gerpp: |
| return "llvm.ppc.mma.xvf64gerpp"; |
| case MMAOp::Xvi16ger2: |
| return "llvm.ppc.mma.xvi16ger2"; |
| case MMAOp::Xvi16ger2pp: |
| return "llvm.ppc.mma.xvi16ger2pp"; |
| case MMAOp::Xvi16ger2s: |
| return "llvm.ppc.mma.xvi16ger2s"; |
| case MMAOp::Xvi16ger2spp: |
| return "llvm.ppc.mma.xvi16ger2spp"; |
| case MMAOp::Xvi4ger8: |
| return "llvm.ppc.mma.xvi4ger8"; |
| case MMAOp::Xvi4ger8pp: |
| return "llvm.ppc.mma.xvi4ger8pp"; |
| case MMAOp::Xvi8ger4: |
| return "llvm.ppc.mma.xvi8ger4"; |
| case MMAOp::Xvi8ger4pp: |
| return "llvm.ppc.mma.xvi8ger4pp"; |
| case MMAOp::Xvi8ger4spp: |
| return "llvm.ppc.mma.xvi8ger4spp"; |
| } |
| llvm_unreachable("getMmaIrIntrName"); |
| } |
| |
| mlir::FunctionType getMmaIrFuncType(mlir::MLIRContext *context, MMAOp mmaOp) { |
| switch (mmaOp) { |
| case MMAOp::AssembleAcc: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 4); |
| case MMAOp::AssemblePair: |
| return genMmaVpFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::DisassembleAcc: |
| return genMmaDisassembleFuncType(context, mmaOp); |
| case MMAOp::DisassemblePair: |
| return genMmaDisassembleFuncType(context, mmaOp); |
| case MMAOp::Xxmfacc: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 0); |
| case MMAOp::Xxmtacc: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 0); |
| case MMAOp::Xxsetaccz: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 0); |
| case MMAOp::Pmxvbf16ger2: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvbf16ger2nn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvbf16ger2np: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvbf16ger2pn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvbf16ger2pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvf16ger2: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvf16ger2nn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvf16ger2np: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvf16ger2pn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvf16ger2pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvf32ger: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf32gernn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf32gernp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf32gerpn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf32gerpp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf64ger: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 1, /*Vector*/ 1, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf64gernn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf64gernp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf64gerpn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvf64gerpp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1, |
| /*Integer*/ 2); |
| case MMAOp::Pmxvi16ger2: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi16ger2pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi16ger2s: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi16ger2spp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi4ger8: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi4ger8pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi8ger4: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi8ger4pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Pmxvi8ger4spp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2, |
| /*Integer*/ 3); |
| case MMAOp::Xvbf16ger2: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvbf16ger2nn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvbf16ger2np: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvbf16ger2pn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvbf16ger2pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf16ger2: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf16ger2nn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf16ger2np: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf16ger2pn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf16ger2pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf32ger: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf32gernn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf32gernp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf32gerpn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf32gerpp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvf64ger: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 1, /*Vector*/ 1); |
| case MMAOp::Xvf64gernn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1); |
| case MMAOp::Xvf64gernp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1); |
| case MMAOp::Xvf64gerpn: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1); |
| case MMAOp::Xvf64gerpp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1); |
| case MMAOp::Xvi16ger2: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi16ger2pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi16ger2s: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi16ger2spp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi4ger8: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi4ger8pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi8ger4: |
| return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi8ger4pp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| case MMAOp::Xvi8ger4spp: |
| return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2); |
| } |
| llvm_unreachable("getMmaIrFuncType"); |
| } |
| |
| template <MMAOp IntrId, MMAHandlerOp HandlerOp> |
| void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) { |
| auto context{builder.getContext()}; |
| mlir::FunctionType intrFuncType{getMmaIrFuncType(context, IntrId)}; |
| mlir::func::FuncOp funcOp{ |
| builder.createFunction(loc, getMmaIrIntrName(IntrId), intrFuncType)}; |
| llvm::SmallVector<mlir::Value> intrArgs; |
| |
| // Depending on SubToFunc, change the subroutine call to a function call. |
| // First argument represents the result. Rest of the arguments |
| // are shifted one position to form the actual argument list. |
| size_t argStart{0}; |
| size_t argStep{1}; |
| size_t e{args.size()}; |
| if (HandlerOp == MMAHandlerOp::SubToFunc) { |
| // The first argument becomes function result. Start from the second |
| // argument. |
| argStart = 1; |
| } else if (HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE) { |
| // Reverse argument order on little-endian target only. |
| // The reversal does not depend on the setting of non-native-order option. |
| const auto triple{fir::getTargetTriple(builder.getModule())}; |
| if (triple.isLittleEndian()) { |
| // Load the arguments in reverse order. |
| argStart = args.size() - 1; |
| // The first argument becomes function result. Stop at the second |
| // argument. |
| e = 0; |
| argStep = -1; |
| } else { |
| // Load the arguments in natural order. |
| // The first argument becomes function result. Start from the second |
| // argument. |
| argStart = 1; |
| } |
| } |
| |
| for (size_t i = argStart, j = 0; i != e; i += argStep, ++j) { |
| auto v{fir::getBase(args[i])}; |
| if (i == 0 && HandlerOp == MMAHandlerOp::FirstArgIsResult) { |
| // First argument is passed in as an address. We need to load |
| // the content to match the LLVM interface. |
| v = builder.create<fir::LoadOp>(loc, v); |
| } |
| auto vType{v.getType()}; |
| mlir::Type targetType{intrFuncType.getInput(j)}; |
| if (vType != targetType) { |
| if (mlir::isa<mlir::VectorType>(targetType)) { |
| // Perform vector type conversion for arguments passed by value. |
| auto eleTy{mlir::dyn_cast<fir::VectorType>(vType).getEleTy()}; |
| auto len{mlir::dyn_cast<fir::VectorType>(vType).getLen()}; |
| mlir::VectorType mlirType = mlir::VectorType::get(len, eleTy); |
| auto v0{builder.createConvert(loc, mlirType, v)}; |
| auto v1{builder.create<mlir::vector::BitCastOp>(loc, targetType, v0)}; |
| intrArgs.push_back(v1); |
| } else if (mlir::isa<mlir::IntegerType>(targetType) && |
| mlir::isa<mlir::IntegerType>(vType)) { |
| auto v0{builder.createConvert(loc, targetType, v)}; |
| intrArgs.push_back(v0); |
| } else { |
| llvm::errs() << "\nUnexpected type conversion requested: " |
| << " from " << vType << " to " << targetType << "\n"; |
| llvm_unreachable("Unsupported type conversion for argument to PowerPC " |
| "MMA intrinsic"); |
| } |
| } else { |
| intrArgs.push_back(v); |
| } |
| } |
| auto callSt{builder.create<fir::CallOp>(loc, funcOp, intrArgs)}; |
| if (HandlerOp == MMAHandlerOp::SubToFunc || |
| HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE || |
| HandlerOp == MMAHandlerOp::FirstArgIsResult) { |
| // Convert pointer type if needed. |
| mlir::Value callResult{callSt.getResult(0)}; |
| mlir::Value destPtr{fir::getBase(args[0])}; |
| mlir::Type callResultPtrType{builder.getRefType(callResult.getType())}; |
| if (destPtr.getType() != callResultPtrType) { |
| destPtr = builder.create<fir::ConvertOp>(loc, callResultPtrType, destPtr); |
| } |
| // Copy the result. |
| builder.create<fir::StoreOp>(loc, callResult, destPtr); |
| } |
| } |
| |
| // VEC_ST, VEC_STE |
| template <VecOp vop> |
| void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 3); |
| |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| auto arg1TyInfo{getVecTypeFromFir(argBases[0])}; |
| |
| auto addr{addOffsetToAddress(builder, loc, argBases[2], argBases[1])}; |
| |
| llvm::StringRef fname{}; |
| mlir::VectorType stTy{nullptr}; |
| auto i32ty{mlir::IntegerType::get(context, 32)}; |
| switch (vop) { |
| case VecOp::St: |
| stTy = mlir::VectorType::get(4, i32ty); |
| fname = "llvm.ppc.altivec.stvx"; |
| break; |
| case VecOp::Ste: { |
| const auto width{arg1TyInfo.eleTy.getIntOrFloatBitWidth()}; |
| const auto len{arg1TyInfo.len}; |
| |
| if (arg1TyInfo.isFloat32()) { |
| stTy = mlir::VectorType::get(len, i32ty); |
| fname = "llvm.ppc.altivec.stvewx"; |
| } else if (mlir::isa<mlir::IntegerType>(arg1TyInfo.eleTy)) { |
| stTy = mlir::VectorType::get(len, mlir::IntegerType::get(context, width)); |
| |
| switch (width) { |
| case 8: |
| fname = "llvm.ppc.altivec.stvebx"; |
| break; |
| case 16: |
| fname = "llvm.ppc.altivec.stvehx"; |
| break; |
| case 32: |
| fname = "llvm.ppc.altivec.stvewx"; |
| break; |
| default: |
| assert(false && "invalid element size"); |
| } |
| } else |
| assert(false && "unknown type"); |
| break; |
| } |
| case VecOp::Stxvp: |
| // __vector_pair type |
| stTy = mlir::VectorType::get(256, mlir::IntegerType::get(context, 1)); |
| fname = "llvm.ppc.vsx.stxvp"; |
| break; |
| default: |
| llvm_unreachable("invalid vector operation for generator"); |
| } |
| |
| auto funcType{ |
| mlir::FunctionType::get(context, {stTy, addr.getType()}, std::nullopt)}; |
| mlir::func::FuncOp funcOp = builder.createFunction(loc, fname, funcType); |
| |
| llvm::SmallVector<mlir::Value, 4> biArgs; |
| |
| if (vop == VecOp::Stxvp) { |
| biArgs.push_back(argBases[0]); |
| biArgs.push_back(addr); |
| builder.create<fir::CallOp>(loc, funcOp, biArgs); |
| return; |
| } |
| |
| auto vecTyInfo{getVecTypeFromFirType(argBases[0].getType())}; |
| auto cnv{builder.createConvert(loc, vecTyInfo.toMlirVectorType(context), |
| argBases[0])}; |
| |
| mlir::Value newArg1{nullptr}; |
| if (stTy != arg1TyInfo.toMlirVectorType(context)) |
| newArg1 = builder.create<mlir::vector::BitCastOp>(loc, stTy, cnv); |
| else |
| newArg1 = cnv; |
| |
| if (isBEVecElemOrderOnLE()) |
| newArg1 = builder.createConvert( |
| loc, stTy, reverseVectorElements(builder, loc, newArg1, 4)); |
| |
| biArgs.push_back(newArg1); |
| biArgs.push_back(addr); |
| |
| builder.create<fir::CallOp>(loc, funcOp, biArgs); |
| } |
| |
| // VEC_XST, VEC_XST_BE, VEC_STXV, VEC_XSTD2, VEC_XSTW4 |
| template <VecOp vop> |
| void PPCIntrinsicLibrary::genVecXStore( |
| llvm::ArrayRef<fir::ExtendedValue> args) { |
| assert(args.size() == 3); |
| auto context{builder.getContext()}; |
| auto argBases{getBasesForArgs(args)}; |
| VecTypeInfo arg1TyInfo{getVecTypeFromFir(argBases[0])}; |
| |
| auto addr{addOffsetToAddress(builder, loc, argBases[2], argBases[1])}; |
| |
| mlir::Value trg{nullptr}; |
| mlir::Value src{nullptr}; |
| |
| switch (vop) { |
| case VecOp::Xst: |
| case VecOp::Xst_be: { |
| src = argBases[0]; |
| trg = builder.createConvert(loc, builder.getRefType(argBases[0].getType()), |
| addr); |
| |
| if (vop == VecOp::Xst_be || isBEVecElemOrderOnLE()) { |
| auto cnv{builder.createConvert(loc, arg1TyInfo.toMlirVectorType(context), |
| argBases[0])}; |
| auto shf{reverseVectorElements(builder, loc, cnv, arg1TyInfo.len)}; |
| |
| src = builder.createConvert(loc, arg1TyInfo.toFirVectorType(), shf); |
| } |
| break; |
| } |
| case VecOp::Xstd2: |
| case VecOp::Xstw4: { |
| // an 16-byte vector arg1 is treated as two 8-byte elements or |
| // four 4-byte elements |
| mlir::IntegerType elemTy; |
| uint64_t numElem = (vop == VecOp::Xstd2) ? 2 : 4; |
| elemTy = builder.getIntegerType(128 / numElem); |
| |
| mlir::VectorType mlirVecTy{mlir::VectorType::get(numElem, elemTy)}; |
| fir::VectorType firVecTy{fir::VectorType::get(numElem, elemTy)}; |
| |
| auto cnv{builder.createConvert(loc, arg1TyInfo.toMlirVectorType(context), |
| argBases[0])}; |
| |
| mlir::Type srcTy{nullptr}; |
| if (numElem != arg1TyInfo.len) { |
| cnv = builder.create<mlir::vector::BitCastOp>(loc, mlirVecTy, cnv); |
| srcTy = firVecTy; |
| } else { |
| srcTy = arg1TyInfo.toFirVectorType(); |
| } |
| |
| trg = builder.createConvert(loc, builder.getRefType(srcTy), addr); |
| |
| if (isBEVecElemOrderOnLE()) { |
| cnv = reverseVectorElements(builder, loc, cnv, numElem); |
| } |
| |
| src = builder.createConvert(loc, srcTy, cnv); |
| break; |
| } |
| case VecOp::Stxv: |
| src = argBases[0]; |
| trg = builder.createConvert(loc, builder.getRefType(argBases[0].getType()), |
| addr); |
| break; |
| default: |
| assert(false && "Invalid vector operation for generator"); |
| } |
| builder.create<fir::StoreOp>(loc, mlir::TypeRange{}, |
| mlir::ValueRange{src, trg}, |
| getAlignmentAttr(builder, 1)); |
| } |
| |
| } // namespace fir |