lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp - llvm-project/llvm - Git at Google

 //===-- SystemZSelectionDAGInfo.cpp - SystemZ SelectionDAG Info -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the SystemZSelectionDAGInfo class.
 //
 //===----------------------------------------------------------------------===//

 #include "SystemZTargetMachine.h"
 #include "llvm/CodeGen/SelectionDAG.h"

 using namespace llvm;

 #define DEBUG_TYPE "systemz-selectiondag-info"

 // Decide whether it is best to use a loop or straight-line code for
 // a block operation of Size bytes with source address Src and destination
 // address Dest.  Sequence is the opcode to use for straight-line code
 // (such as MVC) and Loop is the opcode to use for loops (such as MVC_LOOP).
 // Return the chain for the completed operation.
 static SDValue emitMemMem(SelectionDAG &DAG, const SDLoc &DL, unsigned Sequence,
                           unsigned Loop, SDValue Chain, SDValue Dst,
                           SDValue Src, uint64_t Size) {
   EVT PtrVT = Src.getValueType();
   // The heuristic we use is to prefer loops for anything that would
   // require 7 or more MVCs.  With these kinds of sizes there isn't
   // much to choose between straight-line code and looping code,
   // since the time will be dominated by the MVCs themselves.
   // However, the loop has 4 or 5 instructions (depending on whether
   // the base addresses can be proved equal), so there doesn't seem
   // much point using a loop for 5 * 256 bytes or fewer.  Anything in
   // the range (5 * 256, 6 * 256) will need another instruction after
   // the loop, so it doesn't seem worth using a loop then either.
   // The next value up, 6 * 256, can be implemented in the same
   // number of straight-line MVCs as 6 * 256 - 1.
   if (Size > 6 * 256)
     return DAG.getNode(Loop, DL, MVT::Other, Chain, Dst, Src,
                        DAG.getConstant(Size, DL, PtrVT),
                        DAG.getConstant(Size / 256, DL, PtrVT));
   return DAG.getNode(Sequence, DL, MVT::Other, Chain, Dst, Src,
                      DAG.getConstant(Size, DL, PtrVT));
 }

 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
     SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   if (IsVolatile)
     return SDValue();

   if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
     return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
                       Chain, Dst, Src, CSize->getZExtValue());
   return SDValue();
 }

 // Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
 // Chain, Dst, ByteVal and Size.  These cases are expected to use
 // MVI, MVHHI, MVHI and MVGHI respectively.
 static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
                            SDValue Dst, uint64_t ByteVal, uint64_t Size,
                            unsigned Align, MachinePointerInfo DstPtrInfo) {
   uint64_t StoreVal = ByteVal;
   for (unsigned I = 1; I < Size; ++I)
     StoreVal |= ByteVal << (I * 8);
   return DAG.getStore(
       Chain, DL, DAG.getConstant(StoreVal, DL, MVT::getIntegerVT(Size * 8)),
       Dst, DstPtrInfo, Align);
 }

 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
     SDValue Byte, SDValue Size, Align Alignment, bool IsVolatile,
     MachinePointerInfo DstPtrInfo) const {
   EVT PtrVT = Dst.getValueType();

   if (IsVolatile)
     return SDValue();

   if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
     uint64_t Bytes = CSize->getZExtValue();
     if (Bytes == 0)
       return SDValue();
     if (auto *CByte = dyn_cast<ConstantSDNode>(Byte)) {
       // Handle cases that can be done using at most two of
       // MVI, MVHI, MVHHI and MVGHI.  The latter two can only be
       // used if ByteVal is all zeros or all ones; in other casees,
       // we can move at most 2 halfwords.
       uint64_t ByteVal = CByte->getZExtValue();
       if (ByteVal == 0 || ByteVal == 255 ?
           Bytes <= 16 && countPopulation(Bytes) <= 2 :
           Bytes <= 4) {
         unsigned Size1 = Bytes == 16 ? 8 : 1 << findLastSet(Bytes);
         unsigned Size2 = Bytes - Size1;
         SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1,
                                      Alignment.value(), DstPtrInfo);
         if (Size2 == 0)
           return Chain1;
         Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
                           DAG.getConstant(Size1, DL, PtrVT));
         DstPtrInfo = DstPtrInfo.getWithOffset(Size1);
         SDValue Chain2 = memsetStore(
             DAG, DL, Chain, Dst, ByteVal, Size2,
             std::min((unsigned)Alignment.value(), Size1), DstPtrInfo);
         return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
       }
     } else {
       // Handle one and two bytes using STC.
       if (Bytes <= 2) {
         SDValue Chain1 =
             DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
         if (Bytes == 1)
           return Chain1;
         SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
                                    DAG.getConstant(1, DL, PtrVT));
         SDValue Chain2 = DAG.getStore(Chain, DL, Byte, Dst2,
                                       DstPtrInfo.getWithOffset(1), Align(1));
         return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
       }
     }
     assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already");

     // Handle the special case of a memset of 0, which can use XC.
     auto *CByte = dyn_cast<ConstantSDNode>(Byte);
     if (CByte && CByte->getZExtValue() == 0)
       return emitMemMem(DAG, DL, SystemZISD::XC, SystemZISD::XC_LOOP,
                         Chain, Dst, Dst, Bytes);

     // Copy the byte to the first location and then use MVC to copy
     // it to the rest.
     Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
     SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
                                    DAG.getConstant(1, DL, PtrVT));
     return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
                       Chain, DstPlus1, Dst, Bytes - 1);
   }
   return SDValue();
 }

 // Use CLC to compare [Src1, Src1 + Size) with [Src2, Src2 + Size),
 // deciding whether to use a loop or straight-line code.
 static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
                        SDValue Src1, SDValue Src2, uint64_t Size) {
   SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
   EVT PtrVT = Src1.getValueType();
   // A two-CLC sequence is a clear win over a loop, not least because it
   // needs only one branch.  A three-CLC sequence needs the same number
   // of branches as a loop (i.e. 2), but is shorter.  That brings us to
   // lengths greater than 768 bytes.  It seems relatively likely that
   // a difference will be found within the first 768 bytes, so we just
   // optimize for the smallest number of branch instructions, in order
   // to avoid polluting the prediction buffer too much.  A loop only ever
   // needs 2 branches, whereas a straight-line sequence would need 3 or more.
   if (Size > 3 * 256)
     return DAG.getNode(SystemZISD::CLC_LOOP, DL, VTs, Chain, Src1, Src2,
                        DAG.getConstant(Size, DL, PtrVT),
                        DAG.getConstant(Size / 256, DL, PtrVT));
   return DAG.getNode(SystemZISD::CLC, DL, VTs, Chain, Src1, Src2,
                      DAG.getConstant(Size, DL, PtrVT));
 }

 // Convert the current CC value into an integer that is 0 if CC == 0,
 // greater than zero if CC == 1 and less than zero if CC >= 2.
 // The sequence starts with IPM, which puts CC into bits 29 and 28
 // of an integer and clears bits 30 and 31.
 static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
                               SelectionDAG &DAG) {
   SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
   SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, IPM,
                             DAG.getConstant(30 - SystemZ::IPM_CC, DL, MVT::i32));
   SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL,
                             DAG.getConstant(30, DL, MVT::i32));
   return SRA;
 }

 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
     SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
     MachinePointerInfo Op2PtrInfo) const {
   if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
     uint64_t Bytes = CSize->getZExtValue();
     assert(Bytes > 0 && "Caller should have handled 0-size case");
     // Swap operands to invert CC == 1 vs. CC == 2 cases.
     SDValue CCReg = emitCLC(DAG, DL, Chain, Src2, Src1, Bytes);
     Chain = CCReg.getValue(1);
     return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
   }
   return std::make_pair(SDValue(), SDValue());
 }

 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
     SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
   // Use SRST to find the character.  End is its address on success.
   EVT PtrVT = Src.getValueType();
   SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other);
   Length = DAG.getZExtOrTrunc(Length, DL, PtrVT);
   Char = DAG.getZExtOrTrunc(Char, DL, MVT::i32);
   Char = DAG.getNode(ISD::AND, DL, MVT::i32, Char,
                      DAG.getConstant(255, DL, MVT::i32));
   SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, Length);
   SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
                             Limit, Src, Char);
   SDValue CCReg = End.getValue(1);
   Chain = End.getValue(2);

   // Now select between End and null, depending on whether the character
   // was found.
   SDValue Ops[] = {
       End, DAG.getConstant(0, DL, PtrVT),
       DAG.getTargetConstant(SystemZ::CCMASK_SRST, DL, MVT::i32),
       DAG.getTargetConstant(SystemZ::CCMASK_SRST_FOUND, DL, MVT::i32), CCReg};
   End = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, PtrVT, Ops);
   return std::make_pair(End, Chain);
 }

 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
     SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo,
     bool isStpcpy) const {
   SDVTList VTs = DAG.getVTList(Dest.getValueType(), MVT::Other);
   SDValue EndDest = DAG.getNode(SystemZISD::STPCPY, DL, VTs, Chain, Dest, Src,
                                 DAG.getConstant(0, DL, MVT::i32));
   return std::make_pair(isStpcpy ? EndDest : Dest, EndDest.getValue(1));
 }

 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
     SDValue Src2, MachinePointerInfo Op1PtrInfo,
     MachinePointerInfo Op2PtrInfo) const {
   SDVTList VTs = DAG.getVTList(Src1.getValueType(), MVT::i32, MVT::Other);
   // Swap operands to invert CC == 1 vs. CC == 2 cases.
   SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src2, Src1,
                                DAG.getConstant(0, DL, MVT::i32));
   SDValue CCReg = Unused.getValue(1);
   Chain = Unused.getValue(2);
   return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
 }

 // Search from Src for a null character, stopping once Src reaches Limit.
 // Return a pair of values, the first being the number of nonnull characters
 // and the second being the out chain.
 //
 // This can be used for strlen by setting Limit to 0.
 static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG,
                                                     const SDLoc &DL,
                                                     SDValue Chain, SDValue Src,
                                                     SDValue Limit) {
   EVT PtrVT = Src.getValueType();
   SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other);
   SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
                             Limit, Src, DAG.getConstant(0, DL, MVT::i32));
   Chain = End.getValue(2);
   SDValue Len = DAG.getNode(ISD::SUB, DL, PtrVT, End, Src);
   return std::make_pair(Len, Chain);
 }

 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
     MachinePointerInfo SrcPtrInfo) const {
   EVT PtrVT = Src.getValueType();
   return getBoundedStrlen(DAG, DL, Chain, Src, DAG.getConstant(0, DL, PtrVT));
 }

 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
     SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const {
   EVT PtrVT = Src.getValueType();
   MaxLength = DAG.getZExtOrTrunc(MaxLength, DL, PtrVT);
   SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, MaxLength);
   return getBoundedStrlen(DAG, DL, Chain, Src, Limit);
 }
	//===-- SystemZSelectionDAGInfo.cpp - SystemZ SelectionDAG Info -----------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file implements the SystemZSelectionDAGInfo class.
	//
	//===----------------------------------------------------------------------===//

	#include "SystemZTargetMachine.h"
	#include "llvm/CodeGen/SelectionDAG.h"

	using namespace llvm;

	#define DEBUG_TYPE "systemz-selectiondag-info"

	// Decide whether it is best to use a loop or straight-line code for
	// a block operation of Size bytes with source address Src and destination
	// address Dest. Sequence is the opcode to use for straight-line code
	// (such as MVC) and Loop is the opcode to use for loops (such as MVC_LOOP).
	// Return the chain for the completed operation.
	static SDValue emitMemMem(SelectionDAG &DAG, const SDLoc &DL, unsigned Sequence,
	unsigned Loop, SDValue Chain, SDValue Dst,
	SDValue Src, uint64_t Size) {
	EVT PtrVT = Src.getValueType();
	// The heuristic we use is to prefer loops for anything that would
	// require 7 or more MVCs. With these kinds of sizes there isn't
	// much to choose between straight-line code and looping code,
	// since the time will be dominated by the MVCs themselves.
	// However, the loop has 4 or 5 instructions (depending on whether
	// the base addresses can be proved equal), so there doesn't seem
	// much point using a loop for 5 * 256 bytes or fewer. Anything in
	// the range (5 * 256, 6 * 256) will need another instruction after
	// the loop, so it doesn't seem worth using a loop then either.
	// The next value up, 6 * 256, can be implemented in the same
	// number of straight-line MVCs as 6 * 256 - 1.
	if (Size > 6 * 256)
	return DAG.getNode(Loop, DL, MVT::Other, Chain, Dst, Src,
	DAG.getConstant(Size, DL, PtrVT),
	DAG.getConstant(Size / 256, DL, PtrVT));
	return DAG.getNode(Sequence, DL, MVT::Other, Chain, Dst, Src,
	DAG.getConstant(Size, DL, PtrVT));
	}

	SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
	SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
	MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
	if (IsVolatile)
	return SDValue();

	if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
	return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
	Chain, Dst, Src, CSize->getZExtValue());
	return SDValue();
	}

	// Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
	// Chain, Dst, ByteVal and Size. These cases are expected to use
	// MVI, MVHHI, MVHI and MVGHI respectively.
	static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
	SDValue Dst, uint64_t ByteVal, uint64_t Size,
	unsigned Align, MachinePointerInfo DstPtrInfo) {
	uint64_t StoreVal = ByteVal;
	for (unsigned I = 1; I < Size; ++I)
	StoreVal \|= ByteVal << (I * 8);
	return DAG.getStore(
	Chain, DL, DAG.getConstant(StoreVal, DL, MVT::getIntegerVT(Size * 8)),
	Dst, DstPtrInfo, Align);
	}

	SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
	SDValue Byte, SDValue Size, Align Alignment, bool IsVolatile,
	MachinePointerInfo DstPtrInfo) const {
	EVT PtrVT = Dst.getValueType();

	if (IsVolatile)
	return SDValue();

	if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
	uint64_t Bytes = CSize->getZExtValue();
	if (Bytes == 0)
	return SDValue();
	if (auto *CByte = dyn_cast<ConstantSDNode>(Byte)) {
	// Handle cases that can be done using at most two of
	// MVI, MVHI, MVHHI and MVGHI. The latter two can only be
	// used if ByteVal is all zeros or all ones; in other casees,
	// we can move at most 2 halfwords.
	uint64_t ByteVal = CByte->getZExtValue();
	if (ByteVal == 0 \|\| ByteVal == 255 ?
	Bytes <= 16 && countPopulation(Bytes) <= 2 :
	Bytes <= 4) {
	unsigned Size1 = Bytes == 16 ? 8 : 1 << findLastSet(Bytes);
	unsigned Size2 = Bytes - Size1;
	SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1,
	Alignment.value(), DstPtrInfo);
	if (Size2 == 0)
	return Chain1;
	Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
	DAG.getConstant(Size1, DL, PtrVT));
	DstPtrInfo = DstPtrInfo.getWithOffset(Size1);
	SDValue Chain2 = memsetStore(
	DAG, DL, Chain, Dst, ByteVal, Size2,
	std::min((unsigned)Alignment.value(), Size1), DstPtrInfo);
	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
	}
	} else {
	// Handle one and two bytes using STC.
	if (Bytes <= 2) {
	SDValue Chain1 =
	DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
	if (Bytes == 1)
	return Chain1;
	SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
	DAG.getConstant(1, DL, PtrVT));
	SDValue Chain2 = DAG.getStore(Chain, DL, Byte, Dst2,
	DstPtrInfo.getWithOffset(1), Align(1));
	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
	}
	}
	assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already");

	// Handle the special case of a memset of 0, which can use XC.
	auto *CByte = dyn_cast<ConstantSDNode>(Byte);
	if (CByte && CByte->getZExtValue() == 0)
	return emitMemMem(DAG, DL, SystemZISD::XC, SystemZISD::XC_LOOP,
	Chain, Dst, Dst, Bytes);

	// Copy the byte to the first location and then use MVC to copy
	// it to the rest.
	Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
	SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
	DAG.getConstant(1, DL, PtrVT));
	return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
	Chain, DstPlus1, Dst, Bytes - 1);
	}
	return SDValue();
	}

	// Use CLC to compare [Src1, Src1 + Size) with [Src2, Src2 + Size),
	// deciding whether to use a loop or straight-line code.
	static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
	SDValue Src1, SDValue Src2, uint64_t Size) {
	SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
	EVT PtrVT = Src1.getValueType();
	// A two-CLC sequence is a clear win over a loop, not least because it
	// needs only one branch. A three-CLC sequence needs the same number
	// of branches as a loop (i.e. 2), but is shorter. That brings us to
	// lengths greater than 768 bytes. It seems relatively likely that
	// a difference will be found within the first 768 bytes, so we just
	// optimize for the smallest number of branch instructions, in order
	// to avoid polluting the prediction buffer too much. A loop only ever
	// needs 2 branches, whereas a straight-line sequence would need 3 or more.
	if (Size > 3 * 256)
	return DAG.getNode(SystemZISD::CLC_LOOP, DL, VTs, Chain, Src1, Src2,
	DAG.getConstant(Size, DL, PtrVT),
	DAG.getConstant(Size / 256, DL, PtrVT));
	return DAG.getNode(SystemZISD::CLC, DL, VTs, Chain, Src1, Src2,
	DAG.getConstant(Size, DL, PtrVT));
	}

	// Convert the current CC value into an integer that is 0 if CC == 0,
	// greater than zero if CC == 1 and less than zero if CC >= 2.
	// The sequence starts with IPM, which puts CC into bits 29 and 28
	// of an integer and clears bits 30 and 31.
	static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
	SelectionDAG &DAG) {
	SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
	SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, IPM,
	DAG.getConstant(30 - SystemZ::IPM_CC, DL, MVT::i32));
	SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL,
	DAG.getConstant(30, DL, MVT::i32));
	return SRA;
	}

	std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
	SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
	MachinePointerInfo Op2PtrInfo) const {
	if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
	uint64_t Bytes = CSize->getZExtValue();
	assert(Bytes > 0 && "Caller should have handled 0-size case");
	// Swap operands to invert CC == 1 vs. CC == 2 cases.
	SDValue CCReg = emitCLC(DAG, DL, Chain, Src2, Src1, Bytes);
	Chain = CCReg.getValue(1);
	return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
	}
	return std::make_pair(SDValue(), SDValue());
	}

	std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
	SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
	// Use SRST to find the character. End is its address on success.
	EVT PtrVT = Src.getValueType();
	SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other);
	Length = DAG.getZExtOrTrunc(Length, DL, PtrVT);
	Char = DAG.getZExtOrTrunc(Char, DL, MVT::i32);
	Char = DAG.getNode(ISD::AND, DL, MVT::i32, Char,
	DAG.getConstant(255, DL, MVT::i32));
	SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, Length);
	SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
	Limit, Src, Char);
	SDValue CCReg = End.getValue(1);
	Chain = End.getValue(2);

	// Now select between End and null, depending on whether the character
	// was found.
	SDValue Ops[] = {
	End, DAG.getConstant(0, DL, PtrVT),
	DAG.getTargetConstant(SystemZ::CCMASK_SRST, DL, MVT::i32),
	DAG.getTargetConstant(SystemZ::CCMASK_SRST_FOUND, DL, MVT::i32), CCReg};
	End = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, PtrVT, Ops);
	return std::make_pair(End, Chain);
	}

	std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
	SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo,
	bool isStpcpy) const {
	SDVTList VTs = DAG.getVTList(Dest.getValueType(), MVT::Other);
	SDValue EndDest = DAG.getNode(SystemZISD::STPCPY, DL, VTs, Chain, Dest, Src,
	DAG.getConstant(0, DL, MVT::i32));
	return std::make_pair(isStpcpy ? EndDest : Dest, EndDest.getValue(1));
	}

	std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
	SDValue Src2, MachinePointerInfo Op1PtrInfo,
	MachinePointerInfo Op2PtrInfo) const {
	SDVTList VTs = DAG.getVTList(Src1.getValueType(), MVT::i32, MVT::Other);
	// Swap operands to invert CC == 1 vs. CC == 2 cases.
	SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src2, Src1,
	DAG.getConstant(0, DL, MVT::i32));
	SDValue CCReg = Unused.getValue(1);
	Chain = Unused.getValue(2);
	return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
	}

	// Search from Src for a null character, stopping once Src reaches Limit.
	// Return a pair of values, the first being the number of nonnull characters
	// and the second being the out chain.
	//
	// This can be used for strlen by setting Limit to 0.
	static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG,
	const SDLoc &DL,
	SDValue Chain, SDValue Src,
	SDValue Limit) {
	EVT PtrVT = Src.getValueType();
	SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other);
	SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
	Limit, Src, DAG.getConstant(0, DL, MVT::i32));
	Chain = End.getValue(2);
	SDValue Len = DAG.getNode(ISD::SUB, DL, PtrVT, End, Src);
	return std::make_pair(Len, Chain);
	}

	std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
	MachinePointerInfo SrcPtrInfo) const {
	EVT PtrVT = Src.getValueType();
	return getBoundedStrlen(DAG, DL, Chain, Src, DAG.getConstant(0, DL, PtrVT));
	}

	std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen(
	SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
	SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const {
	EVT PtrVT = Src.getValueType();
	MaxLength = DAG.getZExtOrTrunc(MaxLength, DL, PtrVT);
	SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, MaxLength);
	return getBoundedStrlen(DAG, DL, Chain, Src, Limit);
	}