blob: 2db08b7297258a4fb38ccfa9879515e01cdeef5f [file] [log] [blame]
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9/// \file
10/// This transformation implements the well known scalar replacement of
11/// aggregates transformation. It tries to identify promotable elements of an
12/// aggregate alloca, and promote them to registers. It will also try to
13/// convert uses of an element (or set of elements) of an alloca into a vector
14/// or bitfield-style integer scalar if appropriate.
15///
16/// It works to do this with minimal slicing of the alloca so that regions
17/// which are merely transferred in and out of external memory remain unchanged
18/// and are not decomposed to scalar code.
19///
20/// Because this also performs alloca promotion, it can be thought of as also
21/// serving the purpose of SSA formation. The algorithm iterates on the
22/// function until all opportunities for promotion have been realized.
23///
24//===----------------------------------------------------------------------===//
25
Chandler Carruth29a18a42015-09-12 09:09:14 +000026#include "llvm/Transforms/Scalar/SROA.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000027#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/PointerIntPair.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000031#include "llvm/ADT/STLExtras.h"
Davide Italiano81a26da2017-04-27 23:09:01 +000032#include "llvm/ADT/SetVector.h"
Hiroshi Inoue48e4c7a2017-12-01 06:05:05 +000033#include "llvm/ADT/SmallBitVector.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000034#include "llvm/ADT/SmallPtrSet.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000035#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000037#include "llvm/ADT/StringRef.h"
38#include "llvm/ADT/Twine.h"
39#include "llvm/ADT/iterator.h"
40#include "llvm/ADT/iterator_range.h"
Daniel Jasperaec2fa32016-12-19 08:22:17 +000041#include "llvm/Analysis/AssumptionCache.h"
Chandler Carruth29a18a42015-09-12 09:09:14 +000042#include "llvm/Analysis/GlobalsModRef.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000043#include "llvm/Analysis/Loads.h"
Chandler Carruthe41e7b72012-12-10 08:28:39 +000044#include "llvm/Analysis/PtrUseVisitor.h"
David Blaikie2be39222018-03-21 22:34:23 +000045#include "llvm/Analysis/Utils/Local.h"
Nico Weber432a3882018-04-30 14:59:11 +000046#include "llvm/Config/llvm-config.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000047#include "llvm/IR/BasicBlock.h"
48#include "llvm/IR/Constant.h"
49#include "llvm/IR/ConstantFolder.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000050#include "llvm/IR/Constants.h"
Chandler Carruth12664a02014-03-06 00:22:06 +000051#include "llvm/IR/DIBuilder.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000052#include "llvm/IR/DataLayout.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000053#include "llvm/IR/DebugInfoMetadata.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000054#include "llvm/IR/DerivedTypes.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000055#include "llvm/IR/Dominators.h"
56#include "llvm/IR/Function.h"
57#include "llvm/IR/GetElementPtrTypeIterator.h"
58#include "llvm/IR/GlobalAlias.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000059#include "llvm/IR/IRBuilder.h"
Chandler Carruth7da14f12014-03-06 03:23:41 +000060#include "llvm/IR/InstVisitor.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000061#include "llvm/IR/InstrTypes.h"
62#include "llvm/IR/Instruction.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000063#include "llvm/IR/Instructions.h"
64#include "llvm/IR/IntrinsicInst.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000065#include "llvm/IR/Intrinsics.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000066#include "llvm/IR/LLVMContext.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000067#include "llvm/IR/Metadata.h"
68#include "llvm/IR/Module.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000069#include "llvm/IR/Operator.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000070#include "llvm/IR/PassManager.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/User.h"
74#include "llvm/IR/Value.h"
Chandler Carruth1b398ae2012-09-14 09:22:59 +000075#include "llvm/Pass.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000076#include "llvm/Support/Casting.h"
Chandler Carruth70b44c52012-09-15 11:43:14 +000077#include "llvm/Support/CommandLine.h"
Chandler Carruthf0546402013-07-18 07:15:00 +000078#include "llvm/Support/Compiler.h"
Chandler Carruth1b398ae2012-09-14 09:22:59 +000079#include "llvm/Support/Debug.h"
80#include "llvm/Support/ErrorHandling.h"
Chandler Carruth1b398ae2012-09-14 09:22:59 +000081#include "llvm/Support/MathExtras.h"
Chandler Carruth1b398ae2012-09-14 09:22:59 +000082#include "llvm/Support/raw_ostream.h"
Chandler Carruth29a18a42015-09-12 09:09:14 +000083#include "llvm/Transforms/Scalar.h"
Chandler Carruth1b398ae2012-09-14 09:22:59 +000084#include "llvm/Transforms/Utils/PromoteMemToReg.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000085#include <algorithm>
86#include <cassert>
87#include <chrono>
88#include <cstddef>
89#include <cstdint>
90#include <cstring>
91#include <iterator>
92#include <string>
93#include <tuple>
94#include <utility>
95#include <vector>
Chandler Carruth83cee772014-02-25 03:59:29 +000096
Hal Finkel29f51312016-03-28 11:13:03 +000097#ifndef NDEBUG
98// We only use this for a debug check.
Chandler Carruth83cee772014-02-25 03:59:29 +000099#include <random>
100#endif
101
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000102using namespace llvm;
Chandler Carruth29a18a42015-09-12 09:09:14 +0000103using namespace llvm::sroa;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000104
Chandler Carruth964daaa2014-04-22 02:55:47 +0000105#define DEBUG_TYPE "sroa"
106
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000107STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
Chandler Carruth5f5b6162013-03-20 06:30:46 +0000108STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
Chandler Carruth6c321c12013-07-19 10:57:36 +0000109STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
110STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
111STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
Chandler Carruth5f5b6162013-03-20 06:30:46 +0000112STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
113STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000114STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
Chandler Carruth5f5b6162013-03-20 06:30:46 +0000115STATISTIC(NumDeleted, "Number of instructions deleted");
116STATISTIC(NumVectorized, "Number of vectorized aggregates");
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000117
Chandler Carruth83cee772014-02-25 03:59:29 +0000118/// Hidden option to enable randomly shuffling the slices to help uncover
119/// instability in their order.
120static cl::opt<bool> SROARandomShuffleSlices("sroa-random-shuffle-slices",
121 cl::init(false), cl::Hidden);
122
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000123/// Hidden option to experiment with completely strict handling of inbounds
124/// GEPs.
Chandler Carruth113dc642014-12-20 02:39:18 +0000125static cl::opt<bool> SROAStrictInbounds("sroa-strict-inbounds", cl::init(false),
126 cl::Hidden);
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000127
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000128namespace {
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000129
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000130/// A custom IRBuilder inserter which prefixes all names, but only in
Mehdi Amini1e9c9252016-03-11 17:15:34 +0000131/// Assert builds.
Mehdi Aminiba9fba82016-03-13 21:05:13 +0000132class IRBuilderPrefixedInserter : public IRBuilderDefaultInserter {
Chandler Carruth34f0c7f2013-03-21 09:52:18 +0000133 std::string Prefix;
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000134
Zachary Turner41a9ee92017-10-11 23:54:34 +0000135 const Twine getNameWithPrefix(const Twine &Name) const {
136 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
137 }
138
Chandler Carruth34f0c7f2013-03-21 09:52:18 +0000139public:
140 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
141
142protected:
143 void InsertHelper(Instruction *I, const Twine &Name, BasicBlock *BB,
144 BasicBlock::iterator InsertPt) const {
Zachary Turner41a9ee92017-10-11 23:54:34 +0000145 IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name), BB,
146 InsertPt);
Chandler Carruth34f0c7f2013-03-21 09:52:18 +0000147 }
148};
149
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000150/// Provide a type for IRBuilder that drops names in release builds.
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000151using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
Chandler Carruthd177f862013-03-20 07:30:36 +0000152
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000153/// A used slice of an alloca.
Chandler Carruthf0546402013-07-18 07:15:00 +0000154///
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000155/// This structure represents a slice of an alloca used by some instruction. It
156/// stores both the begin and end offsets of this use, a pointer to the use
157/// itself, and a flag indicating whether we can classify the use as splittable
158/// or not when forming partitions of the alloca.
159class Slice {
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000160 /// The beginning offset of the range.
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000161 uint64_t BeginOffset = 0;
Chandler Carruthf74654d2013-03-18 08:36:46 +0000162
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000163 /// The ending offset, not included in the range.
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000164 uint64_t EndOffset = 0;
Chandler Carruthf74654d2013-03-18 08:36:46 +0000165
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000166 /// Storage for both the use of this slice and whether it can be
Chandler Carruthf0546402013-07-18 07:15:00 +0000167 /// split.
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000168 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
Chandler Carruthf0546402013-07-18 07:15:00 +0000169
170public:
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000171 Slice() = default;
172
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000173 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
Chandler Carruthf0546402013-07-18 07:15:00 +0000174 : BeginOffset(BeginOffset), EndOffset(EndOffset),
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000175 UseAndIsSplittable(U, IsSplittable) {}
Chandler Carruthf0546402013-07-18 07:15:00 +0000176
177 uint64_t beginOffset() const { return BeginOffset; }
178 uint64_t endOffset() const { return EndOffset; }
179
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000180 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
181 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
Chandler Carruthf0546402013-07-18 07:15:00 +0000182
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000183 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
Chandler Carruthf0546402013-07-18 07:15:00 +0000184
Craig Topperf40110f2014-04-25 05:29:35 +0000185 bool isDead() const { return getUse() == nullptr; }
186 void kill() { UseAndIsSplittable.setPointer(nullptr); }
Chandler Carruthf74654d2013-03-18 08:36:46 +0000187
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000188 /// Support for ordering ranges.
Chandler Carruthf74654d2013-03-18 08:36:46 +0000189 ///
190 /// This provides an ordering over ranges such that start offsets are
191 /// always increasing, and within equal start offsets, the end offsets are
192 /// decreasing. Thus the spanning range comes first in a cluster with the
193 /// same start position.
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000194 bool operator<(const Slice &RHS) const {
Chandler Carruth113dc642014-12-20 02:39:18 +0000195 if (beginOffset() < RHS.beginOffset())
196 return true;
197 if (beginOffset() > RHS.beginOffset())
198 return false;
199 if (isSplittable() != RHS.isSplittable())
200 return !isSplittable();
201 if (endOffset() > RHS.endOffset())
202 return true;
Chandler Carruthf74654d2013-03-18 08:36:46 +0000203 return false;
204 }
205
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000206 /// Support comparison with a single offset to allow binary searches.
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000207 friend LLVM_ATTRIBUTE_UNUSED bool operator<(const Slice &LHS,
Chandler Carruthf0546402013-07-18 07:15:00 +0000208 uint64_t RHSOffset) {
209 return LHS.beginOffset() < RHSOffset;
Chandler Carruthf74654d2013-03-18 08:36:46 +0000210 }
Chandler Carruthe3899f22013-07-15 17:36:21 +0000211 friend LLVM_ATTRIBUTE_UNUSED bool operator<(uint64_t LHSOffset,
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000212 const Slice &RHS) {
Chandler Carruthf0546402013-07-18 07:15:00 +0000213 return LHSOffset < RHS.beginOffset();
Chandler Carruthf74654d2013-03-18 08:36:46 +0000214 }
Chandler Carruthe3899f22013-07-15 17:36:21 +0000215
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000216 bool operator==(const Slice &RHS) const {
Chandler Carruthf0546402013-07-18 07:15:00 +0000217 return isSplittable() == RHS.isSplittable() &&
218 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
Chandler Carruthe3899f22013-07-15 17:36:21 +0000219 }
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000220 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
Chandler Carruthf74654d2013-03-18 08:36:46 +0000221};
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000222
Chandler Carruthf0546402013-07-18 07:15:00 +0000223} // end anonymous namespace
Chandler Carruthf74654d2013-03-18 08:36:46 +0000224
225namespace llvm {
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000226
Chandler Carruthf0546402013-07-18 07:15:00 +0000227template <typename T> struct isPodLike;
Chandler Carruth113dc642014-12-20 02:39:18 +0000228template <> struct isPodLike<Slice> { static const bool value = true; };
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000229
230} // end namespace llvm
Chandler Carruthf74654d2013-03-18 08:36:46 +0000231
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000232/// Representation of the alloca slices.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000233///
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000234/// This class represents the slices of an alloca which are formed by its
235/// various uses. If a pointer escapes, we can't fully build a representation
236/// for the slices used and we reflect that in this structure. The uses are
237/// stored, sorted by increasing beginning offset and with unsplittable slices
238/// starting at a particular offset before splittable slices.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000239class llvm::sroa::AllocaSlices {
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000240public:
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000241 /// Construct the slices of a particular alloca.
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000242 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000243
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000244 /// Test whether a pointer to the allocation escapes our analysis.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000245 ///
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000246 /// If this is true, the slices are never fully built and should be
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000247 /// ignored.
248 bool isEscaped() const { return PointerEscapingInstr; }
249
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000250 /// Support for iterating over the slices.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000251 /// @{
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000252 using iterator = SmallVectorImpl<Slice>::iterator;
253 using range = iterator_range<iterator>;
254
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000255 iterator begin() { return Slices.begin(); }
256 iterator end() { return Slices.end(); }
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000257
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000258 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
259 using const_range = iterator_range<const_iterator>;
260
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000261 const_iterator begin() const { return Slices.begin(); }
262 const_iterator end() const { return Slices.end(); }
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000263 /// @}
264
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000265 /// Erase a range of slices.
Chandler Carruth994cde82015-01-01 12:01:03 +0000266 void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); }
Chandler Carruth0715cba2015-01-01 11:54:38 +0000267
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000268 /// Insert new slices for this alloca.
Chandler Carruth0715cba2015-01-01 11:54:38 +0000269 ///
270 /// This moves the slices into the alloca's slices collection, and re-sorts
271 /// everything so that the usual ordering properties of the alloca's slices
272 /// hold.
273 void insert(ArrayRef<Slice> NewSlices) {
274 int OldSize = Slices.size();
Benjamin Kramer4f6ac162015-02-28 10:11:12 +0000275 Slices.append(NewSlices.begin(), NewSlices.end());
Chandler Carruth0715cba2015-01-01 11:54:38 +0000276 auto SliceI = Slices.begin() + OldSize;
Mandeep Singh Grang636d94d2018-04-13 19:47:57 +0000277 llvm::sort(SliceI, Slices.end());
Chandler Carruth0715cba2015-01-01 11:54:38 +0000278 std::inplace_merge(Slices.begin(), SliceI, Slices.end());
279 }
280
Chandler Carruth29a18a42015-09-12 09:09:14 +0000281 // Forward declare the iterator and range accessor for walking the
282 // partitions.
Chandler Carruthe2f66ce2014-12-22 22:46:00 +0000283 class partition_iterator;
Chandler Carruth29a18a42015-09-12 09:09:14 +0000284 iterator_range<partition_iterator> partitions();
Chandler Carruthe2f66ce2014-12-22 22:46:00 +0000285
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000286 /// Access the dead users for this alloca.
Chandler Carruth57d4cae2014-10-16 20:42:08 +0000287 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000288
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000289 /// Access the dead operands referring to this alloca.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000290 ///
291 /// These are operands which have cannot actually be used to refer to the
292 /// alloca as they are outside its range and the user doesn't correct for
293 /// that. These mostly consist of PHI node inputs and the like which we just
294 /// need to replace with undef.
Chandler Carruth57d4cae2014-10-16 20:42:08 +0000295 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000296
Aaron Ballman615eb472017-10-15 14:32:27 +0000297#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000298 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000299 void printSlice(raw_ostream &OS, const_iterator I,
300 StringRef Indent = " ") const;
Chandler Carruthf0546402013-07-18 07:15:00 +0000301 void printUse(raw_ostream &OS, const_iterator I,
302 StringRef Indent = " ") const;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000303 void print(raw_ostream &OS) const;
Alp Tokerf929e092014-01-04 22:47:48 +0000304 void dump(const_iterator I) const;
305 void dump() const;
Chandler Carruth25fb23d2012-09-14 10:18:51 +0000306#endif
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000307
308private:
309 template <typename DerivedT, typename RetT = void> class BuilderBase;
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000310 class SliceBuilder;
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000311
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000312 friend class AllocaSlices::SliceBuilder;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000313
Aaron Ballman615eb472017-10-15 14:32:27 +0000314#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000315 /// Handle to alloca instruction to simplify method interfaces.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000316 AllocaInst &AI;
Nick Lewyckyc7776f72013-08-13 22:51:58 +0000317#endif
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000318
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000319 /// The instruction responsible for this alloca not having a known set
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000320 /// of slices.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000321 ///
322 /// When an instruction (potentially) escapes the pointer to the alloca, we
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000323 /// store a pointer to that here and abort trying to form slices of the
324 /// alloca. This will be null if the alloca slices are analyzed successfully.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000325 Instruction *PointerEscapingInstr;
326
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000327 /// The slices of the alloca.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000328 ///
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000329 /// We store a vector of the slices formed by uses of the alloca here. This
330 /// vector is sorted by increasing begin offset, and then the unsplittable
331 /// slices before the splittable ones. See the Slice inner class for more
332 /// details.
333 SmallVector<Slice, 8> Slices;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000334
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000335 /// Instructions which will become dead if we rewrite the alloca.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000336 ///
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000337 /// Note that these are not separated by slice. This is because we expect an
338 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
339 /// all these instructions can simply be removed and replaced with undef as
340 /// they come from outside of the allocated space.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000341 SmallVector<Instruction *, 8> DeadUsers;
342
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000343 /// Operands which will become dead if we rewrite the alloca.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000344 ///
345 /// These are operands that in their particular use can be replaced with
346 /// undef when we rewrite the alloca. These show up in out-of-bounds inputs
347 /// to PHI nodes and the like. They aren't entirely dead (there might be
348 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
349 /// want to swap this particular input for undef to simplify the use lists of
350 /// the alloca.
351 SmallVector<Use *, 8> DeadOperands;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000352};
Chandler Carruth29a18a42015-09-12 09:09:14 +0000353
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000354/// A partition of the slices.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000355///
356/// An ephemeral representation for a range of slices which can be viewed as
357/// a partition of the alloca. This range represents a span of the alloca's
358/// memory which cannot be split, and provides access to all of the slices
359/// overlapping some part of the partition.
360///
361/// Objects of this type are produced by traversing the alloca's slices, but
362/// are only ephemeral and not persistent.
363class llvm::sroa::Partition {
364private:
365 friend class AllocaSlices;
366 friend class AllocaSlices::partition_iterator;
367
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000368 using iterator = AllocaSlices::iterator;
Chandler Carruth29a18a42015-09-12 09:09:14 +0000369
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000370 /// The beginning and ending offsets of the alloca for this
Chandler Carruth29a18a42015-09-12 09:09:14 +0000371 /// partition.
372 uint64_t BeginOffset, EndOffset;
373
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000374 /// The start and end iterators of this partition.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000375 iterator SI, SJ;
376
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000377 /// A collection of split slice tails overlapping the partition.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000378 SmallVector<Slice *, 4> SplitTails;
379
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000380 /// Raw constructor builds an empty partition starting and ending at
Chandler Carruth29a18a42015-09-12 09:09:14 +0000381 /// the given iterator.
382 Partition(iterator SI) : SI(SI), SJ(SI) {}
383
384public:
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000385 /// The start offset of this partition.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000386 ///
387 /// All of the contained slices start at or after this offset.
388 uint64_t beginOffset() const { return BeginOffset; }
389
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000390 /// The end offset of this partition.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000391 ///
392 /// All of the contained slices end at or before this offset.
393 uint64_t endOffset() const { return EndOffset; }
394
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000395 /// The size of the partition.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000396 ///
397 /// Note that this can never be zero.
398 uint64_t size() const {
399 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
400 return EndOffset - BeginOffset;
401 }
402
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000403 /// Test whether this partition contains no slices, and merely spans
Chandler Carruth29a18a42015-09-12 09:09:14 +0000404 /// a region occupied by split slices.
405 bool empty() const { return SI == SJ; }
406
407 /// \name Iterate slices that start within the partition.
408 /// These may be splittable or unsplittable. They have a begin offset >= the
409 /// partition begin offset.
410 /// @{
411 // FIXME: We should probably define a "concat_iterator" helper and use that
412 // to stitch together pointee_iterators over the split tails and the
413 // contiguous iterators of the partition. That would give a much nicer
414 // interface here. We could then additionally expose filtered iterators for
415 // split, unsplit, and unsplittable splices based on the usage patterns.
416 iterator begin() const { return SI; }
417 iterator end() const { return SJ; }
418 /// @}
419
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000420 /// Get the sequence of split slice tails.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000421 ///
422 /// These tails are of slices which start before this partition but are
423 /// split and overlap into the partition. We accumulate these while forming
424 /// partitions.
425 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
426};
427
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000428/// An iterator over partitions of the alloca's slices.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000429///
430/// This iterator implements the core algorithm for partitioning the alloca's
431/// slices. It is a forward iterator as we don't support backtracking for
432/// efficiency reasons, and re-use a single storage area to maintain the
433/// current set of split slices.
434///
435/// It is templated on the slice iterator type to use so that it can operate
436/// with either const or non-const slice iterators.
437class AllocaSlices::partition_iterator
438 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
439 Partition> {
440 friend class AllocaSlices;
441
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000442 /// Most of the state for walking the partitions is held in a class
Chandler Carruth29a18a42015-09-12 09:09:14 +0000443 /// with a nice interface for examining them.
444 Partition P;
445
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000446 /// We need to keep the end of the slices to know when to stop.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000447 AllocaSlices::iterator SE;
448
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000449 /// We also need to keep track of the maximum split end offset seen.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000450 /// FIXME: Do we really?
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000451 uint64_t MaxSplitSliceEndOffset = 0;
Chandler Carruth29a18a42015-09-12 09:09:14 +0000452
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000453 /// Sets the partition to be empty at given iterator, and sets the
Chandler Carruth29a18a42015-09-12 09:09:14 +0000454 /// end iterator.
455 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000456 : P(SI), SE(SE) {
Chandler Carruth29a18a42015-09-12 09:09:14 +0000457 // If not already at the end, advance our state to form the initial
458 // partition.
459 if (SI != SE)
460 advance();
461 }
462
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000463 /// Advance the iterator to the next partition.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000464 ///
465 /// Requires that the iterator not be at the end of the slices.
466 void advance() {
467 assert((P.SI != SE || !P.SplitTails.empty()) &&
468 "Cannot advance past the end of the slices!");
469
470 // Clear out any split uses which have ended.
471 if (!P.SplitTails.empty()) {
472 if (P.EndOffset >= MaxSplitSliceEndOffset) {
473 // If we've finished all splits, this is easy.
474 P.SplitTails.clear();
475 MaxSplitSliceEndOffset = 0;
476 } else {
477 // Remove the uses which have ended in the prior partition. This
478 // cannot change the max split slice end because we just checked that
479 // the prior partition ended prior to that max.
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000480 P.SplitTails.erase(llvm::remove_if(P.SplitTails,
481 [&](Slice *S) {
482 return S->endOffset() <=
483 P.EndOffset;
484 }),
485 P.SplitTails.end());
486 assert(llvm::any_of(P.SplitTails,
487 [&](Slice *S) {
488 return S->endOffset() == MaxSplitSliceEndOffset;
489 }) &&
Chandler Carruth29a18a42015-09-12 09:09:14 +0000490 "Could not find the current max split slice offset!");
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000491 assert(llvm::all_of(P.SplitTails,
492 [&](Slice *S) {
493 return S->endOffset() <= MaxSplitSliceEndOffset;
494 }) &&
Chandler Carruth29a18a42015-09-12 09:09:14 +0000495 "Max split slice end offset is not actually the max!");
496 }
497 }
498
499 // If P.SI is already at the end, then we've cleared the split tail and
500 // now have an end iterator.
501 if (P.SI == SE) {
502 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
503 return;
504 }
505
506 // If we had a non-empty partition previously, set up the state for
507 // subsequent partitions.
508 if (P.SI != P.SJ) {
509 // Accumulate all the splittable slices which started in the old
510 // partition into the split list.
511 for (Slice &S : P)
512 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
513 P.SplitTails.push_back(&S);
514 MaxSplitSliceEndOffset =
515 std::max(S.endOffset(), MaxSplitSliceEndOffset);
516 }
517
518 // Start from the end of the previous partition.
519 P.SI = P.SJ;
520
521 // If P.SI is now at the end, we at most have a tail of split slices.
522 if (P.SI == SE) {
523 P.BeginOffset = P.EndOffset;
524 P.EndOffset = MaxSplitSliceEndOffset;
525 return;
526 }
527
528 // If the we have split slices and the next slice is after a gap and is
529 // not splittable immediately form an empty partition for the split
530 // slices up until the next slice begins.
531 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
532 !P.SI->isSplittable()) {
533 P.BeginOffset = P.EndOffset;
534 P.EndOffset = P.SI->beginOffset();
535 return;
536 }
537 }
538
539 // OK, we need to consume new slices. Set the end offset based on the
540 // current slice, and step SJ past it. The beginning offset of the
541 // partition is the beginning offset of the next slice unless we have
542 // pre-existing split slices that are continuing, in which case we begin
543 // at the prior end offset.
544 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
545 P.EndOffset = P.SI->endOffset();
546 ++P.SJ;
547
548 // There are two strategies to form a partition based on whether the
549 // partition starts with an unsplittable slice or a splittable slice.
550 if (!P.SI->isSplittable()) {
551 // When we're forming an unsplittable region, it must always start at
552 // the first slice and will extend through its end.
553 assert(P.BeginOffset == P.SI->beginOffset());
554
555 // Form a partition including all of the overlapping slices with this
556 // unsplittable slice.
557 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
558 if (!P.SJ->isSplittable())
559 P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset());
560 ++P.SJ;
561 }
562
563 // We have a partition across a set of overlapping unsplittable
564 // partitions.
565 return;
566 }
567
568 // If we're starting with a splittable slice, then we need to form
569 // a synthetic partition spanning it and any other overlapping splittable
570 // splices.
571 assert(P.SI->isSplittable() && "Forming a splittable partition!");
572
573 // Collect all of the overlapping splittable slices.
574 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
575 P.SJ->isSplittable()) {
576 P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset());
577 ++P.SJ;
578 }
579
580 // Back upiP.EndOffset if we ended the span early when encountering an
581 // unsplittable slice. This synthesizes the early end offset of
582 // a partition spanning only splittable slices.
583 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
584 assert(!P.SJ->isSplittable());
585 P.EndOffset = P.SJ->beginOffset();
586 }
587 }
588
589public:
590 bool operator==(const partition_iterator &RHS) const {
591 assert(SE == RHS.SE &&
592 "End iterators don't match between compared partition iterators!");
593
594 // The observed positions of partitions is marked by the P.SI iterator and
595 // the emptiness of the split slices. The latter is only relevant when
596 // P.SI == SE, as the end iterator will additionally have an empty split
597 // slices list, but the prior may have the same P.SI and a tail of split
598 // slices.
599 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
600 assert(P.SJ == RHS.P.SJ &&
601 "Same set of slices formed two different sized partitions!");
602 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
603 "Same slice position with differently sized non-empty split "
604 "slice tails!");
605 return true;
606 }
607 return false;
608 }
609
610 partition_iterator &operator++() {
611 advance();
612 return *this;
613 }
614
615 Partition &operator*() { return P; }
616};
617
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000618/// A forward range over the partitions of the alloca's slices.
Chandler Carruth29a18a42015-09-12 09:09:14 +0000619///
620/// This accesses an iterator range over the partitions of the alloca's
621/// slices. It computes these partitions on the fly based on the overlapping
622/// offsets of the slices and the ability to split them. It will visit "empty"
623/// partitions to cover regions of the alloca only accessed via split
624/// slices.
625iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
626 return make_range(partition_iterator(begin(), end()),
627 partition_iterator(end(), end()));
Alexander Kornienkof00654e2015-06-23 09:49:53 +0000628}
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000629
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000630static Value *foldSelectInst(SelectInst &SI) {
631 // If the condition being selected on is a constant or the same value is
632 // being selected between, fold the select. Yes this does (rarely) happen
633 // early on.
634 if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition()))
Chandler Carruth113dc642014-12-20 02:39:18 +0000635 return SI.getOperand(1 + CI->isZero());
Jakub Staszak3c6583a2013-02-19 22:14:45 +0000636 if (SI.getOperand(1) == SI.getOperand(2))
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000637 return SI.getOperand(1);
Jakub Staszak3c6583a2013-02-19 22:14:45 +0000638
Craig Topperf40110f2014-04-25 05:29:35 +0000639 return nullptr;
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000640}
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000641
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000642/// A helper that folds a PHI node or a select.
Jingyue Wuec33fa92014-08-22 22:45:57 +0000643static Value *foldPHINodeOrSelectInst(Instruction &I) {
644 if (PHINode *PN = dyn_cast<PHINode>(&I)) {
645 // If PN merges together the same value, return that value.
646 return PN->hasConstantValue();
647 }
648 return foldSelectInst(cast<SelectInst>(I));
649}
650
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000651/// Builder for the alloca slices.
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000652///
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000653/// This class builds a set of alloca slices by recursively visiting the uses
654/// of an alloca and making a slice for each load and store at each offset.
655class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
656 friend class PtrUseVisitor<SliceBuilder>;
657 friend class InstVisitor<SliceBuilder>;
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000658
659 using Base = PtrUseVisitor<SliceBuilder>;
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000660
661 const uint64_t AllocSize;
Chandler Carruth83934062014-10-16 21:11:55 +0000662 AllocaSlices &AS;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000663
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000664 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
Chandler Carruthf0546402013-07-18 07:15:00 +0000665 SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
666
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000667 /// Set to de-duplicate dead instructions found in the use walk.
Chandler Carruthf0546402013-07-18 07:15:00 +0000668 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000669
670public:
Chandler Carruth83934062014-10-16 21:11:55 +0000671 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000672 : PtrUseVisitor<SliceBuilder>(DL),
Chandler Carruth83934062014-10-16 21:11:55 +0000673 AllocSize(DL.getTypeAllocSize(AI.getAllocatedType())), AS(AS) {}
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000674
675private:
Chandler Carruthf0546402013-07-18 07:15:00 +0000676 void markAsDead(Instruction &I) {
David Blaikie70573dc2014-11-19 07:49:26 +0000677 if (VisitedDeadInsts.insert(&I).second)
Chandler Carruth83934062014-10-16 21:11:55 +0000678 AS.DeadUsers.push_back(&I);
Chandler Carruthf0546402013-07-18 07:15:00 +0000679 }
680
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000681 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
Chandler Carruth97121172012-09-16 19:39:50 +0000682 bool IsSplittable = false) {
Chandler Carruthf02b8bf2012-12-03 10:59:55 +0000683 // Completely skip uses which have a zero size or start either before or
684 // past the end of the allocation.
Chandler Carruth6aedc102014-02-26 03:14:14 +0000685 if (Size == 0 || Offset.uge(AllocSize)) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000686 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
687 << Offset
688 << " which has zero size or starts outside of the "
689 << AllocSize << " byte alloca:\n"
690 << " alloca: " << AS.AI << "\n"
691 << " use: " << I << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +0000692 return markAsDead(I);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000693 }
694
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000695 uint64_t BeginOffset = Offset.getZExtValue();
696 uint64_t EndOffset = BeginOffset + Size;
Chandler Carruthe7a1ba52012-09-23 11:43:14 +0000697
698 // Clamp the end offset to the end of the allocation. Note that this is
699 // formulated to handle even the case where "BeginOffset + Size" overflows.
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000700 // This may appear superficially to be something we could ignore entirely,
701 // but that is not so! There may be widened loads or PHI-node uses where
702 // some instructions are dead but not others. We can't completely ignore
703 // them, and so have to record at least the information here.
Chandler Carruthe7a1ba52012-09-23 11:43:14 +0000704 assert(AllocSize >= BeginOffset); // Established above.
705 if (Size > AllocSize - BeginOffset) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000706 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
707 << Offset << " to remain within the " << AllocSize
708 << " byte alloca:\n"
709 << " alloca: " << AS.AI << "\n"
710 << " use: " << I << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000711 EndOffset = AllocSize;
712 }
713
Chandler Carruth83934062014-10-16 21:11:55 +0000714 AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable));
Chandler Carruthf0546402013-07-18 07:15:00 +0000715 }
716
717 void visitBitCastInst(BitCastInst &BC) {
718 if (BC.use_empty())
719 return markAsDead(BC);
720
721 return Base::visitBitCastInst(BC);
722 }
723
724 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
725 if (GEPI.use_empty())
726 return markAsDead(GEPI);
727
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000728 if (SROAStrictInbounds && GEPI.isInBounds()) {
729 // FIXME: This is a manually un-factored variant of the basic code inside
730 // of GEPs with checking of the inbounds invariant specified in the
731 // langref in a very strict sense. If we ever want to enable
732 // SROAStrictInbounds, this code should be factored cleanly into
733 // PtrUseVisitor, but it is easier to experiment with SROAStrictInbounds
Hal Finkel5c83a092016-03-28 11:23:21 +0000734 // by writing out the code here where we have the underlying allocation
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000735 // size readily available.
736 APInt GEPOffset = Offset;
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000737 const DataLayout &DL = GEPI.getModule()->getDataLayout();
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000738 for (gep_type_iterator GTI = gep_type_begin(GEPI),
739 GTE = gep_type_end(GEPI);
740 GTI != GTE; ++GTI) {
741 ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand());
742 if (!OpC)
743 break;
744
745 // Handle a struct index, which adds its field offset to the pointer.
Peter Collingbourneab85225b2016-12-02 02:24:42 +0000746 if (StructType *STy = GTI.getStructTypeOrNull()) {
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000747 unsigned ElementIdx = OpC->getZExtValue();
748 const StructLayout *SL = DL.getStructLayout(STy);
749 GEPOffset +=
750 APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx));
751 } else {
Chandler Carruth113dc642014-12-20 02:39:18 +0000752 // For array or vector indices, scale the index by the size of the
753 // type.
Chandler Carruth3b79b2a2014-02-25 21:24:45 +0000754 APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth());
755 GEPOffset += Index * APInt(Offset.getBitWidth(),
756 DL.getTypeAllocSize(GTI.getIndexedType()));
757 }
758
759 // If this index has computed an intermediate pointer which is not
760 // inbounds, then the result of the GEP is a poison value and we can
761 // delete it and all uses.
762 if (GEPOffset.ugt(AllocSize))
763 return markAsDead(GEPI);
764 }
765 }
766
Chandler Carruthf0546402013-07-18 07:15:00 +0000767 return Base::visitGetElementPtrInst(GEPI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000768 }
769
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000770 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000771 uint64_t Size, bool IsVolatile) {
Chandler Carruth24ac8302015-01-02 03:55:54 +0000772 // We allow splitting of non-volatile loads and stores where the type is an
773 // integer type. These may be used to implement 'memcpy' or other "transfer
774 // of bits" patterns.
775 bool IsSplittable = Ty->isIntegerTy() && !IsVolatile;
Chandler Carruth58d05562012-10-25 04:37:07 +0000776
777 insertUse(I, Offset, Size, IsSplittable);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000778 }
779
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000780 void visitLoadInst(LoadInst &LI) {
Chandler Carruth42cb9cb2012-09-18 12:57:43 +0000781 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
782 "All simple FCA loads should have been pre-split");
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000783
784 if (!IsOffsetKnown)
785 return PI.setAborted(&LI);
786
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000787 const DataLayout &DL = LI.getModule()->getDataLayout();
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000788 uint64_t Size = DL.getTypeStoreSize(LI.getType());
789 return handleLoadOrStore(LI.getType(), LI, Offset, Size, LI.isVolatile());
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000790 }
791
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000792 void visitStoreInst(StoreInst &SI) {
Chandler Carruth42cb9cb2012-09-18 12:57:43 +0000793 Value *ValOp = SI.getValueOperand();
794 if (ValOp == *U)
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000795 return PI.setEscapedAndAborted(&SI);
796 if (!IsOffsetKnown)
797 return PI.setAborted(&SI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000798
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000799 const DataLayout &DL = SI.getModule()->getDataLayout();
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000800 uint64_t Size = DL.getTypeStoreSize(ValOp->getType());
801
802 // If this memory access can be shown to *statically* extend outside the
Hiroshi Inoue0909ca12018-01-26 08:15:29 +0000803 // bounds of the allocation, it's behavior is undefined, so simply
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000804 // ignore it. Note that this is more strict than the generic clamping
805 // behavior of insertUse. We also try to handle cases which might run the
806 // risk of overflow.
807 // FIXME: We should instead consider the pointer to have escaped if this
808 // function is being instrumented for addressing bugs or race conditions.
Chandler Carruth6aedc102014-02-26 03:14:14 +0000809 if (Size > AllocSize || Offset.ugt(AllocSize - Size)) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000810 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
811 << Offset << " which extends past the end of the "
812 << AllocSize << " byte alloca:\n"
813 << " alloca: " << AS.AI << "\n"
814 << " use: " << SI << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +0000815 return markAsDead(SI);
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000816 }
817
Chandler Carruth42cb9cb2012-09-18 12:57:43 +0000818 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
819 "All simple FCA stores should have been pre-split");
Chandler Carrutha1c54bb2013-03-14 11:32:24 +0000820 handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile());
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000821 }
822
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000823 void visitMemSetInst(MemSetInst &II) {
Chandler Carruthb0de6dd2012-09-14 10:26:34 +0000824 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000825 ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000826 if ((Length && Length->getValue() == 0) ||
Chandler Carruth6aedc102014-02-26 03:14:14 +0000827 (IsOffsetKnown && Offset.uge(AllocSize)))
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000828 // Zero-length mem transfer intrinsics can be ignored entirely.
Chandler Carruthf0546402013-07-18 07:15:00 +0000829 return markAsDead(II);
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000830
831 if (!IsOffsetKnown)
832 return PI.setAborted(&II);
833
Chandler Carruth113dc642014-12-20 02:39:18 +0000834 insertUse(II, Offset, Length ? Length->getLimitedValue()
835 : AllocSize - Offset.getLimitedValue(),
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000836 (bool)Length);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000837 }
838
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000839 void visitMemTransferInst(MemTransferInst &II) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000840 ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
Chandler Carruth1bf38c62014-01-19 12:16:54 +0000841 if (Length && Length->getValue() == 0)
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000842 // Zero-length mem transfer intrinsics can be ignored entirely.
Chandler Carruthf0546402013-07-18 07:15:00 +0000843 return markAsDead(II);
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000844
Chandler Carruth1bf38c62014-01-19 12:16:54 +0000845 // Because we can visit these intrinsics twice, also check to see if the
846 // first time marked this instruction as dead. If so, skip it.
847 if (VisitedDeadInsts.count(&II))
848 return;
849
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000850 if (!IsOffsetKnown)
851 return PI.setAborted(&II);
852
Chandler Carruth1bf38c62014-01-19 12:16:54 +0000853 // This side of the transfer is completely out-of-bounds, and so we can
854 // nuke the entire transfer. However, we also need to nuke the other side
855 // if already added to our partitions.
856 // FIXME: Yet another place we really should bypass this when
857 // instrumenting for ASan.
Chandler Carruth6aedc102014-02-26 03:14:14 +0000858 if (Offset.uge(AllocSize)) {
Chandler Carruth113dc642014-12-20 02:39:18 +0000859 SmallDenseMap<Instruction *, unsigned>::iterator MTPI =
860 MemTransferSliceMap.find(&II);
Chandler Carruth1bf38c62014-01-19 12:16:54 +0000861 if (MTPI != MemTransferSliceMap.end())
Chandler Carruth83934062014-10-16 21:11:55 +0000862 AS.Slices[MTPI->second].kill();
Chandler Carruth1bf38c62014-01-19 12:16:54 +0000863 return markAsDead(II);
864 }
865
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000866 uint64_t RawOffset = Offset.getLimitedValue();
Chandler Carruth113dc642014-12-20 02:39:18 +0000867 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000868
Chandler Carruthf0546402013-07-18 07:15:00 +0000869 // Check for the special case where the same exact value is used for both
870 // source and dest.
871 if (*U == II.getRawDest() && *U == II.getRawSource()) {
872 // For non-volatile transfers this is a no-op.
873 if (!II.isVolatile())
874 return markAsDead(II);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000875
Nick Lewycky6ab9d932013-07-22 23:38:27 +0000876 return insertUse(II, Offset, Size, /*IsSplittable=*/false);
Chandler Carruthe5b7a2c2012-10-05 01:29:09 +0000877 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000878
Chandler Carruthf0546402013-07-18 07:15:00 +0000879 // If we have seen both source and destination for a mem transfer, then
880 // they both point to the same alloca.
881 bool Inserted;
882 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
Benjamin Kramerd6f1f842014-03-02 13:30:33 +0000883 std::tie(MTPI, Inserted) =
Chandler Carruth83934062014-10-16 21:11:55 +0000884 MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size()));
Chandler Carruthf0546402013-07-18 07:15:00 +0000885 unsigned PrevIdx = MTPI->second;
886 if (!Inserted) {
Chandler Carruth83934062014-10-16 21:11:55 +0000887 Slice &PrevP = AS.Slices[PrevIdx];
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000888
Chandler Carruthe5b7a2c2012-10-05 01:29:09 +0000889 // Check if the begin offsets match and this is a non-volatile transfer.
890 // In that case, we can completely elide the transfer.
Chandler Carruthf0546402013-07-18 07:15:00 +0000891 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
892 PrevP.kill();
893 return markAsDead(II);
Chandler Carruthe5b7a2c2012-10-05 01:29:09 +0000894 }
895
896 // Otherwise we have an offset transfer within the same alloca. We can't
897 // split those.
Chandler Carruthf0546402013-07-18 07:15:00 +0000898 PrevP.makeUnsplittable();
Chandler Carruthe5b7a2c2012-10-05 01:29:09 +0000899 }
900
Chandler Carruthe3899f22013-07-15 17:36:21 +0000901 // Insert the use now that we've fixed up the splittable nature.
Chandler Carruthf0546402013-07-18 07:15:00 +0000902 insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length);
Chandler Carruthe3899f22013-07-15 17:36:21 +0000903
Chandler Carruthf0546402013-07-18 07:15:00 +0000904 // Check that we ended up with a valid index in the map.
Chandler Carruth83934062014-10-16 21:11:55 +0000905 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000906 "Map index doesn't point back to a slice with this user.");
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000907 }
908
909 // Disable SRoA for any intrinsics except for lifetime invariants.
Jakub Staszak086f6cd2013-02-19 22:02:21 +0000910 // FIXME: What about debug intrinsics? This matches old behavior, but
Chandler Carruth4b40e002012-09-14 10:26:36 +0000911 // doesn't make sense.
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000912 void visitIntrinsicInst(IntrinsicInst &II) {
913 if (!IsOffsetKnown)
914 return PI.setAborted(&II);
915
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000916 if (II.getIntrinsicID() == Intrinsic::lifetime_start ||
917 II.getIntrinsicID() == Intrinsic::lifetime_end) {
918 ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0));
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000919 uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(),
920 Length->getLimitedValue());
Chandler Carruth97121172012-09-16 19:39:50 +0000921 insertUse(II, Offset, Size, true);
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000922 return;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000923 }
924
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000925 Base::visitIntrinsicInst(II);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000926 }
927
928 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
929 // We consider any PHI or select that results in a direct load or store of
Chandler Carruth9f21fe12013-07-19 09:13:58 +0000930 // the same offset to be a viable use for slicing purposes. These uses
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000931 // are considered unsplittable and the size is the maximum loaded or stored
932 // size.
933 SmallPtrSet<Instruction *, 4> Visited;
934 SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses;
935 Visited.insert(Root);
936 Uses.push_back(std::make_pair(cast<Instruction>(*U), Root));
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000937 const DataLayout &DL = Root->getModule()->getDataLayout();
Chandler Carruth8b907e82012-09-25 10:03:40 +0000938 // If there are no loads or stores, the access is dead. We mark that as
939 // a size zero access.
940 Size = 0;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000941 do {
942 Instruction *I, *UsedI;
Benjamin Kramerd6f1f842014-03-02 13:30:33 +0000943 std::tie(UsedI, I) = Uses.pop_back_val();
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000944
945 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000946 Size = std::max(Size, DL.getTypeStoreSize(LI->getType()));
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000947 continue;
948 }
949 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
950 Value *Op = SI->getOperand(0);
951 if (Op == UsedI)
952 return SI;
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000953 Size = std::max(Size, DL.getTypeStoreSize(Op->getType()));
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000954 continue;
955 }
956
957 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {
958 if (!GEP->hasAllZeroIndices())
959 return GEP;
960 } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) &&
961 !isa<SelectInst>(I)) {
962 return I;
963 }
964
Chandler Carruthcdf47882014-03-09 03:16:01 +0000965 for (User *U : I->users())
David Blaikie70573dc2014-11-19 07:49:26 +0000966 if (Visited.insert(cast<Instruction>(U)).second)
Chandler Carruthcdf47882014-03-09 03:16:01 +0000967 Uses.push_back(std::make_pair(I, cast<Instruction>(U)));
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000968 } while (!Uses.empty());
969
Craig Topperf40110f2014-04-25 05:29:35 +0000970 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000971 }
972
Jingyue Wuec33fa92014-08-22 22:45:57 +0000973 void visitPHINodeOrSelectInst(Instruction &I) {
974 assert(isa<PHINode>(I) || isa<SelectInst>(I));
975 if (I.use_empty())
976 return markAsDead(I);
Chandler Carruthe41e7b72012-12-10 08:28:39 +0000977
Jingyue Wuec33fa92014-08-22 22:45:57 +0000978 // TODO: We could use SimplifyInstruction here to fold PHINodes and
979 // SelectInsts. However, doing so requires to change the current
980 // dead-operand-tracking mechanism. For instance, suppose neither loading
981 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
982 // trap either. However, if we simply replace %U with undef using the
983 // current dead-operand-tracking mechanism, "load (select undef, undef,
984 // %other)" may trap because the select may return the first operand
985 // "undef".
986 if (Value *Result = foldPHINodeOrSelectInst(I)) {
Nick Lewyckyc7776f72013-08-13 22:51:58 +0000987 if (Result == *U)
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000988 // If the result of the constant fold will be the pointer, recurse
Jingyue Wuec33fa92014-08-22 22:45:57 +0000989 // through the PHI/select as if we had RAUW'ed it.
990 enqueueUsers(I);
Nick Lewyckyc7776f72013-08-13 22:51:58 +0000991 else
Jingyue Wuec33fa92014-08-22 22:45:57 +0000992 // Otherwise the operand to the PHI/select is dead, and we can replace
993 // it with undef.
Chandler Carruth83934062014-10-16 21:11:55 +0000994 AS.DeadOperands.push_back(U);
Chandler Carruth1b398ae2012-09-14 09:22:59 +0000995
996 return;
997 }
Jingyue Wuec33fa92014-08-22 22:45:57 +0000998
Chandler Carruthf0546402013-07-18 07:15:00 +0000999 if (!IsOffsetKnown)
Jingyue Wuec33fa92014-08-22 22:45:57 +00001000 return PI.setAborted(&I);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001001
Chandler Carruthf0546402013-07-18 07:15:00 +00001002 // See if we already have computed info on this node.
Jingyue Wuec33fa92014-08-22 22:45:57 +00001003 uint64_t &Size = PHIOrSelectSizes[&I];
1004 if (!Size) {
1005 // This is a new PHI/Select, check for an unsafe use of it.
1006 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size))
Chandler Carruthf0546402013-07-18 07:15:00 +00001007 return PI.setAborted(UnsafeI);
1008 }
1009
1010 // For PHI and select operands outside the alloca, we can't nuke the entire
1011 // phi or select -- the other side might still be relevant, so we special
1012 // case them here and use a separate structure to track the operands
1013 // themselves which should be replaced with undef.
1014 // FIXME: This should instead be escaped in the event we're instrumenting
1015 // for address sanitization.
Chandler Carruth6aedc102014-02-26 03:14:14 +00001016 if (Offset.uge(AllocSize)) {
Chandler Carruth83934062014-10-16 21:11:55 +00001017 AS.DeadOperands.push_back(U);
Chandler Carruthf0546402013-07-18 07:15:00 +00001018 return;
1019 }
1020
Jingyue Wuec33fa92014-08-22 22:45:57 +00001021 insertUse(I, Offset, Size);
1022 }
1023
Chandler Carruth113dc642014-12-20 02:39:18 +00001024 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); }
Jingyue Wuec33fa92014-08-22 22:45:57 +00001025
Chandler Carruth113dc642014-12-20 02:39:18 +00001026 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001027
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001028 /// Disable SROA entirely if there are unhandled users of the alloca.
Chandler Carruth113dc642014-12-20 02:39:18 +00001029 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001030};
1031
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001032AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
Nick Lewyckyc7776f72013-08-13 22:51:58 +00001033 :
Aaron Ballman615eb472017-10-15 14:32:27 +00001034#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Nick Lewyckyc7776f72013-08-13 22:51:58 +00001035 AI(AI),
1036#endif
Craig Topperf40110f2014-04-25 05:29:35 +00001037 PointerEscapingInstr(nullptr) {
Nick Lewyckyc7776f72013-08-13 22:51:58 +00001038 SliceBuilder PB(DL, AI, *this);
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001039 SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI);
Chandler Carruthe41e7b72012-12-10 08:28:39 +00001040 if (PtrI.isEscaped() || PtrI.isAborted()) {
1041 // FIXME: We should sink the escape vs. abort info into the caller nicely,
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001042 // possibly by just storing the PtrInfo in the AllocaSlices.
Chandler Carruthe41e7b72012-12-10 08:28:39 +00001043 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1044 : PtrI.getAbortingInst();
1045 assert(PointerEscapingInstr && "Did not track a bad instruction");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001046 return;
Chandler Carruthe41e7b72012-12-10 08:28:39 +00001047 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001048
Eugene Zelenko75075ef2017-09-01 21:37:29 +00001049 Slices.erase(
1050 llvm::remove_if(Slices, [](const Slice &S) { return S.isDead(); }),
1051 Slices.end());
Benjamin Kramer08e50702013-07-20 08:38:34 +00001052
Hal Finkel29f51312016-03-28 11:13:03 +00001053#ifndef NDEBUG
Chandler Carruth83cee772014-02-25 03:59:29 +00001054 if (SROARandomShuffleSlices) {
Pavel Labathc207bec2016-11-09 12:07:12 +00001055 std::mt19937 MT(static_cast<unsigned>(
1056 std::chrono::system_clock::now().time_since_epoch().count()));
Chandler Carruth83cee772014-02-25 03:59:29 +00001057 std::shuffle(Slices.begin(), Slices.end(), MT);
1058 }
1059#endif
1060
Chandler Carruthe5b7a2c2012-10-05 01:29:09 +00001061 // Sort the uses. This arranges for the offsets to be in ascending order,
1062 // and the sizes to be in descending order.
Mandeep Singh Grang636d94d2018-04-13 19:47:57 +00001063 llvm::sort(Slices.begin(), Slices.end());
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001064}
1065
Aaron Ballman615eb472017-10-15 14:32:27 +00001066#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Chandler Carruth25fb23d2012-09-14 10:18:51 +00001067
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001068void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1069 StringRef Indent) const {
1070 printSlice(OS, I, Indent);
Chandler Carruth0715cba2015-01-01 11:54:38 +00001071 OS << "\n";
Chandler Carruthf0546402013-07-18 07:15:00 +00001072 printUse(OS, I, Indent);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001073}
1074
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001075void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1076 StringRef Indent) const {
Chandler Carruthf0546402013-07-18 07:15:00 +00001077 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001078 << " slice #" << (I - begin())
Chandler Carruth0715cba2015-01-01 11:54:38 +00001079 << (I->isSplittable() ? " (splittable)" : "");
Chandler Carruthf0546402013-07-18 07:15:00 +00001080}
1081
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001082void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1083 StringRef Indent) const {
Chandler Carruthf0546402013-07-18 07:15:00 +00001084 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001085}
1086
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001087void AllocaSlices::print(raw_ostream &OS) const {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001088 if (PointerEscapingInstr) {
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001089 OS << "Can't analyze slices for alloca: " << AI << "\n"
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001090 << " A pointer to this alloca escaped by:\n"
1091 << " " << *PointerEscapingInstr << "\n";
1092 return;
1093 }
1094
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001095 OS << "Slices of alloca: " << AI << "\n";
Chandler Carruthf0546402013-07-18 07:15:00 +00001096 for (const_iterator I = begin(), E = end(); I != E; ++I)
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001097 print(OS, I);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001098}
1099
Alp Tokerf929e092014-01-04 22:47:48 +00001100LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1101 print(dbgs(), I);
1102}
1103LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001104
Aaron Ballman615eb472017-10-15 14:32:27 +00001105#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Chandler Carruth25fb23d2012-09-14 10:18:51 +00001106
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001107/// Walk the range of a partitioning looking for a common type to cover this
1108/// sequence of slices.
1109static Type *findCommonType(AllocaSlices::const_iterator B,
1110 AllocaSlices::const_iterator E,
Chandler Carruthf0546402013-07-18 07:15:00 +00001111 uint64_t EndOffset) {
Craig Topperf40110f2014-04-25 05:29:35 +00001112 Type *Ty = nullptr;
Chandler Carruth4de31542014-01-21 23:16:05 +00001113 bool TyIsCommon = true;
Craig Topperf40110f2014-04-25 05:29:35 +00001114 IntegerType *ITy = nullptr;
Chandler Carruth4de31542014-01-21 23:16:05 +00001115
1116 // Note that we need to look at *every* alloca slice's Use to ensure we
1117 // always get consistent results regardless of the order of slices.
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001118 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001119 Use *U = I->getUse();
1120 if (isa<IntrinsicInst>(*U->getUser()))
1121 continue;
1122 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1123 continue;
Chandler Carruth90c4a3a2012-10-05 01:29:06 +00001124
Craig Topperf40110f2014-04-25 05:29:35 +00001125 Type *UserTy = nullptr;
Chandler Carrutha1262002013-11-19 09:03:18 +00001126 if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001127 UserTy = LI->getType();
Chandler Carrutha1262002013-11-19 09:03:18 +00001128 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001129 UserTy = SI->getValueOperand()->getType();
Chandler Carrutha1262002013-11-19 09:03:18 +00001130 }
Chandler Carruth90c4a3a2012-10-05 01:29:06 +00001131
Chandler Carruth4de31542014-01-21 23:16:05 +00001132 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001133 // If the type is larger than the partition, skip it. We only encounter
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001134 // this for split integer operations where we want to use the type of the
Chandler Carrutha1262002013-11-19 09:03:18 +00001135 // entity causing the split. Also skip if the type is not a byte width
1136 // multiple.
Chandler Carruth4de31542014-01-21 23:16:05 +00001137 if (UserITy->getBitWidth() % 8 != 0 ||
1138 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
Chandler Carruthf0546402013-07-18 07:15:00 +00001139 continue;
Chandler Carruth90c4a3a2012-10-05 01:29:06 +00001140
Chandler Carruth4de31542014-01-21 23:16:05 +00001141 // Track the largest bitwidth integer type used in this way in case there
1142 // is no common type.
1143 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1144 ITy = UserITy;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001145 }
Duncan P. N. Exon Smith73686d32014-06-17 00:19:35 +00001146
1147 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1148 // depend on types skipped above.
1149 if (!UserTy || (Ty && Ty != UserTy))
1150 TyIsCommon = false; // Give up on anything but an iN type.
1151 else
1152 Ty = UserTy;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001153 }
Chandler Carruth4de31542014-01-21 23:16:05 +00001154
1155 return TyIsCommon ? Ty : ITy;
Chandler Carruthf0546402013-07-18 07:15:00 +00001156}
Chandler Carruthe3899f22013-07-15 17:36:21 +00001157
Chandler Carruthf0546402013-07-18 07:15:00 +00001158/// PHI instructions that use an alloca and are subsequently loaded can be
1159/// rewritten to load both input pointers in the pred blocks and then PHI the
1160/// results, allowing the load of the alloca to be promoted.
1161/// From this:
1162/// %P2 = phi [i32* %Alloca, i32* %Other]
1163/// %V = load i32* %P2
1164/// to:
1165/// %V1 = load i32* %Alloca -> will be mem2reg'd
1166/// ...
1167/// %V2 = load i32* %Other
1168/// ...
1169/// %V = phi [i32 %V1, i32 %V2]
1170///
1171/// We can do this to a select if its only uses are loads and if the operands
1172/// to the select can be loaded unconditionally.
1173///
1174/// FIXME: This should be hoisted into a generic utility, likely in
1175/// Transforms/Util/Local.h
Mehdi Aminia28d91d2015-03-10 02:37:25 +00001176static bool isSafePHIToSpeculate(PHINode &PN) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001177 // For now, we can only do this promotion if the load is in the same block
1178 // as the PHI, and if there are no stores between the phi and load.
1179 // TODO: Allow recursive phi users.
1180 // TODO: Allow stores.
1181 BasicBlock *BB = PN.getParent();
1182 unsigned MaxAlign = 0;
1183 bool HaveLoad = false;
Chandler Carruthcdf47882014-03-09 03:16:01 +00001184 for (User *U : PN.users()) {
1185 LoadInst *LI = dyn_cast<LoadInst>(U);
Craig Topperf40110f2014-04-25 05:29:35 +00001186 if (!LI || !LI->isSimple())
Chandler Carruthe74ff4c2013-07-15 10:30:19 +00001187 return false;
Chandler Carruthe74ff4c2013-07-15 10:30:19 +00001188
Chandler Carruthf0546402013-07-18 07:15:00 +00001189 // For now we only allow loads in the same block as the PHI. This is
1190 // a common case that happens when instcombine merges two loads through
1191 // a PHI.
1192 if (LI->getParent() != BB)
1193 return false;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001194
Chandler Carruthf0546402013-07-18 07:15:00 +00001195 // Ensure that there are no instructions between the PHI and the load that
1196 // could store.
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00001197 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
Chandler Carruthf0546402013-07-18 07:15:00 +00001198 if (BBI->mayWriteToMemory())
Chandler Carruthe3899f22013-07-15 17:36:21 +00001199 return false;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001200
Chandler Carruthf0546402013-07-18 07:15:00 +00001201 MaxAlign = std::max(MaxAlign, LI->getAlignment());
1202 HaveLoad = true;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001203 }
1204
Chandler Carruthf0546402013-07-18 07:15:00 +00001205 if (!HaveLoad)
1206 return false;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001207
Artur Pilipenko9bb6bea2016-04-27 11:00:48 +00001208 const DataLayout &DL = PN.getModule()->getDataLayout();
1209
Chandler Carruthf0546402013-07-18 07:15:00 +00001210 // We can only transform this if it is safe to push the loads into the
1211 // predecessor blocks. The only thing to watch out for is that we can't put
1212 // a possibly trapping load in the predecessor if it is a critical edge.
1213 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1214 TerminatorInst *TI = PN.getIncomingBlock(Idx)->getTerminator();
1215 Value *InVal = PN.getIncomingValue(Idx);
Chandler Carruthe3899f22013-07-15 17:36:21 +00001216
Chandler Carruthf0546402013-07-18 07:15:00 +00001217 // If the value is produced by the terminator of the predecessor (an
1218 // invoke) or it has side-effects, there is no valid place to put a load
1219 // in the predecessor.
1220 if (TI == InVal || TI->mayHaveSideEffects())
1221 return false;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001222
Chandler Carruthf0546402013-07-18 07:15:00 +00001223 // If the predecessor has a single successor, then the edge isn't
1224 // critical.
1225 if (TI->getNumSuccessors() == 1)
1226 continue;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001227
Chandler Carruthf0546402013-07-18 07:15:00 +00001228 // If this pointer is always safe to load, or if we can prove that there
1229 // is already a load in the block, then we can move the load to the pred
1230 // block.
Artur Pilipenko9bb6bea2016-04-27 11:00:48 +00001231 if (isSafeToLoadUnconditionally(InVal, MaxAlign, DL, TI))
Chandler Carruthf0546402013-07-18 07:15:00 +00001232 continue;
1233
1234 return false;
1235 }
1236
1237 return true;
1238}
1239
1240static void speculatePHINodeLoads(PHINode &PN) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00001241 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00001242
1243 Type *LoadTy = cast<PointerType>(PN.getType())->getElementType();
1244 IRBuilderTy PHIBuilder(&PN);
1245 PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(),
1246 PN.getName() + ".sroa.speculated");
1247
Hal Finkelcc39b672014-07-24 12:16:19 +00001248 // Get the AA tags and alignment to use from one of the loads. It doesn't
Chandler Carruthf0546402013-07-18 07:15:00 +00001249 // matter which one we get and if any differ.
Chandler Carruthcdf47882014-03-09 03:16:01 +00001250 LoadInst *SomeLoad = cast<LoadInst>(PN.user_back());
Hal Finkelcc39b672014-07-24 12:16:19 +00001251
1252 AAMDNodes AATags;
1253 SomeLoad->getAAMetadata(AATags);
Chandler Carruthf0546402013-07-18 07:15:00 +00001254 unsigned Align = SomeLoad->getAlignment();
1255
1256 // Rewrite all loads of the PN to use the new PHI.
1257 while (!PN.use_empty()) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00001258 LoadInst *LI = cast<LoadInst>(PN.user_back());
Chandler Carruthf0546402013-07-18 07:15:00 +00001259 LI->replaceAllUsesWith(NewPN);
1260 LI->eraseFromParent();
1261 }
1262
1263 // Inject loads into all of the pred blocks.
1264 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1265 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1266 TerminatorInst *TI = Pred->getTerminator();
1267 Value *InVal = PN.getIncomingValue(Idx);
1268 IRBuilderTy PredBuilder(TI);
1269
1270 LoadInst *Load = PredBuilder.CreateLoad(
1271 InVal, (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1272 ++NumLoadsSpeculated;
1273 Load->setAlignment(Align);
Hal Finkelcc39b672014-07-24 12:16:19 +00001274 if (AATags)
1275 Load->setAAMetadata(AATags);
Chandler Carruthf0546402013-07-18 07:15:00 +00001276 NewPN->addIncoming(Load, Pred);
1277 }
1278
Nicola Zaghend34e60c2018-05-14 12:53:11 +00001279 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00001280 PN.eraseFromParent();
1281}
1282
1283/// Select instructions that use an alloca and are subsequently loaded can be
1284/// rewritten to load both input pointers and then select between the result,
1285/// allowing the load of the alloca to be promoted.
1286/// From this:
1287/// %P2 = select i1 %cond, i32* %Alloca, i32* %Other
1288/// %V = load i32* %P2
1289/// to:
1290/// %V1 = load i32* %Alloca -> will be mem2reg'd
1291/// %V2 = load i32* %Other
1292/// %V = select i1 %cond, i32 %V1, i32 %V2
1293///
1294/// We can do this to a select if its only uses are loads and if the operand
1295/// to the select can be loaded unconditionally.
Mehdi Aminia28d91d2015-03-10 02:37:25 +00001296static bool isSafeSelectToSpeculate(SelectInst &SI) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001297 Value *TValue = SI.getTrueValue();
1298 Value *FValue = SI.getFalseValue();
Artur Pilipenko9bb6bea2016-04-27 11:00:48 +00001299 const DataLayout &DL = SI.getModule()->getDataLayout();
Chandler Carruthf0546402013-07-18 07:15:00 +00001300
Chandler Carruthcdf47882014-03-09 03:16:01 +00001301 for (User *U : SI.users()) {
1302 LoadInst *LI = dyn_cast<LoadInst>(U);
Craig Topperf40110f2014-04-25 05:29:35 +00001303 if (!LI || !LI->isSimple())
Chandler Carruthf0546402013-07-18 07:15:00 +00001304 return false;
1305
Hiroshi Inoueb3008242017-06-24 15:43:33 +00001306 // Both operands to the select need to be dereferenceable, either
Chandler Carruthf0546402013-07-18 07:15:00 +00001307 // absolutely (e.g. allocas) or at this point because we can see other
1308 // accesses to it.
Artur Pilipenko9bb6bea2016-04-27 11:00:48 +00001309 if (!isSafeToLoadUnconditionally(TValue, LI->getAlignment(), DL, LI))
Chandler Carruthf0546402013-07-18 07:15:00 +00001310 return false;
Artur Pilipenko9bb6bea2016-04-27 11:00:48 +00001311 if (!isSafeToLoadUnconditionally(FValue, LI->getAlignment(), DL, LI))
Chandler Carruthf0546402013-07-18 07:15:00 +00001312 return false;
1313 }
1314
1315 return true;
1316}
1317
1318static void speculateSelectInstLoads(SelectInst &SI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00001319 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00001320
1321 IRBuilderTy IRB(&SI);
1322 Value *TV = SI.getTrueValue();
1323 Value *FV = SI.getFalseValue();
1324 // Replace the loads of the select with a select of two loads.
1325 while (!SI.use_empty()) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00001326 LoadInst *LI = cast<LoadInst>(SI.user_back());
Chandler Carruthf0546402013-07-18 07:15:00 +00001327 assert(LI->isSimple() && "We only speculate simple loads");
1328
1329 IRB.SetInsertPoint(LI);
1330 LoadInst *TL =
Chandler Carruthe3899f22013-07-15 17:36:21 +00001331 IRB.CreateLoad(TV, LI->getName() + ".sroa.speculate.load.true");
Chandler Carruthf0546402013-07-18 07:15:00 +00001332 LoadInst *FL =
Chandler Carruthe3899f22013-07-15 17:36:21 +00001333 IRB.CreateLoad(FV, LI->getName() + ".sroa.speculate.load.false");
Chandler Carruthf0546402013-07-18 07:15:00 +00001334 NumLoadsSpeculated += 2;
Chandler Carruthe3899f22013-07-15 17:36:21 +00001335
Hal Finkelcc39b672014-07-24 12:16:19 +00001336 // Transfer alignment and AA info if present.
Chandler Carruthf0546402013-07-18 07:15:00 +00001337 TL->setAlignment(LI->getAlignment());
1338 FL->setAlignment(LI->getAlignment());
Hal Finkelcc39b672014-07-24 12:16:19 +00001339
1340 AAMDNodes Tags;
1341 LI->getAAMetadata(Tags);
1342 if (Tags) {
1343 TL->setAAMetadata(Tags);
1344 FL->setAAMetadata(Tags);
Chandler Carruthe3899f22013-07-15 17:36:21 +00001345 }
Chandler Carruthf0546402013-07-18 07:15:00 +00001346
1347 Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL,
1348 LI->getName() + ".sroa.speculated");
1349
Nicola Zaghend34e60c2018-05-14 12:53:11 +00001350 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00001351 LI->replaceAllUsesWith(V);
1352 LI->eraseFromParent();
Chandler Carruthe3899f22013-07-15 17:36:21 +00001353 }
Chandler Carruthf0546402013-07-18 07:15:00 +00001354 SI.eraseFromParent();
Chandler Carruth90c4a3a2012-10-05 01:29:06 +00001355}
1356
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001357/// Build a GEP out of a base pointer and indices.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001358///
1359/// This will return the BasePtr if that is valid, or build a new GEP
1360/// instruction using the IRBuilder if GEP-ing is needed.
Chandler Carruthd177f862013-03-20 07:30:36 +00001361static Value *buildGEP(IRBuilderTy &IRB, Value *BasePtr,
Zachary Turner41a9ee92017-10-11 23:54:34 +00001362 SmallVectorImpl<Value *> &Indices, Twine NamePrefix) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001363 if (Indices.empty())
1364 return BasePtr;
1365
1366 // A single zero index is a no-op, so check for this and avoid building a GEP
1367 // in that case.
1368 if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero())
1369 return BasePtr;
1370
David Blaikieaa41cd52015-04-03 21:33:42 +00001371 return IRB.CreateInBoundsGEP(nullptr, BasePtr, Indices,
1372 NamePrefix + "sroa_idx");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001373}
1374
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001375/// Get a natural GEP off of the BasePtr walking through Ty toward
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001376/// TargetTy without changing the offset of the pointer.
1377///
1378/// This routine assumes we've already established a properly offset GEP with
1379/// Indices, and arrived at the Ty type. The goal is to continue to GEP with
1380/// zero-indices down through type layers until we find one the same as
1381/// TargetTy. If we can't find one with the same type, we at least try to use
1382/// one with the same size. If none of that works, we just produce the GEP as
1383/// indicated by Indices to have the correct offset.
Chandler Carruth90a735d2013-07-19 07:21:28 +00001384static Value *getNaturalGEPWithType(IRBuilderTy &IRB, const DataLayout &DL,
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001385 Value *BasePtr, Type *Ty, Type *TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001386 SmallVectorImpl<Value *> &Indices,
Zachary Turner41a9ee92017-10-11 23:54:34 +00001387 Twine NamePrefix) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001388 if (Ty == TargetTy)
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001389 return buildGEP(IRB, BasePtr, Indices, NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001390
Chandler Carruthdfb2efd2014-02-26 10:08:16 +00001391 // Pointer size to use for the indices.
1392 unsigned PtrSize = DL.getPointerTypeSizeInBits(BasePtr->getType());
1393
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001394 // See if we can descend into a struct and locate a field with the correct
1395 // type.
1396 unsigned NumLayers = 0;
1397 Type *ElementTy = Ty;
1398 do {
1399 if (ElementTy->isPointerTy())
1400 break;
Chandler Carruthdfb2efd2014-02-26 10:08:16 +00001401
1402 if (ArrayType *ArrayTy = dyn_cast<ArrayType>(ElementTy)) {
1403 ElementTy = ArrayTy->getElementType();
1404 Indices.push_back(IRB.getIntN(PtrSize, 0));
1405 } else if (VectorType *VectorTy = dyn_cast<VectorType>(ElementTy)) {
1406 ElementTy = VectorTy->getElementType();
1407 Indices.push_back(IRB.getInt32(0));
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001408 } else if (StructType *STy = dyn_cast<StructType>(ElementTy)) {
Chandler Carruth503eb2b2012-10-09 01:58:35 +00001409 if (STy->element_begin() == STy->element_end())
1410 break; // Nothing left to descend into.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001411 ElementTy = *STy->element_begin();
1412 Indices.push_back(IRB.getInt32(0));
1413 } else {
1414 break;
1415 }
1416 ++NumLayers;
1417 } while (ElementTy != TargetTy);
1418 if (ElementTy != TargetTy)
1419 Indices.erase(Indices.end() - NumLayers, Indices.end());
1420
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001421 return buildGEP(IRB, BasePtr, Indices, NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001422}
1423
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001424/// Recursively compute indices for a natural GEP.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001425///
1426/// This is the recursive step for getNaturalGEPWithOffset that walks down the
1427/// element types adding appropriate indices for the GEP.
Chandler Carruth90a735d2013-07-19 07:21:28 +00001428static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL,
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001429 Value *Ptr, Type *Ty, APInt &Offset,
1430 Type *TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001431 SmallVectorImpl<Value *> &Indices,
Zachary Turner41a9ee92017-10-11 23:54:34 +00001432 Twine NamePrefix) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001433 if (Offset == 0)
Chandler Carruth113dc642014-12-20 02:39:18 +00001434 return getNaturalGEPWithType(IRB, DL, Ptr, Ty, TargetTy, Indices,
1435 NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001436
1437 // We can't recurse through pointer types.
1438 if (Ty->isPointerTy())
Craig Topperf40110f2014-04-25 05:29:35 +00001439 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001440
Chandler Carruthdd3cea82012-09-14 10:30:40 +00001441 // We try to analyze GEPs over vectors here, but note that these GEPs are
1442 // extremely poorly defined currently. The long-term goal is to remove GEPing
1443 // over a vector from the IR completely.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001444 if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) {
Chandler Carruth90a735d2013-07-19 07:21:28 +00001445 unsigned ElementSizeInBits = DL.getTypeSizeInBits(VecTy->getScalarType());
Craig Topperf40110f2014-04-25 05:29:35 +00001446 if (ElementSizeInBits % 8 != 0) {
1447 // GEPs over non-multiple of 8 size vector elements are invalid.
1448 return nullptr;
1449 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001450 APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8);
Chandler Carruth6fab42a2012-10-17 09:23:48 +00001451 APInt NumSkippedElements = Offset.sdiv(ElementSize);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001452 if (NumSkippedElements.ugt(VecTy->getNumElements()))
Craig Topperf40110f2014-04-25 05:29:35 +00001453 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001454 Offset -= NumSkippedElements * ElementSize;
1455 Indices.push_back(IRB.getInt(NumSkippedElements));
Chandler Carruth90a735d2013-07-19 07:21:28 +00001456 return getNaturalGEPRecursively(IRB, DL, Ptr, VecTy->getElementType(),
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001457 Offset, TargetTy, Indices, NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001458 }
1459
1460 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
1461 Type *ElementTy = ArrTy->getElementType();
Chandler Carruth90a735d2013-07-19 07:21:28 +00001462 APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy));
Chandler Carruth6fab42a2012-10-17 09:23:48 +00001463 APInt NumSkippedElements = Offset.sdiv(ElementSize);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001464 if (NumSkippedElements.ugt(ArrTy->getNumElements()))
Craig Topperf40110f2014-04-25 05:29:35 +00001465 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001466
1467 Offset -= NumSkippedElements * ElementSize;
1468 Indices.push_back(IRB.getInt(NumSkippedElements));
Chandler Carruth90a735d2013-07-19 07:21:28 +00001469 return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001470 Indices, NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001471 }
1472
1473 StructType *STy = dyn_cast<StructType>(Ty);
1474 if (!STy)
Craig Topperf40110f2014-04-25 05:29:35 +00001475 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001476
Chandler Carruth90a735d2013-07-19 07:21:28 +00001477 const StructLayout *SL = DL.getStructLayout(STy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001478 uint64_t StructOffset = Offset.getZExtValue();
Chandler Carruthcabd96c2012-09-14 10:30:42 +00001479 if (StructOffset >= SL->getSizeInBytes())
Craig Topperf40110f2014-04-25 05:29:35 +00001480 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001481 unsigned Index = SL->getElementContainingOffset(StructOffset);
1482 Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index));
1483 Type *ElementTy = STy->getElementType(Index);
Chandler Carruth90a735d2013-07-19 07:21:28 +00001484 if (Offset.uge(DL.getTypeAllocSize(ElementTy)))
Craig Topperf40110f2014-04-25 05:29:35 +00001485 return nullptr; // The offset points into alignment padding.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001486
1487 Indices.push_back(IRB.getInt32(Index));
Chandler Carruth90a735d2013-07-19 07:21:28 +00001488 return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001489 Indices, NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001490}
1491
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001492/// Get a natural GEP from a base pointer to a particular offset and
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001493/// resulting in a particular type.
1494///
1495/// The goal is to produce a "natural" looking GEP that works with the existing
1496/// composite types to arrive at the appropriate offset and element type for
1497/// a pointer. TargetTy is the element type the returned GEP should point-to if
1498/// possible. We recurse by decreasing Offset, adding the appropriate index to
1499/// Indices, and setting Ty to the result subtype.
1500///
Chandler Carruth93a21e72012-09-14 10:18:49 +00001501/// If no natural GEP can be constructed, this function returns null.
Chandler Carruth90a735d2013-07-19 07:21:28 +00001502static Value *getNaturalGEPWithOffset(IRBuilderTy &IRB, const DataLayout &DL,
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001503 Value *Ptr, APInt Offset, Type *TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001504 SmallVectorImpl<Value *> &Indices,
Zachary Turner41a9ee92017-10-11 23:54:34 +00001505 Twine NamePrefix) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001506 PointerType *Ty = cast<PointerType>(Ptr->getType());
1507
1508 // Don't consider any GEPs through an i8* as natural unless the TargetTy is
1509 // an i8.
Chandler Carruth286d87e2014-02-26 08:25:02 +00001510 if (Ty == IRB.getInt8PtrTy(Ty->getAddressSpace()) && TargetTy->isIntegerTy(8))
Craig Topperf40110f2014-04-25 05:29:35 +00001511 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001512
1513 Type *ElementTy = Ty->getElementType();
Chandler Carruth3f882d42012-09-18 22:37:19 +00001514 if (!ElementTy->isSized())
Craig Topperf40110f2014-04-25 05:29:35 +00001515 return nullptr; // We can't GEP through an unsized element.
Chandler Carruth90a735d2013-07-19 07:21:28 +00001516 APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy));
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001517 if (ElementSize == 0)
Craig Topperf40110f2014-04-25 05:29:35 +00001518 return nullptr; // Zero-length arrays can't help us build a natural GEP.
Chandler Carruth6fab42a2012-10-17 09:23:48 +00001519 APInt NumSkippedElements = Offset.sdiv(ElementSize);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001520
1521 Offset -= NumSkippedElements * ElementSize;
1522 Indices.push_back(IRB.getInt(NumSkippedElements));
Chandler Carruth90a735d2013-07-19 07:21:28 +00001523 return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001524 Indices, NamePrefix);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001525}
1526
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001527/// Compute an adjusted pointer from Ptr by Offset bytes where the
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001528/// resulting pointer has PointerTy.
1529///
1530/// This tries very hard to compute a "natural" GEP which arrives at the offset
1531/// and produces the pointer type desired. Where it cannot, it will try to use
1532/// the natural GEP to arrive at the offset and bitcast to the type. Where that
1533/// fails, it will try to use an existing i8* and GEP to the byte offset and
1534/// bitcast to the type.
1535///
1536/// The strategy for finding the more natural GEPs is to peel off layers of the
1537/// pointer, walking back through bit casts and GEPs, searching for a base
1538/// pointer from which we can compute a natural GEP with the desired
Jakub Staszak086f6cd2013-02-19 22:02:21 +00001539/// properties. The algorithm tries to fold as many constant indices into
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001540/// a single GEP as possible, thus making each GEP more independent of the
1541/// surrounding code.
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001542static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
Zachary Turner41a9ee92017-10-11 23:54:34 +00001543 APInt Offset, Type *PointerTy, Twine NamePrefix) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001544 // Even though we don't look through PHI nodes, we could be called on an
1545 // instruction in an unreachable block, which may be on a cycle.
1546 SmallPtrSet<Value *, 4> Visited;
1547 Visited.insert(Ptr);
1548 SmallVector<Value *, 4> Indices;
1549
1550 // We may end up computing an offset pointer that has the wrong type. If we
1551 // never are able to compute one directly that has the correct type, we'll
Chandler Carruth5986b542015-01-02 02:47:38 +00001552 // fall back to it, so keep it and the base it was computed from around here.
Craig Topperf40110f2014-04-25 05:29:35 +00001553 Value *OffsetPtr = nullptr;
Chandler Carruth5986b542015-01-02 02:47:38 +00001554 Value *OffsetBasePtr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001555
1556 // Remember any i8 pointer we come across to re-use if we need to do a raw
1557 // byte offset.
Craig Topperf40110f2014-04-25 05:29:35 +00001558 Value *Int8Ptr = nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001559 APInt Int8PtrOffset(Offset.getBitWidth(), 0);
1560
1561 Type *TargetTy = PointerTy->getPointerElementType();
1562
1563 do {
1564 // First fold any existing GEPs into the offset.
1565 while (GEPOperator *GEP = dyn_cast<GEPOperator>(Ptr)) {
1566 APInt GEPOffset(Offset.getBitWidth(), 0);
Chandler Carruth90a735d2013-07-19 07:21:28 +00001567 if (!GEP->accumulateConstantOffset(DL, GEPOffset))
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001568 break;
1569 Offset += GEPOffset;
1570 Ptr = GEP->getPointerOperand();
David Blaikie70573dc2014-11-19 07:49:26 +00001571 if (!Visited.insert(Ptr).second)
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001572 break;
1573 }
1574
1575 // See if we can perform a natural GEP here.
1576 Indices.clear();
Chandler Carruth90a735d2013-07-19 07:21:28 +00001577 if (Value *P = getNaturalGEPWithOffset(IRB, DL, Ptr, Offset, TargetTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001578 Indices, NamePrefix)) {
Chandler Carruth5986b542015-01-02 02:47:38 +00001579 // If we have a new natural pointer at the offset, clear out any old
1580 // offset pointer we computed. Unless it is the base pointer or
1581 // a non-instruction, we built a GEP we don't need. Zap it.
1582 if (OffsetPtr && OffsetPtr != OffsetBasePtr)
1583 if (Instruction *I = dyn_cast<Instruction>(OffsetPtr)) {
1584 assert(I->use_empty() && "Built a GEP with uses some how!");
1585 I->eraseFromParent();
1586 }
1587 OffsetPtr = P;
1588 OffsetBasePtr = Ptr;
1589 // If we also found a pointer of the right type, we're done.
1590 if (P->getType() == PointerTy)
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001591 return P;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001592 }
1593
1594 // Stash this pointer if we've found an i8*.
1595 if (Ptr->getType()->isIntegerTy(8)) {
1596 Int8Ptr = Ptr;
1597 Int8PtrOffset = Offset;
1598 }
1599
1600 // Peel off a layer of the pointer and update the offset appropriately.
1601 if (Operator::getOpcode(Ptr) == Instruction::BitCast) {
1602 Ptr = cast<Operator>(Ptr)->getOperand(0);
1603 } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(Ptr)) {
Sanjoy Das5ce32722016-04-08 00:48:30 +00001604 if (GA->isInterposable())
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001605 break;
1606 Ptr = GA->getAliasee();
1607 } else {
1608 break;
1609 }
1610 assert(Ptr->getType()->isPointerTy() && "Unexpected operand type!");
David Blaikie70573dc2014-11-19 07:49:26 +00001611 } while (Visited.insert(Ptr).second);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001612
1613 if (!OffsetPtr) {
1614 if (!Int8Ptr) {
Chandler Carruth286d87e2014-02-26 08:25:02 +00001615 Int8Ptr = IRB.CreateBitCast(
1616 Ptr, IRB.getInt8PtrTy(PointerTy->getPointerAddressSpace()),
1617 NamePrefix + "sroa_raw_cast");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001618 Int8PtrOffset = Offset;
1619 }
1620
Chandler Carruth113dc642014-12-20 02:39:18 +00001621 OffsetPtr = Int8PtrOffset == 0
1622 ? Int8Ptr
David Blaikieaa41cd52015-04-03 21:33:42 +00001623 : IRB.CreateInBoundsGEP(IRB.getInt8Ty(), Int8Ptr,
1624 IRB.getInt(Int8PtrOffset),
Chandler Carruth113dc642014-12-20 02:39:18 +00001625 NamePrefix + "sroa_raw_idx");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001626 }
1627 Ptr = OffsetPtr;
1628
1629 // On the off chance we were targeting i8*, guard the bitcast here.
1630 if (Ptr->getType() != PointerTy)
Chandler Carruthcb93cd22014-02-25 11:19:56 +00001631 Ptr = IRB.CreateBitCast(Ptr, PointerTy, NamePrefix + "sroa_cast");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001632
1633 return Ptr;
1634}
1635
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001636/// Compute the adjusted alignment for a load or store from an offset.
Chandler Carruth0715cba2015-01-01 11:54:38 +00001637static unsigned getAdjustedAlignment(Instruction *I, uint64_t Offset,
1638 const DataLayout &DL) {
1639 unsigned Alignment;
1640 Type *Ty;
1641 if (auto *LI = dyn_cast<LoadInst>(I)) {
1642 Alignment = LI->getAlignment();
1643 Ty = LI->getType();
1644 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
1645 Alignment = SI->getAlignment();
1646 Ty = SI->getValueOperand()->getType();
1647 } else {
1648 llvm_unreachable("Only loads and stores are allowed!");
1649 }
1650
1651 if (!Alignment)
1652 Alignment = DL.getABITypeAlignment(Ty);
1653
1654 return MinAlign(Alignment, Offset);
1655}
1656
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001657/// Test whether we can convert a value from the old to the new type.
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001658///
1659/// This predicate should be used to guard calls to convertValue in order to
1660/// ensure that we only try to convert viable values. The strategy is that we
1661/// will peel off single element struct and array wrappings to get to an
1662/// underlying value, and convert that value.
1663static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
1664 if (OldTy == NewTy)
1665 return true;
Chandler Carruthccffdaf2015-07-22 03:32:42 +00001666
1667 // For integer types, we can't handle any bit-width differences. This would
1668 // break both vector conversions with extension and introduce endianness
1669 // issues when in conjunction with loads and stores.
1670 if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) {
1671 assert(cast<IntegerType>(OldTy)->getBitWidth() !=
1672 cast<IntegerType>(NewTy)->getBitWidth() &&
1673 "We can't have the same bitwidth for different int types");
1674 return false;
1675 }
1676
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001677 if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy))
1678 return false;
1679 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
1680 return false;
1681
Benjamin Kramer56262592013-09-22 11:24:58 +00001682 // We can convert pointers to integers and vice-versa. Same for vectors
Benjamin Kramer90901a32013-09-21 20:36:04 +00001683 // of pointers and integers.
1684 OldTy = OldTy->getScalarType();
1685 NewTy = NewTy->getScalarType();
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001686 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
Jack Liuf101c0f2016-05-03 19:30:48 +00001687 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
1688 return cast<PointerType>(NewTy)->getPointerAddressSpace() ==
1689 cast<PointerType>(OldTy)->getPointerAddressSpace();
1690 }
Sanjoy Dasb70ddd82017-06-17 20:28:13 +00001691
1692 // We can convert integers to integral pointers, but not to non-integral
1693 // pointers.
1694 if (OldTy->isIntegerTy())
1695 return !DL.isNonIntegralPointerType(NewTy);
1696
1697 // We can convert integral pointers to integers, but non-integral pointers
1698 // need to remain pointers.
1699 if (!DL.isNonIntegralPointerType(OldTy))
1700 return NewTy->isIntegerTy();
1701
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001702 return false;
1703 }
1704
1705 return true;
1706}
1707
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001708/// Generic routine to convert an SSA value to a value of a different
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001709/// type.
1710///
1711/// This will try various different casting techniques, such as bitcasts,
1712/// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test
1713/// two types for viability with this routine.
Chandler Carruthd177f862013-03-20 07:30:36 +00001714static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
Benjamin Kramer90901a32013-09-21 20:36:04 +00001715 Type *NewTy) {
1716 Type *OldTy = V->getType();
1717 assert(canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type");
1718
1719 if (OldTy == NewTy)
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001720 return V;
Benjamin Kramer90901a32013-09-21 20:36:04 +00001721
Chandler Carruthccffdaf2015-07-22 03:32:42 +00001722 assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
1723 "Integer types must be the exact same to convert.");
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001724
Benjamin Kramer90901a32013-09-21 20:36:04 +00001725 // See if we need inttoptr for this type pair. A cast involving both scalars
1726 // and vectors requires and additional bitcast.
Craig Topper95d23472017-07-09 07:04:00 +00001727 if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
Benjamin Kramer90901a32013-09-21 20:36:04 +00001728 // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
1729 if (OldTy->isVectorTy() && !NewTy->isVectorTy())
1730 return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)),
1731 NewTy);
1732
1733 // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
1734 if (!OldTy->isVectorTy() && NewTy->isVectorTy())
1735 return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)),
1736 NewTy);
1737
1738 return IRB.CreateIntToPtr(V, NewTy);
1739 }
1740
1741 // See if we need ptrtoint for this type pair. A cast involving both scalars
1742 // and vectors requires and additional bitcast.
Craig Topper95d23472017-07-09 07:04:00 +00001743 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) {
Benjamin Kramer90901a32013-09-21 20:36:04 +00001744 // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
1745 if (OldTy->isVectorTy() && !NewTy->isVectorTy())
1746 return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
1747 NewTy);
1748
1749 // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
1750 if (!OldTy->isVectorTy() && NewTy->isVectorTy())
1751 return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
1752 NewTy);
1753
1754 return IRB.CreatePtrToInt(V, NewTy);
1755 }
1756
1757 return IRB.CreateBitCast(V, NewTy);
Chandler Carruthaa6afbb2012-10-15 08:40:22 +00001758}
1759
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001760/// Test whether the given slice use can be promoted to a vector.
Chandler Carruthf0546402013-07-18 07:15:00 +00001761///
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00001762/// This function is called to test each entry in a partition which is slated
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001763/// for a single slice.
Chandler Carruth29a18a42015-09-12 09:09:14 +00001764static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
1765 VectorType *Ty,
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001766 uint64_t ElementSize,
1767 const DataLayout &DL) {
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001768 // First validate the slice offsets.
Chandler Carruthf0546402013-07-18 07:15:00 +00001769 uint64_t BeginOffset =
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001770 std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset();
Chandler Carruthf0546402013-07-18 07:15:00 +00001771 uint64_t BeginIndex = BeginOffset / ElementSize;
1772 if (BeginIndex * ElementSize != BeginOffset ||
1773 BeginIndex >= Ty->getNumElements())
1774 return false;
1775 uint64_t EndOffset =
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001776 std::min(S.endOffset(), P.endOffset()) - P.beginOffset();
Chandler Carruthf0546402013-07-18 07:15:00 +00001777 uint64_t EndIndex = EndOffset / ElementSize;
1778 if (EndIndex * ElementSize != EndOffset || EndIndex > Ty->getNumElements())
1779 return false;
1780
1781 assert(EndIndex > BeginIndex && "Empty vector!");
1782 uint64_t NumElements = EndIndex - BeginIndex;
Chandler Carruthc659df92014-10-16 20:24:07 +00001783 Type *SliceTy = (NumElements == 1)
1784 ? Ty->getElementType()
1785 : VectorType::get(Ty->getElementType(), NumElements);
Chandler Carruthf0546402013-07-18 07:15:00 +00001786
1787 Type *SplitIntTy =
1788 Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8);
1789
Chandler Carruthc659df92014-10-16 20:24:07 +00001790 Use *U = S.getUse();
Chandler Carruthf0546402013-07-18 07:15:00 +00001791
1792 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) {
1793 if (MI->isVolatile())
1794 return false;
Chandler Carruthc659df92014-10-16 20:24:07 +00001795 if (!S.isSplittable())
Chandler Carruthf0546402013-07-18 07:15:00 +00001796 return false; // Skip any unsplittable intrinsics.
Owen Anderson6c19ab12014-08-07 21:07:35 +00001797 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
1798 if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
1799 II->getIntrinsicID() != Intrinsic::lifetime_end)
1800 return false;
Chandler Carruthf0546402013-07-18 07:15:00 +00001801 } else if (U->get()->getType()->getPointerElementType()->isStructTy()) {
1802 // Disable vector promotion when there are loads or stores of an FCA.
1803 return false;
1804 } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
1805 if (LI->isVolatile())
1806 return false;
1807 Type *LTy = LI->getType();
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001808 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001809 assert(LTy->isIntegerTy());
1810 LTy = SplitIntTy;
1811 }
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001812 if (!canConvertValue(DL, SliceTy, LTy))
Chandler Carruthf0546402013-07-18 07:15:00 +00001813 return false;
1814 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
1815 if (SI->isVolatile())
1816 return false;
1817 Type *STy = SI->getValueOperand()->getType();
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001818 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001819 assert(STy->isIntegerTy());
1820 STy = SplitIntTy;
1821 }
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001822 if (!canConvertValue(DL, STy, SliceTy))
Chandler Carruthf0546402013-07-18 07:15:00 +00001823 return false;
Chandler Carruth1ed848d2013-07-19 10:57:32 +00001824 } else {
1825 return false;
Chandler Carruthf0546402013-07-18 07:15:00 +00001826 }
1827
1828 return true;
1829}
1830
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001831/// Test whether the given alloca partitioning and range of slices can be
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001832/// promoted to a vector.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001833///
1834/// This is a quick test to check whether we can rewrite a particular alloca
1835/// partition (and its newly formed alloca) into a vector alloca with only
1836/// whole-vector loads and stores such that it could be promoted to a vector
1837/// SSA value. We only can ensure this for a limited set of operations, and we
1838/// don't want to do the rewrites unless we are confident that the result will
1839/// be promotable, so we have an early test here.
Chandler Carruth29a18a42015-09-12 09:09:14 +00001840static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
Chandler Carruth2dc96822014-10-18 00:44:02 +00001841 // Collect the candidate types for vector-based promotion. Also track whether
1842 // we have different element types.
1843 SmallVector<VectorType *, 4> CandidateTys;
1844 Type *CommonEltTy = nullptr;
1845 bool HaveCommonEltTy = true;
1846 auto CheckCandidateType = [&](Type *Ty) {
1847 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1848 CandidateTys.push_back(VTy);
1849 if (!CommonEltTy)
1850 CommonEltTy = VTy->getElementType();
1851 else if (CommonEltTy != VTy->getElementType())
1852 HaveCommonEltTy = false;
1853 }
1854 };
Chandler Carruth2dc96822014-10-18 00:44:02 +00001855 // Consider any loads or stores that are the exact size of the slice.
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001856 for (const Slice &S : P)
1857 if (S.beginOffset() == P.beginOffset() &&
1858 S.endOffset() == P.endOffset()) {
Chandler Carruth2dc96822014-10-18 00:44:02 +00001859 if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser()))
1860 CheckCandidateType(LI->getType());
1861 else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser()))
1862 CheckCandidateType(SI->getValueOperand()->getType());
1863 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001864
Chandler Carruth2dc96822014-10-18 00:44:02 +00001865 // If we didn't find a vector type, nothing to do here.
1866 if (CandidateTys.empty())
1867 return nullptr;
Chandler Carruthf0546402013-07-18 07:15:00 +00001868
Chandler Carruth2dc96822014-10-18 00:44:02 +00001869 // Remove non-integer vector types if we had multiple common element types.
1870 // FIXME: It'd be nice to replace them with integer vector types, but we can't
1871 // do that until all the backends are known to produce good code for all
1872 // integer vector types.
1873 if (!HaveCommonEltTy) {
Eugene Zelenko75075ef2017-09-01 21:37:29 +00001874 CandidateTys.erase(
1875 llvm::remove_if(CandidateTys,
1876 [](VectorType *VTy) {
1877 return !VTy->getElementType()->isIntegerTy();
1878 }),
1879 CandidateTys.end());
Chandler Carruth2dc96822014-10-18 00:44:02 +00001880
1881 // If there were no integer vector types, give up.
1882 if (CandidateTys.empty())
1883 return nullptr;
1884
1885 // Rank the remaining candidate vector types. This is easy because we know
1886 // they're all integer vectors. We sort by ascending number of elements.
1887 auto RankVectorTypes = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
David L. Jones41cecba2017-01-13 21:02:41 +00001888 (void)DL;
Chandler Carruth2dc96822014-10-18 00:44:02 +00001889 assert(DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) &&
1890 "Cannot have vector types of different sizes!");
1891 assert(RHSTy->getElementType()->isIntegerTy() &&
1892 "All non-integer types eliminated!");
1893 assert(LHSTy->getElementType()->isIntegerTy() &&
1894 "All non-integer types eliminated!");
1895 return RHSTy->getNumElements() < LHSTy->getNumElements();
1896 };
Mandeep Singh Grang636d94d2018-04-13 19:47:57 +00001897 llvm::sort(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes);
Chandler Carruth2dc96822014-10-18 00:44:02 +00001898 CandidateTys.erase(
1899 std::unique(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes),
1900 CandidateTys.end());
1901 } else {
1902// The only way to have the same element type in every vector type is to
1903// have the same vector type. Check that and remove all but one.
1904#ifndef NDEBUG
1905 for (VectorType *VTy : CandidateTys) {
1906 assert(VTy->getElementType() == CommonEltTy &&
1907 "Unaccounted for element type!");
1908 assert(VTy == CandidateTys[0] &&
1909 "Different vector types with the same element type!");
1910 }
1911#endif
1912 CandidateTys.resize(1);
1913 }
1914
1915 // Try each vector type, and return the one which works.
1916 auto CheckVectorTypeForPromotion = [&](VectorType *VTy) {
1917 uint64_t ElementSize = DL.getTypeSizeInBits(VTy->getElementType());
1918
1919 // While the definition of LLVM vectors is bitpacked, we don't support sizes
1920 // that aren't byte sized.
1921 if (ElementSize % 8)
1922 return false;
1923 assert((DL.getTypeSizeInBits(VTy) % 8) == 0 &&
1924 "vector size not a multiple of element size?");
1925 ElementSize /= 8;
1926
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001927 for (const Slice &S : P)
1928 if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL))
Chandler Carruth2dc96822014-10-18 00:44:02 +00001929 return false;
1930
Chandler Carruthffb7ce52014-12-24 01:48:09 +00001931 for (const Slice *S : P.splitSliceTails())
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001932 if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL))
Chandler Carruth2dc96822014-10-18 00:44:02 +00001933 return false;
1934
1935 return true;
1936 };
1937 for (VectorType *VTy : CandidateTys)
1938 if (CheckVectorTypeForPromotion(VTy))
1939 return VTy;
1940
1941 return nullptr;
Chandler Carruthf0546402013-07-18 07:15:00 +00001942}
1943
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001944/// Test whether a slice of an alloca is valid for integer widening.
Chandler Carruthf0546402013-07-18 07:15:00 +00001945///
1946/// This implements the necessary checking for the \c isIntegerWideningViable
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001947/// test below on a single slice of the alloca.
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001948static bool isIntegerWideningViableForSlice(const Slice &S,
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001949 uint64_t AllocBeginOffset,
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001950 Type *AllocaTy,
1951 const DataLayout &DL,
Chandler Carruth9f21fe12013-07-19 09:13:58 +00001952 bool &WholeAllocaOp) {
Chandler Carruth5031bbe2014-12-24 01:05:14 +00001953 uint64_t Size = DL.getTypeStoreSize(AllocaTy);
1954
Chandler Carruthc659df92014-10-16 20:24:07 +00001955 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
1956 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
Chandler Carruthf0546402013-07-18 07:15:00 +00001957
1958 // We can't reasonably handle cases where the load or store extends past
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00001959 // the end of the alloca's type and into its padding.
Chandler Carruthf0546402013-07-18 07:15:00 +00001960 if (RelEnd > Size)
1961 return false;
1962
Chandler Carruthc659df92014-10-16 20:24:07 +00001963 Use *U = S.getUse();
Chandler Carruthf0546402013-07-18 07:15:00 +00001964
1965 if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
1966 if (LI->isVolatile())
1967 return false;
Chandler Carruthccffdaf2015-07-22 03:32:42 +00001968 // We can't handle loads that extend past the allocated memory.
1969 if (DL.getTypeStoreSize(LI->getType()) > Size)
1970 return false;
Chandler Carruth2dc96822014-10-18 00:44:02 +00001971 // Note that we don't count vector loads or stores as whole-alloca
1972 // operations which enable integer widening because we would prefer to use
1973 // vector widening instead.
1974 if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size)
Chandler Carruthf0546402013-07-18 07:15:00 +00001975 WholeAllocaOp = true;
1976 if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) {
Chandler Carruth90a735d2013-07-19 07:21:28 +00001977 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy))
Chandler Carruthe3899f22013-07-15 17:36:21 +00001978 return false;
Chandler Carruthf0546402013-07-18 07:15:00 +00001979 } else if (RelBegin != 0 || RelEnd != Size ||
Chandler Carruth90a735d2013-07-19 07:21:28 +00001980 !canConvertValue(DL, AllocaTy, LI->getType())) {
Chandler Carruthf0546402013-07-18 07:15:00 +00001981 // Non-integer loads need to be convertible from the alloca type so that
1982 // they are promotable.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00001983 return false;
1984 }
Chandler Carruthf0546402013-07-18 07:15:00 +00001985 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
1986 Type *ValueTy = SI->getValueOperand()->getType();
1987 if (SI->isVolatile())
1988 return false;
Chandler Carruthccffdaf2015-07-22 03:32:42 +00001989 // We can't handle stores that extend past the allocated memory.
1990 if (DL.getTypeStoreSize(ValueTy) > Size)
1991 return false;
Chandler Carruth2dc96822014-10-18 00:44:02 +00001992 // Note that we don't count vector loads or stores as whole-alloca
1993 // operations which enable integer widening because we would prefer to use
1994 // vector widening instead.
1995 if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size)
Chandler Carruthf0546402013-07-18 07:15:00 +00001996 WholeAllocaOp = true;
1997 if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) {
Chandler Carruth90a735d2013-07-19 07:21:28 +00001998 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy))
Chandler Carruthf0546402013-07-18 07:15:00 +00001999 return false;
2000 } else if (RelBegin != 0 || RelEnd != Size ||
Chandler Carruth90a735d2013-07-19 07:21:28 +00002001 !canConvertValue(DL, ValueTy, AllocaTy)) {
Chandler Carruthf0546402013-07-18 07:15:00 +00002002 // Non-integer stores need to be convertible to the alloca type so that
2003 // they are promotable.
2004 return false;
2005 }
2006 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) {
2007 if (MI->isVolatile() || !isa<Constant>(MI->getLength()))
2008 return false;
Chandler Carruthc659df92014-10-16 20:24:07 +00002009 if (!S.isSplittable())
Chandler Carruthf0546402013-07-18 07:15:00 +00002010 return false; // Skip any unsplittable intrinsics.
2011 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
2012 if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
2013 II->getIntrinsicID() != Intrinsic::lifetime_end)
2014 return false;
2015 } else {
2016 return false;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002017 }
Chandler Carruthf0546402013-07-18 07:15:00 +00002018
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002019 return true;
2020}
2021
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00002022/// Test whether the given alloca partition's integer operations can be
Chandler Carruth435c4e02012-10-15 08:40:30 +00002023/// widened to promotable ones.
Chandler Carruth92924fd2012-09-24 00:34:20 +00002024///
Chandler Carruth435c4e02012-10-15 08:40:30 +00002025/// This is a quick test to check whether we can rewrite the integer loads and
2026/// stores to a particular alloca into wider loads and stores and be able to
2027/// promote the resulting alloca.
Chandler Carruth29a18a42015-09-12 09:09:14 +00002028static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
Chandler Carruth5031bbe2014-12-24 01:05:14 +00002029 const DataLayout &DL) {
Chandler Carruth90a735d2013-07-19 07:21:28 +00002030 uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy);
Benjamin Kramer47534c72012-12-01 11:53:32 +00002031 // Don't create integer types larger than the maximum bitwidth.
2032 if (SizeInBits > IntegerType::MAX_INT_BITS)
2033 return false;
Chandler Carruth435c4e02012-10-15 08:40:30 +00002034
2035 // Don't try to handle allocas with bit-padding.
Chandler Carruth90a735d2013-07-19 07:21:28 +00002036 if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy))
Chandler Carruth92924fd2012-09-24 00:34:20 +00002037 return false;
2038
Chandler Carruth58d05562012-10-25 04:37:07 +00002039 // We need to ensure that an integer type with the appropriate bitwidth can
2040 // be converted to the alloca type, whatever that is. We don't want to force
2041 // the alloca itself to have an integer type if there is a more suitable one.
2042 Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits);
Chandler Carruth90a735d2013-07-19 07:21:28 +00002043 if (!canConvertValue(DL, AllocaTy, IntTy) ||
2044 !canConvertValue(DL, IntTy, AllocaTy))
Chandler Carruth58d05562012-10-25 04:37:07 +00002045 return false;
2046
Chandler Carruthf0546402013-07-18 07:15:00 +00002047 // While examining uses, we ensure that the alloca has a covering load or
2048 // store. We don't want to widen the integer operations only to fail to
2049 // promote due to some other unsplittable entry (which we may make splittable
Chandler Carruth5955c9e2013-07-19 07:12:23 +00002050 // later). However, if there are only splittable uses, go ahead and assume
2051 // that we cover the alloca.
Chandler Carruth5031bbe2014-12-24 01:05:14 +00002052 // FIXME: We shouldn't consider split slices that happen to start in the
2053 // partition here...
Chandler Carruthc659df92014-10-16 20:24:07 +00002054 bool WholeAllocaOp =
Chandler Carruth5031bbe2014-12-24 01:05:14 +00002055 P.begin() != P.end() ? false : DL.isLegalInteger(SizeInBits);
Chandler Carruth43c8b462012-10-04 10:39:28 +00002056
Chandler Carruth5031bbe2014-12-24 01:05:14 +00002057 for (const Slice &S : P)
2058 if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL,
2059 WholeAllocaOp))
Chandler Carruth43c8b462012-10-04 10:39:28 +00002060 return false;
2061
Chandler Carruthffb7ce52014-12-24 01:48:09 +00002062 for (const Slice *S : P.splitSliceTails())
Chandler Carruth5031bbe2014-12-24 01:05:14 +00002063 if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL,
2064 WholeAllocaOp))
Chandler Carruth92924fd2012-09-24 00:34:20 +00002065 return false;
Chandler Carruthf0546402013-07-18 07:15:00 +00002066
Chandler Carruth92924fd2012-09-24 00:34:20 +00002067 return WholeAllocaOp;
2068}
2069
Chandler Carruthd177f862013-03-20 07:30:36 +00002070static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002071 IntegerType *Ty, uint64_t Offset,
2072 const Twine &Name) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002073 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002074 IntegerType *IntTy = cast<IntegerType>(V->getType());
2075 assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&
2076 "Element extends past full value");
Chandler Carruth113dc642014-12-20 02:39:18 +00002077 uint64_t ShAmt = 8 * Offset;
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002078 if (DL.isBigEndian())
Chandler Carruth113dc642014-12-20 02:39:18 +00002079 ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset);
Chandler Carruth18db7952012-11-20 01:12:50 +00002080 if (ShAmt) {
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002081 V = IRB.CreateLShr(V, ShAmt, Name + ".shift");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002082 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002083 }
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002084 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2085 "Cannot extract to a larger integer!");
Chandler Carruth18db7952012-11-20 01:12:50 +00002086 if (Ty != IntTy) {
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002087 V = IRB.CreateTrunc(V, Ty, Name + ".trunc");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002088 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002089 }
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002090 return V;
2091}
2092
Chandler Carruthd177f862013-03-20 07:30:36 +00002093static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002094 Value *V, uint64_t Offset, const Twine &Name) {
2095 IntegerType *IntTy = cast<IntegerType>(Old->getType());
2096 IntegerType *Ty = cast<IntegerType>(V->getType());
2097 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2098 "Cannot insert a larger integer!");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002099 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002100 if (Ty != IntTy) {
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002101 V = IRB.CreateZExt(V, IntTy, Name + ".ext");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002102 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002103 }
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002104 assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&
2105 "Element store outside of alloca store");
Chandler Carruth113dc642014-12-20 02:39:18 +00002106 uint64_t ShAmt = 8 * Offset;
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002107 if (DL.isBigEndian())
Chandler Carruth113dc642014-12-20 02:39:18 +00002108 ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset);
Chandler Carruth18db7952012-11-20 01:12:50 +00002109 if (ShAmt) {
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002110 V = IRB.CreateShl(V, ShAmt, Name + ".shift");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002111 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002112 }
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002113
2114 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2115 APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt);
2116 Old = IRB.CreateAnd(Old, Mask, Name + ".mask");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002117 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002118 V = IRB.CreateOr(Old, V, Name + ".insert");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002119 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002120 }
2121 return V;
2122}
2123
Chandler Carruth113dc642014-12-20 02:39:18 +00002124static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2125 unsigned EndIndex, const Twine &Name) {
Chandler Carruthb6bc8742012-12-17 13:07:30 +00002126 VectorType *VecTy = cast<VectorType>(V->getType());
2127 unsigned NumElements = EndIndex - BeginIndex;
2128 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2129
2130 if (NumElements == VecTy->getNumElements())
2131 return V;
2132
2133 if (NumElements == 1) {
2134 V = IRB.CreateExtractElement(V, IRB.getInt32(BeginIndex),
2135 Name + ".extract");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002136 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
Chandler Carruthb6bc8742012-12-17 13:07:30 +00002137 return V;
2138 }
2139
Chandler Carruth113dc642014-12-20 02:39:18 +00002140 SmallVector<Constant *, 8> Mask;
Chandler Carruthb6bc8742012-12-17 13:07:30 +00002141 Mask.reserve(NumElements);
2142 for (unsigned i = BeginIndex; i != EndIndex; ++i)
2143 Mask.push_back(IRB.getInt32(i));
2144 V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()),
Chandler Carruth113dc642014-12-20 02:39:18 +00002145 ConstantVector::get(Mask), Name + ".extract");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002146 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
Chandler Carruthb6bc8742012-12-17 13:07:30 +00002147 return V;
2148}
2149
Chandler Carruthd177f862013-03-20 07:30:36 +00002150static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
Chandler Carruthce4562b2012-12-17 13:41:21 +00002151 unsigned BeginIndex, const Twine &Name) {
2152 VectorType *VecTy = cast<VectorType>(Old->getType());
2153 assert(VecTy && "Can only insert a vector into a vector");
2154
2155 VectorType *Ty = dyn_cast<VectorType>(V->getType());
2156 if (!Ty) {
2157 // Single element to insert.
2158 V = IRB.CreateInsertElement(Old, V, IRB.getInt32(BeginIndex),
2159 Name + ".insert");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002160 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
Chandler Carruthce4562b2012-12-17 13:41:21 +00002161 return V;
2162 }
2163
2164 assert(Ty->getNumElements() <= VecTy->getNumElements() &&
2165 "Too many elements!");
2166 if (Ty->getNumElements() == VecTy->getNumElements()) {
2167 assert(V->getType() == VecTy && "Vector type mismatch");
2168 return V;
2169 }
2170 unsigned EndIndex = BeginIndex + Ty->getNumElements();
2171
2172 // When inserting a smaller vector into the larger to store, we first
2173 // use a shuffle vector to widen it with undef elements, and then
2174 // a second shuffle vector to select between the loaded vector and the
2175 // incoming vector.
Chandler Carruth113dc642014-12-20 02:39:18 +00002176 SmallVector<Constant *, 8> Mask;
Chandler Carruthce4562b2012-12-17 13:41:21 +00002177 Mask.reserve(VecTy->getNumElements());
2178 for (unsigned i = 0; i != VecTy->getNumElements(); ++i)
2179 if (i >= BeginIndex && i < EndIndex)
2180 Mask.push_back(IRB.getInt32(i - BeginIndex));
2181 else
2182 Mask.push_back(UndefValue::get(IRB.getInt32Ty()));
2183 V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()),
Chandler Carruth113dc642014-12-20 02:39:18 +00002184 ConstantVector::get(Mask), Name + ".expand");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002185 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
Chandler Carruthce4562b2012-12-17 13:41:21 +00002186
2187 Mask.clear();
2188 for (unsigned i = 0; i != VecTy->getNumElements(); ++i)
Nadav Rotem1e211912013-05-01 19:53:30 +00002189 Mask.push_back(IRB.getInt1(i >= BeginIndex && i < EndIndex));
2190
2191 V = IRB.CreateSelect(ConstantVector::get(Mask), V, Old, Name + "blend");
2192
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002193 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
Chandler Carruthce4562b2012-12-17 13:41:21 +00002194 return V;
2195}
2196
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00002197/// Visitor to rewrite instructions using p particular slice of an alloca
Chandler Carruth9f21fe12013-07-19 09:13:58 +00002198/// to use a new alloca.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002199///
2200/// Also implements the rewriting to vector-based accesses when the partition
2201/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2202/// lives here.
Chandler Carruth29a18a42015-09-12 09:09:14 +00002203class llvm::sroa::AllocaSliceRewriter
2204 : public InstVisitor<AllocaSliceRewriter, bool> {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002205 // Befriend the base class so it can delegate to private visit methods.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00002206 friend class InstVisitor<AllocaSliceRewriter, bool>;
2207
2208 using Base = InstVisitor<AllocaSliceRewriter, bool>;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002209
Chandler Carruth90a735d2013-07-19 07:21:28 +00002210 const DataLayout &DL;
Chandler Carruth83934062014-10-16 21:11:55 +00002211 AllocaSlices &AS;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002212 SROA &Pass;
2213 AllocaInst &OldAI, &NewAI;
2214 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
Chandler Carruth891fec02012-10-13 02:41:05 +00002215 Type *NewAllocaTy;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002216
Chandler Carruth2dc96822014-10-18 00:44:02 +00002217 // This is a convenience and flag variable that will be null unless the new
2218 // alloca's integer operations should be widened to this integer type due to
2219 // passing isIntegerWideningViable above. If it is non-null, the desired
2220 // integer type will be stored here for easy access during rewriting.
2221 IntegerType *IntTy;
2222
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002223 // If we are rewriting an alloca partition which can be written as pure
2224 // vector operations, we stash extra information here. When VecTy is
Jakub Staszak086f6cd2013-02-19 22:02:21 +00002225 // non-null, we have some strict guarantees about the rewritten alloca:
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002226 // - The new alloca is exactly the size of the vector type here.
2227 // - The accesses all either map to the entire vector or to a single
2228 // element.
2229 // - The set of accessing instructions is only one of those handled above
2230 // in isVectorPromotionViable. Generally these are the same access kinds
2231 // which are promotable via mem2reg.
2232 VectorType *VecTy;
2233 Type *ElementTy;
2234 uint64_t ElementSize;
2235
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002236 // The original offset of the slice currently being rewritten relative to
2237 // the original alloca.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00002238 uint64_t BeginOffset = 0;
2239 uint64_t EndOffset = 0;
2240
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002241 // The new offsets of the slice currently being rewritten relative to the
2242 // original alloca.
2243 uint64_t NewBeginOffset, NewEndOffset;
2244
2245 uint64_t SliceSize;
Eugene Zelenko75075ef2017-09-01 21:37:29 +00002246 bool IsSplittable = false;
2247 bool IsSplit = false;
2248 Use *OldUse = nullptr;
2249 Instruction *OldPtr = nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002250
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00002251 // Track post-rewrite users which are PHI nodes and Selects.
Davide Italiano81a26da2017-04-27 23:09:01 +00002252 SmallSetVector<PHINode *, 8> &PHIUsers;
2253 SmallSetVector<SelectInst *, 8> &SelectUsers;
Chandler Carruth83ea1952013-07-24 09:47:28 +00002254
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002255 // Utility IR builder, whose name prefix is setup for each visited use, and
2256 // the insertion point is set to point to the user.
2257 IRBuilderTy IRB;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002258
2259public:
Chandler Carruth83934062014-10-16 21:11:55 +00002260 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
Chandler Carruth9f21fe12013-07-19 09:13:58 +00002261 AllocaInst &OldAI, AllocaInst &NewAI,
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002262 uint64_t NewAllocaBeginOffset,
Chandler Carruth2dc96822014-10-18 00:44:02 +00002263 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2264 VectorType *PromotableVecTy,
Davide Italiano81a26da2017-04-27 23:09:01 +00002265 SmallSetVector<PHINode *, 8> &PHIUsers,
2266 SmallSetVector<SelectInst *, 8> &SelectUsers)
Chandler Carruth83934062014-10-16 21:11:55 +00002267 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002268 NewAllocaBeginOffset(NewAllocaBeginOffset),
2269 NewAllocaEndOffset(NewAllocaEndOffset),
Chandler Carruthf0546402013-07-18 07:15:00 +00002270 NewAllocaTy(NewAI.getAllocatedType()),
Chandler Carruthf0546402013-07-18 07:15:00 +00002271 IntTy(IsIntegerPromotable
2272 ? Type::getIntNTy(
2273 NewAI.getContext(),
Chandler Carruth90a735d2013-07-19 07:21:28 +00002274 DL.getTypeSizeInBits(NewAI.getAllocatedType()))
Craig Topperf40110f2014-04-25 05:29:35 +00002275 : nullptr),
Chandler Carruth2dc96822014-10-18 00:44:02 +00002276 VecTy(PromotableVecTy),
2277 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2278 ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy) / 8 : 0),
Eugene Zelenko75075ef2017-09-01 21:37:29 +00002279 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
Chandler Carruth83ea1952013-07-24 09:47:28 +00002280 IRB(NewAI.getContext(), ConstantFolder()) {
Chandler Carruthf0546402013-07-18 07:15:00 +00002281 if (VecTy) {
Chandler Carruth90a735d2013-07-19 07:21:28 +00002282 assert((DL.getTypeSizeInBits(ElementTy) % 8) == 0 &&
Chandler Carruthf0546402013-07-18 07:15:00 +00002283 "Only multiple-of-8 sized vector elements are viable");
2284 ++NumVectorized;
2285 }
Chandler Carruth2dc96822014-10-18 00:44:02 +00002286 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002287 }
2288
Chandler Carruth9f21fe12013-07-19 09:13:58 +00002289 bool visit(AllocaSlices::const_iterator I) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002290 bool CanSROA = true;
Chandler Carruthf0546402013-07-18 07:15:00 +00002291 BeginOffset = I->beginOffset();
2292 EndOffset = I->endOffset();
2293 IsSplittable = I->isSplittable();
2294 IsSplit =
2295 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002296 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2297 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2298 LLVM_DEBUG(dbgs() << "\n");
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002299
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002300 // Compute the intersecting offset range.
2301 assert(BeginOffset < NewAllocaEndOffset);
2302 assert(EndOffset > NewAllocaBeginOffset);
2303 NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset);
2304 NewEndOffset = std::min(EndOffset, NewAllocaEndOffset);
2305
2306 SliceSize = NewEndOffset - NewBeginOffset;
2307
Chandler Carruthf0546402013-07-18 07:15:00 +00002308 OldUse = I->getUse();
2309 OldPtr = cast<Instruction>(OldUse->get());
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002310
Chandler Carruthf0546402013-07-18 07:15:00 +00002311 Instruction *OldUserI = cast<Instruction>(OldUse->getUser());
2312 IRB.SetInsertPoint(OldUserI);
2313 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2314 IRB.SetNamePrefix(Twine(NewAI.getName()) + "." + Twine(BeginOffset) + ".");
2315
2316 CanSROA &= visit(cast<Instruction>(OldUse->getUser()));
2317 if (VecTy || IntTy)
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002318 assert(CanSROA);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002319 return CanSROA;
2320 }
2321
2322private:
Chandler Carruthf0546402013-07-18 07:15:00 +00002323 // Make sure the other visit overloads are visible.
2324 using Base::visit;
2325
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002326 // Every instruction which can end up as a user must have a rewrite rule.
2327 bool visitInstruction(Instruction &I) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002328 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002329 llvm_unreachable("No rewrite rule for this instruction!");
2330 }
2331
Chandler Carruth47954c82014-02-26 05:12:43 +00002332 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
2333 // Note that the offset computation can use BeginOffset or NewBeginOffset
2334 // interchangeably for unsplit slices.
2335 assert(IsSplit || BeginOffset == NewBeginOffset);
2336 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
2337
Chandler Carruthcb93cd22014-02-25 11:19:56 +00002338#ifndef NDEBUG
2339 StringRef OldName = OldPtr->getName();
2340 // Skip through the last '.sroa.' component of the name.
2341 size_t LastSROAPrefix = OldName.rfind(".sroa.");
2342 if (LastSROAPrefix != StringRef::npos) {
2343 OldName = OldName.substr(LastSROAPrefix + strlen(".sroa."));
2344 // Look for an SROA slice index.
2345 size_t IndexEnd = OldName.find_first_not_of("0123456789");
2346 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
2347 // Strip the index and look for the offset.
2348 OldName = OldName.substr(IndexEnd + 1);
2349 size_t OffsetEnd = OldName.find_first_not_of("0123456789");
2350 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
2351 // Strip the offset.
2352 OldName = OldName.substr(OffsetEnd + 1);
2353 }
2354 }
2355 // Strip any SROA suffixes as well.
2356 OldName = OldName.substr(0, OldName.find(".sroa_"));
2357#endif
Chandler Carruth47954c82014-02-26 05:12:43 +00002358
2359 return getAdjustedPtr(IRB, DL, &NewAI,
Matt Arsenault3c1fc762017-04-10 22:27:50 +00002360 APInt(DL.getPointerTypeSizeInBits(PointerTy), Offset),
2361 PointerTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00002362#ifndef NDEBUG
2363 Twine(OldName) + "."
2364#else
2365 Twine()
2366#endif
2367 );
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002368 }
2369
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00002370 /// Compute suitable alignment to access this slice of the *new*
Chandler Carruth113dc642014-12-20 02:39:18 +00002371 /// alloca.
Chandler Carruth2659e502014-02-26 05:02:19 +00002372 ///
2373 /// You can optionally pass a type to this routine and if that type's ABI
2374 /// alignment is itself suitable, this will return zero.
Craig Topperf40110f2014-04-25 05:29:35 +00002375 unsigned getSliceAlign(Type *Ty = nullptr) {
Chandler Carruth176ca712012-10-01 12:16:54 +00002376 unsigned NewAIAlign = NewAI.getAlignment();
2377 if (!NewAIAlign)
Chandler Carruth90a735d2013-07-19 07:21:28 +00002378 NewAIAlign = DL.getABITypeAlignment(NewAI.getAllocatedType());
Chandler Carruth113dc642014-12-20 02:39:18 +00002379 unsigned Align =
2380 MinAlign(NewAIAlign, NewBeginOffset - NewAllocaBeginOffset);
Chandler Carruth2659e502014-02-26 05:02:19 +00002381 return (Ty && Align == DL.getABITypeAlignment(Ty)) ? 0 : Align;
Chandler Carruth4b2b38d2012-10-03 08:14:02 +00002382 }
2383
Chandler Carruth845b73c2012-11-21 08:16:30 +00002384 unsigned getIndex(uint64_t Offset) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002385 assert(VecTy && "Can only call getIndex when rewriting a vector");
2386 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
2387 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
2388 uint32_t Index = RelOffset / ElementSize;
2389 assert(Index * ElementSize == RelOffset);
Chandler Carruth845b73c2012-11-21 08:16:30 +00002390 return Index;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002391 }
2392
2393 void deleteIfTriviallyDead(Value *V) {
2394 Instruction *I = cast<Instruction>(V);
2395 if (isInstructionTriviallyDead(I))
Chandler Carruth18db7952012-11-20 01:12:50 +00002396 Pass.DeadInsts.insert(I);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002397 }
2398
Chandler Carruthea27cf02014-02-26 04:25:04 +00002399 Value *rewriteVectorizedLoadInst() {
Chandler Carruthf0546402013-07-18 07:15:00 +00002400 unsigned BeginIndex = getIndex(NewBeginOffset);
2401 unsigned EndIndex = getIndex(NewEndOffset);
Chandler Carruth769445e2012-12-17 12:50:21 +00002402 assert(EndIndex > BeginIndex && "Empty vector!");
Chandler Carruthb6bc8742012-12-17 13:07:30 +00002403
Chandler Carruth113dc642014-12-20 02:39:18 +00002404 Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002405 return extractVector(IRB, V, BeginIndex, EndIndex, "vec");
Chandler Carruth769445e2012-12-17 12:50:21 +00002406 }
2407
Chandler Carruthea27cf02014-02-26 04:25:04 +00002408 Value *rewriteIntegerLoad(LoadInst &LI) {
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002409 assert(IntTy && "We cannot insert an integer to the alloca");
Chandler Carruth92924fd2012-09-24 00:34:20 +00002410 assert(!LI.isVolatile());
Chandler Carruth113dc642014-12-20 02:39:18 +00002411 Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
Chandler Carruth90a735d2013-07-19 07:21:28 +00002412 V = convertValue(DL, IRB, V, IntTy);
Chandler Carruthf0546402013-07-18 07:15:00 +00002413 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
2414 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
Chandler Carruth4b682f62015-08-28 09:03:52 +00002415 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
2416 IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8);
2417 V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract");
2418 }
2419 // It is possible that the extracted type is not the load type. This
2420 // happens if there is a load past the end of the alloca, and as
2421 // a consequence the slice is narrower but still a candidate for integer
2422 // lowering. To handle this case, we just zero extend the extracted
2423 // integer.
2424 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
2425 "Can only handle an extract for an overly wide load");
2426 if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8)
2427 V = IRB.CreateZExt(V, LI.getType());
Chandler Carruth18db7952012-11-20 01:12:50 +00002428 return V;
Chandler Carruth92924fd2012-09-24 00:34:20 +00002429 }
2430
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002431 bool visitLoadInst(LoadInst &LI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002432 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002433 Value *OldOp = LI.getOperand(0);
2434 assert(OldOp == OldPtr);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002435
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002436 AAMDNodes AATags;
2437 LI.getAAMetadata(AATags);
2438
Matt Arsenault3c1fc762017-04-10 22:27:50 +00002439 unsigned AS = LI.getPointerAddressSpace();
2440
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002441 Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
Chandler Carrutha1c54bb2013-03-14 11:32:24 +00002442 : LI.getType();
Chandler Carruthccffdaf2015-07-22 03:32:42 +00002443 const bool IsLoadPastEnd = DL.getTypeStoreSize(TargetTy) > SliceSize;
Chandler Carruth18db7952012-11-20 01:12:50 +00002444 bool IsPtrAdjusted = false;
2445 Value *V;
2446 if (VecTy) {
Chandler Carruthea27cf02014-02-26 04:25:04 +00002447 V = rewriteVectorizedLoadInst();
Chandler Carruth18db7952012-11-20 01:12:50 +00002448 } else if (IntTy && LI.getType()->isIntegerTy()) {
Chandler Carruthea27cf02014-02-26 04:25:04 +00002449 V = rewriteIntegerLoad(LI);
Chandler Carruthf0546402013-07-18 07:15:00 +00002450 } else if (NewBeginOffset == NewAllocaBeginOffset &&
Chandler Carruthccffdaf2015-07-22 03:32:42 +00002451 NewEndOffset == NewAllocaEndOffset &&
2452 (canConvertValue(DL, NewAllocaTy, TargetTy) ||
2453 (IsLoadPastEnd && NewAllocaTy->isIntegerTy() &&
2454 TargetTy->isIntegerTy()))) {
David Majnemer62690b12015-07-14 06:19:58 +00002455 LoadInst *NewLI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
2456 LI.isVolatile(), LI.getName());
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002457 if (AATags)
2458 NewLI->setAAMetadata(AATags);
David Majnemer62690b12015-07-14 06:19:58 +00002459 if (LI.isVolatile())
Konstantin Zhuravlyovbb80d3e2017-07-11 22:23:00 +00002460 NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
Luqman Aden3f807c92017-03-22 19:16:39 +00002461
Chandler Carruth3f81d802017-06-27 08:32:03 +00002462 // Any !nonnull metadata or !range metadata on the old load is also valid
2463 // on the new load. This is even true in some cases even when the loads
2464 // are different types, for example by mapping !nonnull metadata to
2465 // !range metadata by modeling the null pointer constant converted to the
2466 // integer type.
Rafael Espindolac06f55e2017-11-28 01:25:38 +00002467 // FIXME: Add support for range metadata here. Currently the utilities
2468 // for this don't propagate range metadata in trivial cases from one
2469 // integer load to another, don't handle non-addrspace-0 null pointers
2470 // correctly, and don't have any support for mapping ranges as the
2471 // integer type becomes winder or narrower.
Chandler Carruth3f81d802017-06-27 08:32:03 +00002472 if (MDNode *N = LI.getMetadata(LLVMContext::MD_nonnull))
2473 copyNonnullMetadata(LI, N, *NewLI);
Rafael Espindolac06f55e2017-11-28 01:25:38 +00002474
2475 // Try to preserve nonnull metadata
David Majnemer62690b12015-07-14 06:19:58 +00002476 V = NewLI;
Chandler Carruthccffdaf2015-07-22 03:32:42 +00002477
2478 // If this is an integer load past the end of the slice (which means the
2479 // bytes outside the slice are undef or this load is dead) just forcibly
2480 // fix the integer size with correct handling of endianness.
2481 if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy))
2482 if (auto *TITy = dyn_cast<IntegerType>(TargetTy))
2483 if (AITy->getBitWidth() < TITy->getBitWidth()) {
2484 V = IRB.CreateZExt(V, TITy, "load.ext");
2485 if (DL.isBigEndian())
2486 V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(),
2487 "endian_shift");
2488 }
Chandler Carruth18db7952012-11-20 01:12:50 +00002489 } else {
Matt Arsenault3c1fc762017-04-10 22:27:50 +00002490 Type *LTy = TargetTy->getPointerTo(AS);
David Majnemer62690b12015-07-14 06:19:58 +00002491 LoadInst *NewLI = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy),
2492 getSliceAlign(TargetTy),
2493 LI.isVolatile(), LI.getName());
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002494 if (AATags)
2495 NewLI->setAAMetadata(AATags);
David Majnemer62690b12015-07-14 06:19:58 +00002496 if (LI.isVolatile())
Konstantin Zhuravlyovbb80d3e2017-07-11 22:23:00 +00002497 NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
David Majnemer62690b12015-07-14 06:19:58 +00002498
2499 V = NewLI;
Chandler Carruth18db7952012-11-20 01:12:50 +00002500 IsPtrAdjusted = true;
2501 }
Chandler Carruth90a735d2013-07-19 07:21:28 +00002502 V = convertValue(DL, IRB, V, TargetTy);
Chandler Carruth18db7952012-11-20 01:12:50 +00002503
Chandler Carrutha1c54bb2013-03-14 11:32:24 +00002504 if (IsSplit) {
Chandler Carruth58d05562012-10-25 04:37:07 +00002505 assert(!LI.isVolatile());
2506 assert(LI.getType()->isIntegerTy() &&
2507 "Only integer type loads and stores are split");
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002508 assert(SliceSize < DL.getTypeStoreSize(LI.getType()) &&
Chandler Carrutha1c54bb2013-03-14 11:32:24 +00002509 "Split load isn't smaller than original load");
Chandler Carruth58d05562012-10-25 04:37:07 +00002510 assert(LI.getType()->getIntegerBitWidth() ==
Chandler Carruth113dc642014-12-20 02:39:18 +00002511 DL.getTypeStoreSizeInBits(LI.getType()) &&
Chandler Carruth58d05562012-10-25 04:37:07 +00002512 "Non-byte-multiple bit width");
Chandler Carruth58d05562012-10-25 04:37:07 +00002513 // Move the insertion point just past the load so that we can refer to it.
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00002514 IRB.SetInsertPoint(&*std::next(BasicBlock::iterator(&LI)));
Chandler Carruth58d05562012-10-25 04:37:07 +00002515 // Create a placeholder value with the same type as LI to use as the
2516 // basis for the new value. This allows us to replace the uses of LI with
2517 // the computed value, and then replace the placeholder with LI, leaving
2518 // LI only used for this computation.
Chandler Carruth113dc642014-12-20 02:39:18 +00002519 Value *Placeholder =
Matt Arsenault3c1fc762017-04-10 22:27:50 +00002520 new LoadInst(UndefValue::get(LI.getType()->getPointerTo(AS)));
Chandler Carruth24ac8302015-01-02 03:55:54 +00002521 V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset,
2522 "insert");
Chandler Carruth58d05562012-10-25 04:37:07 +00002523 LI.replaceAllUsesWith(V);
2524 Placeholder->replaceAllUsesWith(&LI);
Reid Kleckner96ab8722017-05-18 17:24:10 +00002525 Placeholder->deleteValue();
Chandler Carruth18db7952012-11-20 01:12:50 +00002526 } else {
2527 LI.replaceAllUsesWith(V);
Chandler Carruth58d05562012-10-25 04:37:07 +00002528 }
2529
Chandler Carruth18db7952012-11-20 01:12:50 +00002530 Pass.DeadInsts.insert(&LI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002531 deleteIfTriviallyDead(OldOp);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002532 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002533 return !LI.isVolatile() && !IsPtrAdjusted;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002534 }
2535
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002536 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
2537 AAMDNodes AATags) {
Bob Wilsonacfc01d2013-06-25 19:09:50 +00002538 if (V->getType() != VecTy) {
Chandler Carruthf0546402013-07-18 07:15:00 +00002539 unsigned BeginIndex = getIndex(NewBeginOffset);
2540 unsigned EndIndex = getIndex(NewEndOffset);
Bob Wilsonacfc01d2013-06-25 19:09:50 +00002541 assert(EndIndex > BeginIndex && "Empty vector!");
2542 unsigned NumElements = EndIndex - BeginIndex;
2543 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
Chandler Carruth113dc642014-12-20 02:39:18 +00002544 Type *SliceTy = (NumElements == 1)
2545 ? ElementTy
2546 : VectorType::get(ElementTy, NumElements);
Chandler Carruth9f21fe12013-07-19 09:13:58 +00002547 if (V->getType() != SliceTy)
2548 V = convertValue(DL, IRB, V, SliceTy);
Chandler Carruth845b73c2012-11-21 08:16:30 +00002549
Bob Wilsonacfc01d2013-06-25 19:09:50 +00002550 // Mix in the existing elements.
Chandler Carruth113dc642014-12-20 02:39:18 +00002551 Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
Bob Wilsonacfc01d2013-06-25 19:09:50 +00002552 V = insertVector(IRB, Old, V, BeginIndex, "vec");
2553 }
Chandler Carruth871ba722012-09-26 10:27:46 +00002554 StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002555 if (AATags)
2556 Store->setAAMetadata(AATags);
Chandler Carruth18db7952012-11-20 01:12:50 +00002557 Pass.DeadInsts.insert(&SI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002558
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002559 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002560 return true;
2561 }
2562
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002563 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002564 assert(IntTy && "We cannot extract an integer from the alloca");
Chandler Carruth92924fd2012-09-24 00:34:20 +00002565 assert(!SI.isVolatile());
Chandler Carruth90a735d2013-07-19 07:21:28 +00002566 if (DL.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) {
Chandler Carruth113dc642014-12-20 02:39:18 +00002567 Value *Old =
2568 IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
Chandler Carruth90a735d2013-07-19 07:21:28 +00002569 Old = convertValue(DL, IRB, Old, IntTy);
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002570 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
2571 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
Chandler Carruth113dc642014-12-20 02:39:18 +00002572 V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert");
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002573 }
Chandler Carruth90a735d2013-07-19 07:21:28 +00002574 V = convertValue(DL, IRB, V, NewAllocaTy);
Chandler Carruth59ff93af2012-10-18 09:56:08 +00002575 StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
Dorit Nuzmand1247a62016-09-22 07:56:23 +00002576 Store->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access);
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002577 if (AATags)
2578 Store->setAAMetadata(AATags);
Chandler Carruth18db7952012-11-20 01:12:50 +00002579 Pass.DeadInsts.insert(&SI);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002580 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
Chandler Carruth92924fd2012-09-24 00:34:20 +00002581 return true;
2582 }
2583
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002584 bool visitStoreInst(StoreInst &SI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002585 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002586 Value *OldOp = SI.getOperand(1);
2587 assert(OldOp == OldPtr);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002588
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002589 AAMDNodes AATags;
2590 SI.getAAMetadata(AATags);
2591
Chandler Carruth18db7952012-11-20 01:12:50 +00002592 Value *V = SI.getValueOperand();
Chandler Carruth891fec02012-10-13 02:41:05 +00002593
Chandler Carruthac8317f2012-10-04 12:33:50 +00002594 // Strip all inbounds GEPs and pointer casts to try to dig out any root
2595 // alloca that should be re-examined after promoting this alloca.
Chandler Carruth18db7952012-11-20 01:12:50 +00002596 if (V->getType()->isPointerTy())
2597 if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets()))
Chandler Carruthac8317f2012-10-04 12:33:50 +00002598 Pass.PostPromotionWorklist.insert(AI);
2599
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002600 if (SliceSize < DL.getTypeStoreSize(V->getType())) {
Chandler Carruth18db7952012-11-20 01:12:50 +00002601 assert(!SI.isVolatile());
2602 assert(V->getType()->isIntegerTy() &&
2603 "Only integer type loads and stores are split");
2604 assert(V->getType()->getIntegerBitWidth() ==
Chandler Carruth113dc642014-12-20 02:39:18 +00002605 DL.getTypeStoreSizeInBits(V->getType()) &&
Chandler Carruth18db7952012-11-20 01:12:50 +00002606 "Non-byte-multiple bit width");
Chandler Carruthc46b6eb2014-02-26 04:20:00 +00002607 IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8);
Chandler Carruth24ac8302015-01-02 03:55:54 +00002608 V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset,
2609 "extract");
Chandler Carruth891fec02012-10-13 02:41:05 +00002610 }
2611
Chandler Carruth18db7952012-11-20 01:12:50 +00002612 if (VecTy)
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002613 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
Chandler Carruth18db7952012-11-20 01:12:50 +00002614 if (IntTy && V->getType()->isIntegerTy())
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002615 return rewriteIntegerStore(V, SI, AATags);
Chandler Carruth435c4e02012-10-15 08:40:30 +00002616
Chandler Carruthccffdaf2015-07-22 03:32:42 +00002617 const bool IsStorePastEnd = DL.getTypeStoreSize(V->getType()) > SliceSize;
Chandler Carruth18db7952012-11-20 01:12:50 +00002618 StoreInst *NewSI;
Chandler Carruthf0546402013-07-18 07:15:00 +00002619 if (NewBeginOffset == NewAllocaBeginOffset &&
2620 NewEndOffset == NewAllocaEndOffset &&
Chandler Carruthccffdaf2015-07-22 03:32:42 +00002621 (canConvertValue(DL, V->getType(), NewAllocaTy) ||
2622 (IsStorePastEnd && NewAllocaTy->isIntegerTy() &&
2623 V->getType()->isIntegerTy()))) {
2624 // If this is an integer store past the end of slice (and thus the bytes
2625 // past that point are irrelevant or this is unreachable), truncate the
2626 // value prior to storing.
2627 if (auto *VITy = dyn_cast<IntegerType>(V->getType()))
2628 if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy))
2629 if (VITy->getBitWidth() > AITy->getBitWidth()) {
2630 if (DL.isBigEndian())
2631 V = IRB.CreateLShr(V, VITy->getBitWidth() - AITy->getBitWidth(),
2632 "endian_shift");
2633 V = IRB.CreateTrunc(V, AITy, "load.trunc");
2634 }
2635
Chandler Carruth90a735d2013-07-19 07:21:28 +00002636 V = convertValue(DL, IRB, V, NewAllocaTy);
Chandler Carruth18db7952012-11-20 01:12:50 +00002637 NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(),
2638 SI.isVolatile());
2639 } else {
Matt Arsenault3c1fc762017-04-10 22:27:50 +00002640 unsigned AS = SI.getPointerAddressSpace();
2641 Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo(AS));
Chandler Carruth2659e502014-02-26 05:02:19 +00002642 NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()),
2643 SI.isVolatile());
Chandler Carruth18db7952012-11-20 01:12:50 +00002644 }
Dorit Nuzmand1247a62016-09-22 07:56:23 +00002645 NewSI->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access);
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002646 if (AATags)
2647 NewSI->setAAMetadata(AATags);
David Majnemer62690b12015-07-14 06:19:58 +00002648 if (SI.isVolatile())
Konstantin Zhuravlyovbb80d3e2017-07-11 22:23:00 +00002649 NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
Chandler Carruth18db7952012-11-20 01:12:50 +00002650 Pass.DeadInsts.insert(&SI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002651 deleteIfTriviallyDead(OldOp);
Chandler Carruth18db7952012-11-20 01:12:50 +00002652
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002653 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
Chandler Carruth18db7952012-11-20 01:12:50 +00002654 return NewSI->getPointerOperand() == &NewAI && !SI.isVolatile();
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002655 }
2656
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00002657 /// Compute an integer value from splatting an i8 across the given
Chandler Carruth514f34f2012-12-17 04:07:30 +00002658 /// number of bytes.
2659 ///
2660 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
2661 /// call this routine.
Jakub Staszak086f6cd2013-02-19 22:02:21 +00002662 /// FIXME: Heed the advice above.
Chandler Carruth514f34f2012-12-17 04:07:30 +00002663 ///
2664 /// \param V The i8 value to splat.
2665 /// \param Size The number of bytes in the output (assuming i8 is one byte)
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002666 Value *getIntegerSplat(Value *V, unsigned Size) {
Chandler Carruth514f34f2012-12-17 04:07:30 +00002667 assert(Size > 0 && "Expected a positive number of bytes.");
2668 IntegerType *VTy = cast<IntegerType>(V->getType());
2669 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
2670 if (Size == 1)
2671 return V;
2672
Chandler Carruth113dc642014-12-20 02:39:18 +00002673 Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8);
2674 V = IRB.CreateMul(
2675 IRB.CreateZExt(V, SplatIntTy, "zext"),
2676 ConstantExpr::getUDiv(
2677 Constant::getAllOnesValue(SplatIntTy),
2678 ConstantExpr::getZExt(Constant::getAllOnesValue(V->getType()),
2679 SplatIntTy)),
2680 "isplat");
Chandler Carruth514f34f2012-12-17 04:07:30 +00002681 return V;
2682 }
2683
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00002684 /// Compute a vector splat for a given element value.
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002685 Value *getVectorSplat(Value *V, unsigned NumElements) {
2686 V = IRB.CreateVectorSplat(NumElements, V, "vsplat");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002687 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
Chandler Carruthccca5042012-12-17 04:07:37 +00002688 return V;
2689 }
2690
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002691 bool visitMemSetInst(MemSetInst &II) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002692 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002693 assert(II.getRawDest() == OldPtr);
2694
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002695 AAMDNodes AATags;
2696 II.getAAMetadata(AATags);
2697
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002698 // If the memset has a variable size, it cannot be split, just adjust the
2699 // pointer to the new alloca.
2700 if (!isa<Constant>(II.getLength())) {
Chandler Carruthf0546402013-07-18 07:15:00 +00002701 assert(!IsSplit);
Chandler Carruth735d5bee2014-02-26 04:45:24 +00002702 assert(NewBeginOffset == BeginOffset);
Chandler Carruth47954c82014-02-26 05:12:43 +00002703 II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType()));
Daniel Neilson41e781d2018-03-13 14:25:33 +00002704 II.setDestAlignment(getSliceAlign());
Chandler Carruth208124f2012-09-26 10:59:22 +00002705
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002706 deleteIfTriviallyDead(OldPtr);
2707 return false;
2708 }
2709
2710 // Record this instruction for deletion.
Chandler Carruth18db7952012-11-20 01:12:50 +00002711 Pass.DeadInsts.insert(&II);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002712
2713 Type *AllocaTy = NewAI.getAllocatedType();
2714 Type *ScalarTy = AllocaTy->getScalarType();
2715
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002716 // If this doesn't map cleanly onto the alloca type, and that type isn't
2717 // a single value type, just emit a memset.
Chandler Carruth9d966a22012-10-15 10:24:40 +00002718 if (!VecTy && !IntTy &&
Chandler Carruth113dc642014-12-20 02:39:18 +00002719 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
Reid Klecknerc36f48f2014-08-22 00:09:56 +00002720 SliceSize != DL.getTypeStoreSize(AllocaTy) ||
Chandler Carruth9d966a22012-10-15 10:24:40 +00002721 !AllocaTy->isSingleValueType() ||
Chandler Carruth90a735d2013-07-19 07:21:28 +00002722 !DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy)) ||
Chandler Carruth113dc642014-12-20 02:39:18 +00002723 DL.getTypeSizeInBits(ScalarTy) % 8 != 0)) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002724 Type *SizeTy = II.getLength()->getType();
Chandler Carruthf0546402013-07-18 07:15:00 +00002725 Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset);
2726 CallInst *New = IRB.CreateMemSet(
Chandler Carruth47954c82014-02-26 05:12:43 +00002727 getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size,
2728 getSliceAlign(), II.isVolatile());
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002729 if (AATags)
2730 New->setAAMetadata(AATags);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002731 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002732 return false;
2733 }
2734
2735 // If we can represent this as a simple value, we have to build the actual
2736 // value to store, which requires expanding the byte present in memset to
2737 // a sensible representation for the alloca type. This is essentially
Chandler Carruthccca5042012-12-17 04:07:37 +00002738 // splatting the byte to a sufficiently wide integer, splatting it across
2739 // any desired vector width, and bitcasting to the final type.
Benjamin Kramerc003a452013-01-01 16:13:35 +00002740 Value *V;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002741
Chandler Carruthccca5042012-12-17 04:07:37 +00002742 if (VecTy) {
2743 // If this is a memset of a vectorized alloca, insert it.
2744 assert(ElementTy == ScalarTy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002745
Chandler Carruthf0546402013-07-18 07:15:00 +00002746 unsigned BeginIndex = getIndex(NewBeginOffset);
2747 unsigned EndIndex = getIndex(NewEndOffset);
Chandler Carruthccca5042012-12-17 04:07:37 +00002748 assert(EndIndex > BeginIndex && "Empty vector!");
2749 unsigned NumElements = EndIndex - BeginIndex;
2750 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2751
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002752 Value *Splat =
Chandler Carruth90a735d2013-07-19 07:21:28 +00002753 getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ElementTy) / 8);
2754 Splat = convertValue(DL, IRB, Splat, ElementTy);
Chandler Carruthcacda252012-12-17 14:03:01 +00002755 if (NumElements > 1)
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002756 Splat = getVectorSplat(Splat, NumElements);
Chandler Carruthccca5042012-12-17 04:07:37 +00002757
Chandler Carruth113dc642014-12-20 02:39:18 +00002758 Value *Old =
2759 IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002760 V = insertVector(IRB, Old, Splat, BeginIndex, "vec");
Chandler Carruthccca5042012-12-17 04:07:37 +00002761 } else if (IntTy) {
2762 // If this is a memset on an alloca where we can widen stores, insert the
2763 // set integer.
Chandler Carruth9d966a22012-10-15 10:24:40 +00002764 assert(!II.isVolatile());
Chandler Carruthccca5042012-12-17 04:07:37 +00002765
Chandler Carruthf0546402013-07-18 07:15:00 +00002766 uint64_t Size = NewEndOffset - NewBeginOffset;
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002767 V = getIntegerSplat(II.getValue(), Size);
Chandler Carruthccca5042012-12-17 04:07:37 +00002768
2769 if (IntTy && (BeginOffset != NewAllocaBeginOffset ||
2770 EndOffset != NewAllocaBeginOffset)) {
Chandler Carruth113dc642014-12-20 02:39:18 +00002771 Value *Old =
2772 IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
Chandler Carruth90a735d2013-07-19 07:21:28 +00002773 Old = convertValue(DL, IRB, Old, IntTy);
Chandler Carruthf0546402013-07-18 07:15:00 +00002774 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
Chandler Carruth90a735d2013-07-19 07:21:28 +00002775 V = insertInteger(DL, IRB, Old, V, Offset, "insert");
Chandler Carruthccca5042012-12-17 04:07:37 +00002776 } else {
2777 assert(V->getType() == IntTy &&
2778 "Wrong type for an alloca wide integer!");
2779 }
Chandler Carruth90a735d2013-07-19 07:21:28 +00002780 V = convertValue(DL, IRB, V, AllocaTy);
Chandler Carruthccca5042012-12-17 04:07:37 +00002781 } else {
2782 // Established these invariants above.
Chandler Carruthf0546402013-07-18 07:15:00 +00002783 assert(NewBeginOffset == NewAllocaBeginOffset);
2784 assert(NewEndOffset == NewAllocaEndOffset);
Chandler Carruthccca5042012-12-17 04:07:37 +00002785
Chandler Carruth90a735d2013-07-19 07:21:28 +00002786 V = getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ScalarTy) / 8);
Chandler Carruthccca5042012-12-17 04:07:37 +00002787 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy))
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002788 V = getVectorSplat(V, AllocaVecTy->getNumElements());
Chandler Carruth95e1fb82012-12-17 13:51:03 +00002789
Chandler Carruth90a735d2013-07-19 07:21:28 +00002790 V = convertValue(DL, IRB, V, AllocaTy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002791 }
2792
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002793 StoreInst *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(),
2794 II.isVolatile());
2795 if (AATags)
2796 New->setAAMetadata(AATags);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002797 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002798 return !II.isVolatile();
2799 }
2800
2801 bool visitMemTransferInst(MemTransferInst &II) {
2802 // Rewriting of memory transfer instructions can be a bit tricky. We break
2803 // them into two categories: split intrinsics and unsplit intrinsics.
2804
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002805 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002806
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002807 AAMDNodes AATags;
2808 II.getAAMetadata(AATags);
2809
Chandler Carruthbb2a9322014-02-25 03:50:14 +00002810 bool IsDest = &II.getRawDestUse() == OldUse;
Alexey Samsonov26af6f72014-02-25 07:56:00 +00002811 assert((IsDest && II.getRawDest() == OldPtr) ||
Chandler Carruthbb2a9322014-02-25 03:50:14 +00002812 (!IsDest && II.getRawSource() == OldPtr));
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002813
Chandler Carruthaa72b932014-02-26 07:29:54 +00002814 unsigned SliceAlign = getSliceAlign();
Chandler Carruth176ca712012-10-01 12:16:54 +00002815
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002816 // For unsplit intrinsics, we simply modify the source and destination
2817 // pointers in place. This isn't just an optimization, it is a matter of
2818 // correctness. With unsplit intrinsics we may be dealing with transfers
2819 // within a single alloca before SROA ran, or with transfers that have
2820 // a variable length. We may also be dealing with memmove instead of
2821 // memcpy, and so simply updating the pointers is the necessary for us to
2822 // update both source and dest of a single call.
Chandler Carruthf0546402013-07-18 07:15:00 +00002823 if (!IsSplittable) {
Chandler Carruth47954c82014-02-26 05:12:43 +00002824 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
Daniel Neilson41e781d2018-03-13 14:25:33 +00002825 if (IsDest) {
Chandler Carruth8183a502014-02-25 11:08:02 +00002826 II.setDest(AdjustedPtr);
Daniel Neilson41e781d2018-03-13 14:25:33 +00002827 II.setDestAlignment(SliceAlign);
2828 }
2829 else {
Chandler Carruth8183a502014-02-25 11:08:02 +00002830 II.setSource(AdjustedPtr);
Daniel Neilson41e781d2018-03-13 14:25:33 +00002831 II.setSourceAlignment(SliceAlign);
Chandler Carruth181ed052014-02-26 05:33:36 +00002832 }
Chandler Carruth208124f2012-09-26 10:59:22 +00002833
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002834 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
Chandler Carruth8183a502014-02-25 11:08:02 +00002835 deleteIfTriviallyDead(OldPtr);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002836 return false;
2837 }
2838 // For split transfer intrinsics we have an incredibly useful assurance:
2839 // the source and destination do not reside within the same alloca, and at
2840 // least one of them does not escape. This means that we can replace
2841 // memmove with memcpy, and we don't need to worry about all manner of
2842 // downsides to splitting and transforming the operations.
2843
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002844 // If this doesn't map cleanly onto the alloca type, and that type isn't
2845 // a single value type, just emit a memcpy.
Reid Klecknerc36f48f2014-08-22 00:09:56 +00002846 bool EmitMemCpy =
2847 !VecTy && !IntTy &&
2848 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
2849 SliceSize != DL.getTypeStoreSize(NewAI.getAllocatedType()) ||
2850 !NewAI.getAllocatedType()->isSingleValueType());
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002851
2852 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
2853 // size hasn't been shrunk based on analysis of the viable range, this is
2854 // a no-op.
2855 if (EmitMemCpy && &OldAI == &NewAI) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002856 // Ensure the start lines up.
Chandler Carruthf0546402013-07-18 07:15:00 +00002857 assert(NewBeginOffset == BeginOffset);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002858
2859 // Rewrite the size as needed.
Chandler Carruthf0546402013-07-18 07:15:00 +00002860 if (NewEndOffset != EndOffset)
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002861 II.setLength(ConstantInt::get(II.getLength()->getType(),
Chandler Carruthf0546402013-07-18 07:15:00 +00002862 NewEndOffset - NewBeginOffset));
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002863 return false;
2864 }
2865 // Record this instruction for deletion.
Chandler Carruth18db7952012-11-20 01:12:50 +00002866 Pass.DeadInsts.insert(&II);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002867
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002868 // Strip all inbounds GEPs and pointer casts to try to dig out any root
2869 // alloca that should be re-examined after rewriting this instruction.
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002870 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
Chandler Carruth113dc642014-12-20 02:39:18 +00002871 if (AllocaInst *AI =
2872 dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets())) {
Chandler Carruth1bf38c62014-01-19 12:16:54 +00002873 assert(AI != &OldAI && AI != &NewAI &&
2874 "Splittable transfers cannot reach the same alloca on both ends.");
Chandler Carruth4bd8f662012-09-26 07:41:40 +00002875 Pass.Worklist.insert(AI);
Chandler Carruth1bf38c62014-01-19 12:16:54 +00002876 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002877
Chandler Carruth286d87e2014-02-26 08:25:02 +00002878 Type *OtherPtrTy = OtherPtr->getType();
2879 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
2880
Chandler Carruth181ed052014-02-26 05:33:36 +00002881 // Compute the relative offset for the other pointer within the transfer.
Chandler Carruth286d87e2014-02-26 08:25:02 +00002882 unsigned IntPtrWidth = DL.getPointerSizeInBits(OtherAS);
Chandler Carruth181ed052014-02-26 05:33:36 +00002883 APInt OtherOffset(IntPtrWidth, NewBeginOffset - BeginOffset);
Daniel Neilson41e781d2018-03-13 14:25:33 +00002884 unsigned OtherAlign =
2885 IsDest ? II.getSourceAlignment() : II.getDestAlignment();
2886 OtherAlign = MinAlign(OtherAlign ? OtherAlign : 1,
2887 OtherOffset.zextOrTrunc(64).getZExtValue());
Chandler Carruth181ed052014-02-26 05:33:36 +00002888
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002889 if (EmitMemCpy) {
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002890 // Compute the other pointer, folding as much as possible to produce
2891 // a single, simple GEP in most cases.
Chandler Carruth181ed052014-02-26 05:33:36 +00002892 OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00002893 OtherPtr->getName() + ".");
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002894
Chandler Carruth47954c82014-02-26 05:12:43 +00002895 Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002896 Type *SizeTy = II.getLength()->getType();
Chandler Carruthf0546402013-07-18 07:15:00 +00002897 Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002898
Daniel Neilson41e781d2018-03-13 14:25:33 +00002899 Value *DestPtr, *SrcPtr;
2900 unsigned DestAlign, SrcAlign;
2901 // Note: IsDest is true iff we're copying into the new alloca slice
2902 if (IsDest) {
2903 DestPtr = OurPtr;
2904 DestAlign = SliceAlign;
2905 SrcPtr = OtherPtr;
2906 SrcAlign = OtherAlign;
2907 } else {
2908 DestPtr = OtherPtr;
2909 DestAlign = OtherAlign;
2910 SrcPtr = OurPtr;
2911 SrcAlign = SliceAlign;
2912 }
2913 CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign,
2914 Size, II.isVolatile());
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002915 if (AATags)
2916 New->setAAMetadata(AATags);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002917 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002918 return false;
2919 }
2920
Chandler Carruthf0546402013-07-18 07:15:00 +00002921 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
2922 NewEndOffset == NewAllocaEndOffset;
2923 uint64_t Size = NewEndOffset - NewBeginOffset;
2924 unsigned BeginIndex = VecTy ? getIndex(NewBeginOffset) : 0;
2925 unsigned EndIndex = VecTy ? getIndex(NewEndOffset) : 0;
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002926 unsigned NumElements = EndIndex - BeginIndex;
Chandler Carruth113dc642014-12-20 02:39:18 +00002927 IntegerType *SubIntTy =
2928 IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr;
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002929
Chandler Carruth286d87e2014-02-26 08:25:02 +00002930 // Reset the other pointer type to match the register type we're going to
2931 // use, but using the address space of the original other pointer.
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002932 if (VecTy && !IsWholeAlloca) {
2933 if (NumElements == 1)
2934 OtherPtrTy = VecTy->getElementType();
2935 else
2936 OtherPtrTy = VectorType::get(VecTy->getElementType(), NumElements);
2937
Chandler Carruth286d87e2014-02-26 08:25:02 +00002938 OtherPtrTy = OtherPtrTy->getPointerTo(OtherAS);
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002939 } else if (IntTy && !IsWholeAlloca) {
Chandler Carruth286d87e2014-02-26 08:25:02 +00002940 OtherPtrTy = SubIntTy->getPointerTo(OtherAS);
2941 } else {
2942 OtherPtrTy = NewAllocaTy->getPointerTo(OtherAS);
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002943 }
2944
Chandler Carruth181ed052014-02-26 05:33:36 +00002945 Value *SrcPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
Chandler Carruthcb93cd22014-02-25 11:19:56 +00002946 OtherPtr->getName() + ".");
Pete Cooper67cf9a72015-11-19 05:56:52 +00002947 unsigned SrcAlign = OtherAlign;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002948 Value *DstPtr = &NewAI;
Chandler Carruthaa72b932014-02-26 07:29:54 +00002949 unsigned DstAlign = SliceAlign;
2950 if (!IsDest) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002951 std::swap(SrcPtr, DstPtr);
Chandler Carruthaa72b932014-02-26 07:29:54 +00002952 std::swap(SrcAlign, DstAlign);
2953 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002954
2955 Value *Src;
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002956 if (VecTy && !IsWholeAlloca && !IsDest) {
Chandler Carruth113dc642014-12-20 02:39:18 +00002957 Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002958 Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec");
Chandler Carruth49c8eea2012-10-15 10:24:43 +00002959 } else if (IntTy && !IsWholeAlloca && !IsDest) {
Chandler Carruth113dc642014-12-20 02:39:18 +00002960 Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
Chandler Carruth90a735d2013-07-19 07:21:28 +00002961 Src = convertValue(DL, IRB, Src, IntTy);
Chandler Carruthf0546402013-07-18 07:15:00 +00002962 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
Chandler Carruth90a735d2013-07-19 07:21:28 +00002963 Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002964 } else {
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002965 LoadInst *Load = IRB.CreateAlignedLoad(SrcPtr, SrcAlign, II.isVolatile(),
2966 "copyload");
2967 if (AATags)
2968 Load->setAAMetadata(AATags);
2969 Src = Load;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002970 }
2971
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002972 if (VecTy && !IsWholeAlloca && IsDest) {
Chandler Carruth113dc642014-12-20 02:39:18 +00002973 Value *Old =
2974 IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
Chandler Carruth34f0c7f2013-03-21 09:52:18 +00002975 Src = insertVector(IRB, Old, Src, BeginIndex, "vec");
Chandler Carruth21eb4e92012-12-17 14:51:24 +00002976 } else if (IntTy && !IsWholeAlloca && IsDest) {
Chandler Carruth113dc642014-12-20 02:39:18 +00002977 Value *Old =
2978 IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
Chandler Carruth90a735d2013-07-19 07:21:28 +00002979 Old = convertValue(DL, IRB, Old, IntTy);
Chandler Carruthf0546402013-07-18 07:15:00 +00002980 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
Chandler Carruth90a735d2013-07-19 07:21:28 +00002981 Src = insertInteger(DL, IRB, Old, Src, Offset, "insert");
2982 Src = convertValue(DL, IRB, Src, NewAllocaTy);
Chandler Carruth49c8eea2012-10-15 10:24:43 +00002983 }
2984
Chandler Carruth871ba722012-09-26 10:27:46 +00002985 StoreInst *Store = cast<StoreInst>(
Chandler Carruthaa72b932014-02-26 07:29:54 +00002986 IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00002987 if (AATags)
2988 Store->setAAMetadata(AATags);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002989 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002990 return !II.isVolatile();
2991 }
2992
2993 bool visitIntrinsicInst(IntrinsicInst &II) {
2994 assert(II.getIntrinsicID() == Intrinsic::lifetime_start ||
2995 II.getIntrinsicID() == Intrinsic::lifetime_end);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00002996 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002997 assert(II.getArgOperand(1) == OldPtr);
2998
Chandler Carruth1b398ae2012-09-14 09:22:59 +00002999 // Record this instruction for deletion.
Chandler Carruth18db7952012-11-20 01:12:50 +00003000 Pass.DeadInsts.insert(&II);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003001
Eli Friedman50967752016-11-28 21:50:34 +00003002 // Lifetime intrinsics are only promotable if they cover the whole alloca.
3003 // Therefore, we drop lifetime intrinsics which don't cover the whole
3004 // alloca.
3005 // (In theory, intrinsics which partially cover an alloca could be
3006 // promoted, but PromoteMemToReg doesn't handle that case.)
3007 // FIXME: Check whether the alloca is promotable before dropping the
3008 // lifetime intrinsics?
3009 if (NewBeginOffset != NewAllocaBeginOffset ||
3010 NewEndOffset != NewAllocaEndOffset)
3011 return true;
3012
Chandler Carruth113dc642014-12-20 02:39:18 +00003013 ConstantInt *Size =
3014 ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()),
Chandler Carruthf0546402013-07-18 07:15:00 +00003015 NewEndOffset - NewBeginOffset);
Chandler Carruth47954c82014-02-26 05:12:43 +00003016 Value *Ptr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003017 Value *New;
3018 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
3019 New = IRB.CreateLifetimeStart(Ptr, Size);
3020 else
3021 New = IRB.CreateLifetimeEnd(Ptr, Size);
3022
Edwin Vane82f80d42013-01-29 17:42:24 +00003023 (void)New;
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003024 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
Eli Friedman2a65dd12016-08-08 01:30:53 +00003025
Eli Friedman50967752016-11-28 21:50:34 +00003026 return true;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003027 }
3028
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003029 bool visitPHINode(PHINode &PN) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003030 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00003031 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
3032 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
Chandler Carruth82a57542012-10-01 10:54:05 +00003033
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003034 // We would like to compute a new pointer in only one place, but have it be
3035 // as local as possible to the PHI. To do that, we re-use the location of
3036 // the old pointer, which necessarily must be in the right position to
3037 // dominate the PHI.
Chandler Carruth51175532014-02-25 11:12:04 +00003038 IRBuilderTy PtrBuilder(IRB);
David Majnemerd4cffcf2014-09-01 21:20:14 +00003039 if (isa<PHINode>(OldPtr))
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00003040 PtrBuilder.SetInsertPoint(&*OldPtr->getParent()->getFirstInsertionPt());
David Majnemerd4cffcf2014-09-01 21:20:14 +00003041 else
3042 PtrBuilder.SetInsertPoint(OldPtr);
Chandler Carruth51175532014-02-25 11:12:04 +00003043 PtrBuilder.SetCurrentDebugLocation(OldPtr->getDebugLoc());
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003044
Chandler Carruth47954c82014-02-26 05:12:43 +00003045 Value *NewPtr = getNewAllocaSlicePtr(PtrBuilder, OldPtr->getType());
Chandler Carruth82a57542012-10-01 10:54:05 +00003046 // Replace the operands which were using the old pointer.
Benjamin Kramer7ddd7052012-10-20 12:04:57 +00003047 std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003048
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003049 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
Chandler Carruth82a57542012-10-01 10:54:05 +00003050 deleteIfTriviallyDead(OldPtr);
Chandler Carruthf0546402013-07-18 07:15:00 +00003051
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00003052 // PHIs can't be promoted on their own, but often can be speculated. We
3053 // check the speculation outside of the rewriter so that we see the
3054 // fully-rewritten alloca.
3055 PHIUsers.insert(&PN);
3056 return true;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003057 }
3058
3059 bool visitSelectInst(SelectInst &SI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003060 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
Benjamin Kramer0212dc22013-04-21 17:48:39 +00003061 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
3062 "Pointer isn't an operand!");
Chandler Carruthf0546402013-07-18 07:15:00 +00003063 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
3064 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
Chandler Carruth82a57542012-10-01 10:54:05 +00003065
Chandler Carruth47954c82014-02-26 05:12:43 +00003066 Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
Benjamin Kramer0212dc22013-04-21 17:48:39 +00003067 // Replace the operands which were using the old pointer.
3068 if (SI.getOperand(1) == OldPtr)
3069 SI.setOperand(1, NewPtr);
3070 if (SI.getOperand(2) == OldPtr)
3071 SI.setOperand(2, NewPtr);
3072
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003073 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003074 deleteIfTriviallyDead(OldPtr);
Chandler Carruthf0546402013-07-18 07:15:00 +00003075
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00003076 // Selects can't be promoted on their own, but often can be speculated. We
3077 // check the speculation outside of the rewriter so that we see the
3078 // fully-rewritten alloca.
3079 SelectUsers.insert(&SI);
3080 return true;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003081 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003082};
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003083
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003084namespace {
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003085
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003086/// Visitor to rewrite aggregate loads and stores as scalar.
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003087///
3088/// This pass aggressively rewrites all aggregate loads and stores on
3089/// a particular pointer (or any pointer derived from it which we can identify)
3090/// with scalar loads and stores.
3091class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
3092 // Befriend the base class so it can delegate to private visit methods.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003093 friend class InstVisitor<AggLoadStoreRewriter, bool>;
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003094
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003095 /// Queue of pointer uses to analyze and potentially rewrite.
3096 SmallVector<Use *, 8> Queue;
3097
3098 /// Set to prevent us from cycling with phi nodes and loops.
3099 SmallPtrSet<User *, 8> Visited;
3100
3101 /// The current pointer use being rewritten. This is used to dig up the used
3102 /// value (as opposed to the user).
3103 Use *U;
3104
3105public:
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003106 /// Rewrite loads and stores through a pointer and all pointers derived from
3107 /// it.
3108 bool rewrite(Instruction &I) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003109 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003110 enqueueUsers(I);
3111 bool Changed = false;
3112 while (!Queue.empty()) {
3113 U = Queue.pop_back_val();
3114 Changed |= visit(cast<Instruction>(U->getUser()));
3115 }
3116 return Changed;
3117 }
3118
3119private:
3120 /// Enqueue all the users of the given instruction for further processing.
3121 /// This uses a set to de-duplicate users.
3122 void enqueueUsers(Instruction &I) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00003123 for (Use &U : I.uses())
David Blaikie70573dc2014-11-19 07:49:26 +00003124 if (Visited.insert(U.getUser()).second)
Chandler Carruthcdf47882014-03-09 03:16:01 +00003125 Queue.push_back(&U);
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003126 }
3127
3128 // Conservative default is to not rewrite anything.
3129 bool visitInstruction(Instruction &I) { return false; }
3130
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003131 /// Generic recursive split emission class.
Chandler Carruth113dc642014-12-20 02:39:18 +00003132 template <typename Derived> class OpSplitter {
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003133 protected:
3134 /// The builder used to form new instructions.
Chandler Carruthd177f862013-03-20 07:30:36 +00003135 IRBuilderTy IRB;
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003136
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003137 /// The indices which to be used with insert- or extractvalue to select the
3138 /// appropriate value within the aggregate.
3139 SmallVector<unsigned, 4> Indices;
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003140
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003141 /// The indices to a GEP instruction which will move Ptr to the correct slot
3142 /// within the aggregate.
3143 SmallVector<Value *, 4> GEPIndices;
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003144
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003145 /// The base pointer of the original op, used as a base for GEPing the
3146 /// split operations.
3147 Value *Ptr;
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003148
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003149 /// Initialize the splitter with an insertion point, Ptr and start with a
3150 /// single zero GEP index.
3151 OpSplitter(Instruction *InsertionPoint, Value *Ptr)
Chandler Carruth113dc642014-12-20 02:39:18 +00003152 : IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr) {}
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003153
3154 public:
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003155 /// Generic recursive split emission routine.
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003156 ///
3157 /// This method recursively splits an aggregate op (load or store) into
3158 /// scalar or vector ops. It splits recursively until it hits a single value
3159 /// and emits that single value operation via the template argument.
3160 ///
3161 /// The logic of this routine relies on GEPs and insertvalue and
3162 /// extractvalue all operating with the same fundamental index list, merely
3163 /// formatted differently (GEPs need actual values).
3164 ///
3165 /// \param Ty The type being split recursively into smaller ops.
3166 /// \param Agg The aggregate value being built up or stored, depending on
3167 /// whether this is splitting a load or a store respectively.
3168 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
3169 if (Ty->isSingleValueType())
Benjamin Kramer73a9e4a2012-09-18 17:06:32 +00003170 return static_cast<Derived *>(this)->emitFunc(Ty, Agg, Name);
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003171
3172 if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
3173 unsigned OldSize = Indices.size();
3174 (void)OldSize;
3175 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
3176 ++Idx) {
3177 assert(Indices.size() == OldSize && "Did not return to the old size");
3178 Indices.push_back(Idx);
3179 GEPIndices.push_back(IRB.getInt32(Idx));
3180 emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx));
3181 GEPIndices.pop_back();
3182 Indices.pop_back();
3183 }
3184 return;
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003185 }
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003186
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003187 if (StructType *STy = dyn_cast<StructType>(Ty)) {
3188 unsigned OldSize = Indices.size();
3189 (void)OldSize;
3190 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
3191 ++Idx) {
3192 assert(Indices.size() == OldSize && "Did not return to the old size");
3193 Indices.push_back(Idx);
3194 GEPIndices.push_back(IRB.getInt32(Idx));
3195 emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx));
3196 GEPIndices.pop_back();
3197 Indices.pop_back();
3198 }
3199 return;
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003200 }
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003201
3202 llvm_unreachable("Only arrays and structs are aggregate loadable types");
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003203 }
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003204 };
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003205
Benjamin Kramer73a9e4a2012-09-18 17:06:32 +00003206 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00003207 AAMDNodes AATags;
3208
3209 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, AAMDNodes AATags)
3210 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr), AATags(AATags) {}
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003211
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003212 /// Emit a leaf load of a single value. This is called at the leaves of the
3213 /// recursive emission to actually load values.
Benjamin Kramer73a9e4a2012-09-18 17:06:32 +00003214 void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) {
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003215 assert(Ty->isSingleValueType());
3216 // Load the single value and insert it using the indices.
David Blaikieaa41cd52015-04-03 21:33:42 +00003217 Value *GEP =
3218 IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep");
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00003219 LoadInst *Load = IRB.CreateLoad(GEP, Name + ".load");
3220 if (AATags)
3221 Load->setAAMetadata(AATags);
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003222 Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003223 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003224 }
3225 };
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003226
3227 bool visitLoadInst(LoadInst &LI) {
3228 assert(LI.getPointerOperand() == *U);
3229 if (!LI.isSimple() || LI.getType()->isSingleValueType())
3230 return false;
3231
3232 // We have an aggregate being loaded, split it apart.
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003233 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00003234 AAMDNodes AATags;
3235 LI.getAAMetadata(AATags);
3236 LoadOpSplitter Splitter(&LI, *U, AATags);
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003237 Value *V = UndefValue::get(LI.getType());
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003238 Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca");
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003239 LI.replaceAllUsesWith(V);
3240 LI.eraseFromParent();
3241 return true;
3242 }
3243
Benjamin Kramer73a9e4a2012-09-18 17:06:32 +00003244 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00003245 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, AAMDNodes AATags)
3246 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr), AATags(AATags) {}
3247 AAMDNodes AATags;
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003248
3249 /// Emit a leaf store of a single value. This is called at the leaves of the
3250 /// recursive emission to actually produce stores.
Benjamin Kramer73a9e4a2012-09-18 17:06:32 +00003251 void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) {
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003252 assert(Ty->isSingleValueType());
3253 // Extract the single value and store it using the indices.
Patrik Hagglunda83706e2016-06-20 10:19:00 +00003254 //
3255 // The gep and extractvalue values are factored out of the CreateStore
3256 // call to make the output independent of the argument evaluation order.
Patrik Hagglund4e0bd842016-06-20 11:19:58 +00003257 Value *ExtractValue =
3258 IRB.CreateExtractValue(Agg, Indices, Name + ".extract");
3259 Value *InBoundsGEP =
3260 IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep");
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00003261 StoreInst *Store = IRB.CreateStore(ExtractValue, InBoundsGEP);
3262 if (AATags)
3263 Store->setAAMetadata(AATags);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003264 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003265 }
3266 };
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003267
3268 bool visitStoreInst(StoreInst &SI) {
3269 if (!SI.isSimple() || SI.getPointerOperand() != *U)
3270 return false;
3271 Value *V = SI.getValueOperand();
3272 if (V->getType()->isSingleValueType())
3273 return false;
3274
3275 // We have an aggregate being stored, split it apart.
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003276 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
Ivan A. Kosarev53270d02018-02-16 10:10:29 +00003277 AAMDNodes AATags;
3278 SI.getAAMetadata(AATags);
3279 StoreOpSplitter Splitter(&SI, *U, AATags);
Benjamin Kramer65f8c882012-09-18 16:20:46 +00003280 Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003281 SI.eraseFromParent();
3282 return true;
3283 }
3284
3285 bool visitBitCastInst(BitCastInst &BC) {
3286 enqueueUsers(BC);
3287 return false;
3288 }
3289
3290 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
3291 enqueueUsers(GEPI);
3292 return false;
3293 }
3294
3295 bool visitPHINode(PHINode &PN) {
3296 enqueueUsers(PN);
3297 return false;
3298 }
3299
3300 bool visitSelectInst(SelectInst &SI) {
3301 enqueueUsers(SI);
3302 return false;
3303 }
3304};
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003305
3306} // end anonymous namespace
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00003307
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003308/// Strip aggregate type wrapping.
Chandler Carruthba931992012-10-13 10:49:33 +00003309///
3310/// This removes no-op aggregate types wrapping an underlying type. It will
3311/// strip as many layers of types as it can without changing either the type
3312/// size or the allocated size.
3313static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
3314 if (Ty->isSingleValueType())
3315 return Ty;
3316
3317 uint64_t AllocSize = DL.getTypeAllocSize(Ty);
3318 uint64_t TypeSize = DL.getTypeSizeInBits(Ty);
3319
3320 Type *InnerTy;
3321 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
3322 InnerTy = ArrTy->getElementType();
3323 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
3324 const StructLayout *SL = DL.getStructLayout(STy);
3325 unsigned Index = SL->getElementContainingOffset(0);
3326 InnerTy = STy->getElementType(Index);
3327 } else {
3328 return Ty;
3329 }
3330
3331 if (AllocSize > DL.getTypeAllocSize(InnerTy) ||
3332 TypeSize > DL.getTypeSizeInBits(InnerTy))
3333 return Ty;
3334
3335 return stripAggregateTypeWrapping(DL, InnerTy);
3336}
3337
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003338/// Try to find a partition of the aggregate type passed in for a given
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003339/// offset and size.
3340///
3341/// This recurses through the aggregate type and tries to compute a subtype
3342/// based on the offset and size. When the offset and size span a sub-section
Chandler Carruth054a40a2012-09-14 11:08:31 +00003343/// of an array, it will even compute a new array type for that sub-section,
3344/// and the same for structs.
3345///
3346/// Note that this routine is very strict and tries to find a partition of the
3347/// type which produces the *exact* right offset and size. It is not forgiving
3348/// when the size or offset cause either end of type-based partition to be off.
3349/// Also, this is a best-effort routine. It is reasonable to give up and not
3350/// return a type if necessary.
Chandler Carruth113dc642014-12-20 02:39:18 +00003351static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
3352 uint64_t Size) {
Chandler Carruth90a735d2013-07-19 07:21:28 +00003353 if (Offset == 0 && DL.getTypeAllocSize(Ty) == Size)
3354 return stripAggregateTypeWrapping(DL, Ty);
3355 if (Offset > DL.getTypeAllocSize(Ty) ||
3356 (DL.getTypeAllocSize(Ty) - Offset) < Size)
Craig Topperf40110f2014-04-25 05:29:35 +00003357 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003358
3359 if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003360 Type *ElementTy = SeqTy->getElementType();
Chandler Carruth90a735d2013-07-19 07:21:28 +00003361 uint64_t ElementSize = DL.getTypeAllocSize(ElementTy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003362 uint64_t NumSkippedElements = Offset / ElementSize;
Peter Collingbournebc070522016-12-02 03:20:58 +00003363 if (NumSkippedElements >= SeqTy->getNumElements())
3364 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003365 Offset -= NumSkippedElements * ElementSize;
3366
3367 // First check if we need to recurse.
3368 if (Offset > 0 || Size < ElementSize) {
3369 // Bail if the partition ends in a different array element.
3370 if ((Offset + Size) > ElementSize)
Craig Topperf40110f2014-04-25 05:29:35 +00003371 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003372 // Recurse through the element type trying to peel off offset bytes.
Chandler Carruth90a735d2013-07-19 07:21:28 +00003373 return getTypePartition(DL, ElementTy, Offset, Size);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003374 }
3375 assert(Offset == 0);
3376
3377 if (Size == ElementSize)
Chandler Carruth90a735d2013-07-19 07:21:28 +00003378 return stripAggregateTypeWrapping(DL, ElementTy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003379 assert(Size > ElementSize);
3380 uint64_t NumElements = Size / ElementSize;
3381 if (NumElements * ElementSize != Size)
Craig Topperf40110f2014-04-25 05:29:35 +00003382 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003383 return ArrayType::get(ElementTy, NumElements);
3384 }
3385
3386 StructType *STy = dyn_cast<StructType>(Ty);
3387 if (!STy)
Craig Topperf40110f2014-04-25 05:29:35 +00003388 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003389
Chandler Carruth90a735d2013-07-19 07:21:28 +00003390 const StructLayout *SL = DL.getStructLayout(STy);
Chandler Carruth054a40a2012-09-14 11:08:31 +00003391 if (Offset >= SL->getSizeInBytes())
Craig Topperf40110f2014-04-25 05:29:35 +00003392 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003393 uint64_t EndOffset = Offset + Size;
3394 if (EndOffset > SL->getSizeInBytes())
Craig Topperf40110f2014-04-25 05:29:35 +00003395 return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003396
3397 unsigned Index = SL->getElementContainingOffset(Offset);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003398 Offset -= SL->getElementOffset(Index);
3399
3400 Type *ElementTy = STy->getElementType(Index);
Chandler Carruth90a735d2013-07-19 07:21:28 +00003401 uint64_t ElementSize = DL.getTypeAllocSize(ElementTy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003402 if (Offset >= ElementSize)
Craig Topperf40110f2014-04-25 05:29:35 +00003403 return nullptr; // The offset points into alignment padding.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003404
3405 // See if any partition must be contained by the element.
3406 if (Offset > 0 || Size < ElementSize) {
3407 if ((Offset + Size) > ElementSize)
Craig Topperf40110f2014-04-25 05:29:35 +00003408 return nullptr;
Chandler Carruth90a735d2013-07-19 07:21:28 +00003409 return getTypePartition(DL, ElementTy, Offset, Size);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003410 }
3411 assert(Offset == 0);
3412
3413 if (Size == ElementSize)
Chandler Carruth90a735d2013-07-19 07:21:28 +00003414 return stripAggregateTypeWrapping(DL, ElementTy);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003415
3416 StructType::element_iterator EI = STy->element_begin() + Index,
3417 EE = STy->element_end();
3418 if (EndOffset < SL->getSizeInBytes()) {
3419 unsigned EndIndex = SL->getElementContainingOffset(EndOffset);
3420 if (Index == EndIndex)
Craig Topperf40110f2014-04-25 05:29:35 +00003421 return nullptr; // Within a single element and its padding.
Chandler Carruth054a40a2012-09-14 11:08:31 +00003422
3423 // Don't try to form "natural" types if the elements don't line up with the
3424 // expected size.
3425 // FIXME: We could potentially recurse down through the last element in the
3426 // sub-struct to find a natural end point.
3427 if (SL->getElementOffset(EndIndex) != EndOffset)
Craig Topperf40110f2014-04-25 05:29:35 +00003428 return nullptr;
Chandler Carruth054a40a2012-09-14 11:08:31 +00003429
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003430 assert(Index < EndIndex);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003431 EE = STy->element_begin() + EndIndex;
3432 }
3433
3434 // Try to build up a sub-structure.
Chandler Carruth113dc642014-12-20 02:39:18 +00003435 StructType *SubTy =
3436 StructType::get(STy->getContext(), makeArrayRef(EI, EE), STy->isPacked());
Chandler Carruth90a735d2013-07-19 07:21:28 +00003437 const StructLayout *SubSL = DL.getStructLayout(SubTy);
Chandler Carruth054a40a2012-09-14 11:08:31 +00003438 if (Size != SubSL->getSizeInBytes())
Craig Topperf40110f2014-04-25 05:29:35 +00003439 return nullptr; // The sub-struct doesn't have quite the size needed.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003440
Chandler Carruth054a40a2012-09-14 11:08:31 +00003441 return SubTy;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003442}
3443
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003444/// Pre-split loads and stores to simplify rewriting.
Chandler Carruth0715cba2015-01-01 11:54:38 +00003445///
3446/// We want to break up the splittable load+store pairs as much as
3447/// possible. This is important to do as a preprocessing step, as once we
3448/// start rewriting the accesses to partitions of the alloca we lose the
3449/// necessary information to correctly split apart paired loads and stores
3450/// which both point into this alloca. The case to consider is something like
3451/// the following:
3452///
3453/// %a = alloca [12 x i8]
3454/// %gep1 = getelementptr [12 x i8]* %a, i32 0, i32 0
3455/// %gep2 = getelementptr [12 x i8]* %a, i32 0, i32 4
3456/// %gep3 = getelementptr [12 x i8]* %a, i32 0, i32 8
3457/// %iptr1 = bitcast i8* %gep1 to i64*
3458/// %iptr2 = bitcast i8* %gep2 to i64*
3459/// %fptr1 = bitcast i8* %gep1 to float*
3460/// %fptr2 = bitcast i8* %gep2 to float*
3461/// %fptr3 = bitcast i8* %gep3 to float*
3462/// store float 0.0, float* %fptr1
3463/// store float 1.0, float* %fptr2
3464/// %v = load i64* %iptr1
3465/// store i64 %v, i64* %iptr2
3466/// %f1 = load float* %fptr2
3467/// %f2 = load float* %fptr3
3468///
3469/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
3470/// promote everything so we recover the 2 SSA values that should have been
3471/// there all along.
3472///
3473/// \returns true if any changes are made.
3474bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003475 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003476
3477 // Track the loads and stores which are candidates for pre-splitting here, in
3478 // the order they first appear during the partition scan. These give stable
3479 // iteration order and a basis for tracking which loads and stores we
3480 // actually split.
3481 SmallVector<LoadInst *, 4> Loads;
3482 SmallVector<StoreInst *, 4> Stores;
3483
3484 // We need to accumulate the splits required of each load or store where we
3485 // can find them via a direct lookup. This is important to cross-check loads
3486 // and stores against each other. We also track the slice so that we can kill
3487 // all the slices that end up split.
3488 struct SplitOffsets {
3489 Slice *S;
3490 std::vector<uint64_t> Splits;
3491 };
3492 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
3493
Chandler Carruth73b01642015-01-05 04:17:53 +00003494 // Track loads out of this alloca which cannot, for any reason, be pre-split.
3495 // This is important as we also cannot pre-split stores of those loads!
3496 // FIXME: This is all pretty gross. It means that we can be more aggressive
3497 // in pre-splitting when the load feeding the store happens to come from
3498 // a separate alloca. Put another way, the effectiveness of SROA would be
3499 // decreased by a frontend which just concatenated all of its local allocas
3500 // into one big flat alloca. But defeating such patterns is exactly the job
3501 // SROA is tasked with! Sadly, to not have this discrepancy we would have
3502 // change store pre-splitting to actually force pre-splitting of the load
3503 // that feeds it *and all stores*. That makes pre-splitting much harder, but
3504 // maybe it would make it more principled?
3505 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
3506
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003507 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003508 for (auto &P : AS.partitions()) {
3509 for (Slice &S : P) {
Chandler Carruth73b01642015-01-05 04:17:53 +00003510 Instruction *I = cast<Instruction>(S.getUse()->getUser());
Chandler Carruth37f1f122016-03-10 15:31:17 +00003511 if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
3512 // If this is a load we have to track that it can't participate in any
3513 // pre-splitting. If this is a store of a load we have to track that
3514 // that load also can't participate in any pre-splitting.
Chandler Carruth73b01642015-01-05 04:17:53 +00003515 if (auto *LI = dyn_cast<LoadInst>(I))
3516 UnsplittableLoads.insert(LI);
Chandler Carruth37f1f122016-03-10 15:31:17 +00003517 else if (auto *SI = dyn_cast<StoreInst>(I))
3518 if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand()))
3519 UnsplittableLoads.insert(LI);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003520 continue;
Chandler Carruth73b01642015-01-05 04:17:53 +00003521 }
Chandler Carruth0715cba2015-01-01 11:54:38 +00003522 assert(P.endOffset() > S.beginOffset() &&
3523 "Empty or backwards partition!");
3524
3525 // Determine if this is a pre-splittable slice.
Chandler Carruth0715cba2015-01-01 11:54:38 +00003526 if (auto *LI = dyn_cast<LoadInst>(I)) {
3527 assert(!LI->isVolatile() && "Cannot split volatile loads!");
3528
3529 // The load must be used exclusively to store into other pointers for
3530 // us to be able to arbitrarily pre-split it. The stores must also be
3531 // simple to avoid changing semantics.
3532 auto IsLoadSimplyStored = [](LoadInst *LI) {
3533 for (User *LU : LI->users()) {
3534 auto *SI = dyn_cast<StoreInst>(LU);
3535 if (!SI || !SI->isSimple())
3536 return false;
3537 }
3538 return true;
3539 };
Chandler Carruth73b01642015-01-05 04:17:53 +00003540 if (!IsLoadSimplyStored(LI)) {
3541 UnsplittableLoads.insert(LI);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003542 continue;
Chandler Carruth73b01642015-01-05 04:17:53 +00003543 }
Chandler Carruth0715cba2015-01-01 11:54:38 +00003544
3545 Loads.push_back(LI);
Chandler Carruthd94a5962016-03-10 14:16:18 +00003546 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
3547 if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex()))
3548 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
Chandler Carruth994cde82015-01-01 12:01:03 +00003549 continue;
3550 auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand());
3551 if (!StoredLoad || !StoredLoad->isSimple())
3552 continue;
3553 assert(!SI->isVolatile() && "Cannot split volatile stores!");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003554
Chandler Carruth994cde82015-01-01 12:01:03 +00003555 Stores.push_back(SI);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003556 } else {
3557 // Other uses cannot be pre-split.
3558 continue;
3559 }
3560
3561 // Record the initial split.
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003562 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003563 auto &Offsets = SplitOffsetsMap[I];
3564 assert(Offsets.Splits.empty() &&
3565 "Should not have splits the first time we see an instruction!");
3566 Offsets.S = &S;
Chandler Carruth24ac8302015-01-02 03:55:54 +00003567 Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
Chandler Carruth0715cba2015-01-01 11:54:38 +00003568 }
3569
3570 // Now scan the already split slices, and add a split for any of them which
3571 // we're going to pre-split.
3572 for (Slice *S : P.splitSliceTails()) {
3573 auto SplitOffsetsMapI =
3574 SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser()));
3575 if (SplitOffsetsMapI == SplitOffsetsMap.end())
3576 continue;
3577 auto &Offsets = SplitOffsetsMapI->second;
3578
3579 assert(Offsets.S == S && "Found a mismatched slice!");
3580 assert(!Offsets.Splits.empty() &&
3581 "Cannot have an empty set of splits on the second partition!");
Chandler Carruth24ac8302015-01-02 03:55:54 +00003582 assert(Offsets.Splits.back() ==
3583 P.beginOffset() - Offsets.S->beginOffset() &&
Chandler Carruth0715cba2015-01-01 11:54:38 +00003584 "Previous split does not end where this one begins!");
3585
3586 // Record each split. The last partition's end isn't needed as the size
3587 // of the slice dictates that.
3588 if (S->endOffset() > P.endOffset())
Chandler Carruth24ac8302015-01-02 03:55:54 +00003589 Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset());
Chandler Carruth0715cba2015-01-01 11:54:38 +00003590 }
3591 }
3592
3593 // We may have split loads where some of their stores are split stores. For
3594 // such loads and stores, we can only pre-split them if their splits exactly
3595 // match relative to their starting offset. We have to verify this prior to
3596 // any rewriting.
Chandler Carruth0715cba2015-01-01 11:54:38 +00003597 Stores.erase(
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003598 llvm::remove_if(Stores,
3599 [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
3600 // Lookup the load we are storing in our map of split
3601 // offsets.
3602 auto *LI = cast<LoadInst>(SI->getValueOperand());
3603 // If it was completely unsplittable, then we're done,
3604 // and this store can't be pre-split.
3605 if (UnsplittableLoads.count(LI))
3606 return true;
Chandler Carruth73b01642015-01-05 04:17:53 +00003607
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003608 auto LoadOffsetsI = SplitOffsetsMap.find(LI);
3609 if (LoadOffsetsI == SplitOffsetsMap.end())
3610 return false; // Unrelated loads are definitely safe.
3611 auto &LoadOffsets = LoadOffsetsI->second;
Chandler Carruth0715cba2015-01-01 11:54:38 +00003612
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003613 // Now lookup the store's offsets.
3614 auto &StoreOffsets = SplitOffsetsMap[SI];
Chandler Carruth0715cba2015-01-01 11:54:38 +00003615
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003616 // If the relative offsets of each split in the load and
3617 // store match exactly, then we can split them and we
3618 // don't need to remove them here.
3619 if (LoadOffsets.Splits == StoreOffsets.Splits)
3620 return false;
Chandler Carruth0715cba2015-01-01 11:54:38 +00003621
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003622 LLVM_DEBUG(
3623 dbgs()
3624 << " Mismatched splits for load and store:\n"
3625 << " " << *LI << "\n"
3626 << " " << *SI << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003627
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003628 // We've found a store and load that we need to split
3629 // with mismatched relative splits. Just give up on them
3630 // and remove both instructions from our list of
3631 // candidates.
3632 UnsplittableLoads.insert(LI);
3633 return true;
3634 }),
Chandler Carruth0715cba2015-01-01 11:54:38 +00003635 Stores.end());
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00003636 // Now we have to go *back* through all the stores, because a later store may
Chandler Carruth73b01642015-01-05 04:17:53 +00003637 // have caused an earlier store's load to become unsplittable and if it is
3638 // unsplittable for the later store, then we can't rely on it being split in
3639 // the earlier store either.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003640 Stores.erase(llvm::remove_if(Stores,
3641 [&UnsplittableLoads](StoreInst *SI) {
3642 auto *LI =
3643 cast<LoadInst>(SI->getValueOperand());
3644 return UnsplittableLoads.count(LI);
3645 }),
Chandler Carruth73b01642015-01-05 04:17:53 +00003646 Stores.end());
3647 // Once we've established all the loads that can't be split for some reason,
3648 // filter any that made it into our list out.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003649 Loads.erase(llvm::remove_if(Loads,
3650 [&UnsplittableLoads](LoadInst *LI) {
3651 return UnsplittableLoads.count(LI);
3652 }),
Chandler Carruth0715cba2015-01-01 11:54:38 +00003653 Loads.end());
3654
Chandler Carruth0715cba2015-01-01 11:54:38 +00003655 // If no loads or stores are left, there is no pre-splitting to be done for
3656 // this alloca.
3657 if (Loads.empty() && Stores.empty())
3658 return false;
3659
3660 // From here on, we can't fail and will be building new accesses, so rig up
3661 // an IR builder.
3662 IRBuilderTy IRB(&AI);
3663
3664 // Collect the new slices which we will merge into the alloca slices.
3665 SmallVector<Slice, 4> NewSlices;
3666
3667 // Track any allocas we end up splitting loads and stores for so we iterate
3668 // on them.
3669 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
3670
3671 // At this point, we have collected all of the loads and stores we can
3672 // pre-split, and the specific splits needed for them. We actually do the
3673 // splitting in a specific order in order to handle when one of the loads in
3674 // the value operand to one of the stores.
3675 //
3676 // First, we rewrite all of the split loads, and just accumulate each split
3677 // load in a parallel structure. We also build the slices for them and append
3678 // them to the alloca slices.
3679 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
3680 std::vector<LoadInst *> SplitLoads;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003681 const DataLayout &DL = AI.getModule()->getDataLayout();
Chandler Carruth0715cba2015-01-01 11:54:38 +00003682 for (LoadInst *LI : Loads) {
3683 SplitLoads.clear();
3684
3685 IntegerType *Ty = cast<IntegerType>(LI->getType());
3686 uint64_t LoadSize = Ty->getBitWidth() / 8;
3687 assert(LoadSize > 0 && "Cannot have a zero-sized integer load!");
3688
3689 auto &Offsets = SplitOffsetsMap[LI];
3690 assert(LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
3691 "Slice size should always match load size exactly!");
3692 uint64_t BaseOffset = Offsets.S->beginOffset();
3693 assert(BaseOffset + LoadSize > BaseOffset &&
3694 "Cannot represent alloca access size using 64-bit integers!");
3695
3696 Instruction *BasePtr = cast<Instruction>(LI->getPointerOperand());
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00003697 IRB.SetInsertPoint(LI);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003698
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003699 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003700
3701 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
3702 int Idx = 0, Size = Offsets.Splits.size();
3703 for (;;) {
3704 auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8);
Yaxun Liu7c44f342017-06-27 18:26:06 +00003705 auto AS = LI->getPointerAddressSpace();
3706 auto *PartPtrTy = PartTy->getPointerTo(AS);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003707 LoadInst *PLoad = IRB.CreateAlignedLoad(
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003708 getAdjustedPtr(IRB, DL, BasePtr,
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00003709 APInt(DL.getIndexSizeInBits(AS), PartOffset),
Chandler Carruth994cde82015-01-01 12:01:03 +00003710 PartPtrTy, BasePtr->getName() + "."),
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003711 getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false,
Chandler Carruth0715cba2015-01-01 11:54:38 +00003712 LI->getName());
Rafael Espindolac06f55e2017-11-28 01:25:38 +00003713 PLoad->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003714
3715 // Append this load onto the list of split loads so we can find it later
3716 // to rewrite the stores.
3717 SplitLoads.push_back(PLoad);
3718
3719 // Now build a new slice for the alloca.
Chandler Carruth994cde82015-01-01 12:01:03 +00003720 NewSlices.push_back(
3721 Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
3722 &PLoad->getOperandUse(PLoad->getPointerOperandIndex()),
Chandler Carruth24ac8302015-01-02 03:55:54 +00003723 /*IsSplittable*/ false));
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003724 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
3725 << ", " << NewSlices.back().endOffset()
3726 << "): " << *PLoad << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003727
Chandler Carruth29c22fa2015-01-02 00:10:22 +00003728 // See if we've handled all the splits.
3729 if (Idx >= Size)
3730 break;
3731
Chandler Carruth0715cba2015-01-01 11:54:38 +00003732 // Setup the next partition.
3733 PartOffset = Offsets.Splits[Idx];
3734 ++Idx;
Chandler Carruth0715cba2015-01-01 11:54:38 +00003735 PartSize = (Idx < Size ? Offsets.Splits[Idx] : LoadSize) - PartOffset;
3736 }
3737
3738 // Now that we have the split loads, do the slow walk over all uses of the
3739 // load and rewrite them as split stores, or save the split loads to use
3740 // below if the store is going to be split there anyways.
3741 bool DeferredStores = false;
3742 for (User *LU : LI->users()) {
3743 StoreInst *SI = cast<StoreInst>(LU);
3744 if (!Stores.empty() && SplitOffsetsMap.count(SI)) {
3745 DeferredStores = true;
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003746 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
3747 << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003748 continue;
3749 }
3750
Chandler Carruthc39eaa52015-01-01 23:26:16 +00003751 Value *StoreBasePtr = SI->getPointerOperand();
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00003752 IRB.SetInsertPoint(SI);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003753
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003754 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003755
3756 for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) {
3757 LoadInst *PLoad = SplitLoads[Idx];
3758 uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1];
Chandler Carruth994cde82015-01-01 12:01:03 +00003759 auto *PartPtrTy =
3760 PLoad->getType()->getPointerTo(SI->getPointerAddressSpace());
Chandler Carruth0715cba2015-01-01 11:54:38 +00003761
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003762 auto AS = SI->getPointerAddressSpace();
Chandler Carruth0715cba2015-01-01 11:54:38 +00003763 StoreInst *PStore = IRB.CreateAlignedStore(
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003764 PLoad,
3765 getAdjustedPtr(IRB, DL, StoreBasePtr,
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00003766 APInt(DL.getIndexSizeInBits(AS), PartOffset),
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003767 PartPtrTy, StoreBasePtr->getName() + "."),
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003768 getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false);
Dorit Nuzmand1247a62016-09-22 07:56:23 +00003769 PStore->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003770 LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003771 }
3772
3773 // We want to immediately iterate on any allocas impacted by splitting
3774 // this store, and we have to track any promotable alloca (indicated by
3775 // a direct store) as needing to be resplit because it is no longer
3776 // promotable.
3777 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) {
3778 ResplitPromotableAllocas.insert(OtherAI);
3779 Worklist.insert(OtherAI);
3780 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
3781 StoreBasePtr->stripInBoundsOffsets())) {
3782 Worklist.insert(OtherAI);
3783 }
3784
3785 // Mark the original store as dead.
3786 DeadInsts.insert(SI);
3787 }
3788
3789 // Save the split loads if there are deferred stores among the users.
3790 if (DeferredStores)
3791 SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads)));
3792
3793 // Mark the original load as dead and kill the original slice.
3794 DeadInsts.insert(LI);
3795 Offsets.S->kill();
3796 }
3797
3798 // Second, we rewrite all of the split stores. At this point, we know that
3799 // all loads from this alloca have been split already. For stores of such
3800 // loads, we can simply look up the pre-existing split loads. For stores of
3801 // other loads, we split those loads first and then write split stores of
3802 // them.
3803 for (StoreInst *SI : Stores) {
3804 auto *LI = cast<LoadInst>(SI->getValueOperand());
3805 IntegerType *Ty = cast<IntegerType>(LI->getType());
3806 uint64_t StoreSize = Ty->getBitWidth() / 8;
3807 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
3808
3809 auto &Offsets = SplitOffsetsMap[SI];
3810 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
3811 "Slice size should always match load size exactly!");
3812 uint64_t BaseOffset = Offsets.S->beginOffset();
3813 assert(BaseOffset + StoreSize > BaseOffset &&
3814 "Cannot represent alloca access size using 64-bit integers!");
3815
Chandler Carruthc39eaa52015-01-01 23:26:16 +00003816 Value *LoadBasePtr = LI->getPointerOperand();
Chandler Carruth0715cba2015-01-01 11:54:38 +00003817 Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand());
3818
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003819 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003820
3821 // Check whether we have an already split load.
3822 auto SplitLoadsMapI = SplitLoadsMap.find(LI);
3823 std::vector<LoadInst *> *SplitLoads = nullptr;
3824 if (SplitLoadsMapI != SplitLoadsMap.end()) {
3825 SplitLoads = &SplitLoadsMapI->second;
3826 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
3827 "Too few split loads for the number of splits in the store!");
3828 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003829 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003830 }
3831
Chandler Carruth0715cba2015-01-01 11:54:38 +00003832 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
3833 int Idx = 0, Size = Offsets.Splits.size();
3834 for (;;) {
3835 auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8);
Keno Fischer514a6a52017-06-02 19:04:17 +00003836 auto *LoadPartPtrTy = PartTy->getPointerTo(LI->getPointerAddressSpace());
3837 auto *StorePartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace());
Chandler Carruth0715cba2015-01-01 11:54:38 +00003838
3839 // Either lookup a split load or create one.
3840 LoadInst *PLoad;
3841 if (SplitLoads) {
3842 PLoad = (*SplitLoads)[Idx];
3843 } else {
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00003844 IRB.SetInsertPoint(LI);
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003845 auto AS = LI->getPointerAddressSpace();
Chandler Carruth0715cba2015-01-01 11:54:38 +00003846 PLoad = IRB.CreateAlignedLoad(
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003847 getAdjustedPtr(IRB, DL, LoadBasePtr,
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00003848 APInt(DL.getIndexSizeInBits(AS), PartOffset),
Keno Fischer514a6a52017-06-02 19:04:17 +00003849 LoadPartPtrTy, LoadBasePtr->getName() + "."),
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003850 getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false,
Chandler Carruth0715cba2015-01-01 11:54:38 +00003851 LI->getName());
3852 }
3853
3854 // And store this partition.
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00003855 IRB.SetInsertPoint(SI);
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003856 auto AS = SI->getPointerAddressSpace();
Chandler Carruth0715cba2015-01-01 11:54:38 +00003857 StoreInst *PStore = IRB.CreateAlignedStore(
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003858 PLoad,
3859 getAdjustedPtr(IRB, DL, StoreBasePtr,
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00003860 APInt(DL.getIndexSizeInBits(AS), PartOffset),
Yaxun Liu6455b0d2017-06-09 20:46:29 +00003861 StorePartPtrTy, StoreBasePtr->getName() + "."),
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003862 getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false);
Chandler Carruth0715cba2015-01-01 11:54:38 +00003863
3864 // Now build a new slice for the alloca.
3865 NewSlices.push_back(
3866 Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
3867 &PStore->getOperandUse(PStore->getPointerOperandIndex()),
Chandler Carruth24ac8302015-01-02 03:55:54 +00003868 /*IsSplittable*/ false));
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003869 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
3870 << ", " << NewSlices.back().endOffset()
3871 << "): " << *PStore << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003872 if (!SplitLoads) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003873 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003874 }
3875
Chandler Carruth29c22fa2015-01-02 00:10:22 +00003876 // See if we've finished all the splits.
3877 if (Idx >= Size)
3878 break;
3879
Chandler Carruth0715cba2015-01-01 11:54:38 +00003880 // Setup the next partition.
3881 PartOffset = Offsets.Splits[Idx];
3882 ++Idx;
Chandler Carruth0715cba2015-01-01 11:54:38 +00003883 PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset;
3884 }
3885
3886 // We want to immediately iterate on any allocas impacted by splitting
3887 // this load, which is only relevant if it isn't a load of this alloca and
3888 // thus we didn't already split the loads above. We also have to keep track
3889 // of any promotable allocas we split loads on as they can no longer be
3890 // promoted.
3891 if (!SplitLoads) {
3892 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) {
3893 assert(OtherAI != &AI && "We can't re-split our own alloca!");
3894 ResplitPromotableAllocas.insert(OtherAI);
3895 Worklist.insert(OtherAI);
3896 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
3897 LoadBasePtr->stripInBoundsOffsets())) {
3898 assert(OtherAI != &AI && "We can't re-split our own alloca!");
3899 Worklist.insert(OtherAI);
3900 }
3901 }
3902
3903 // Mark the original store as dead now that we've split it up and kill its
Chandler Carruth24ac8302015-01-02 03:55:54 +00003904 // slice. Note that we leave the original load in place unless this store
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00003905 // was its only use. It may in turn be split up if it is an alloca load
Chandler Carruth24ac8302015-01-02 03:55:54 +00003906 // for some other alloca, but it may be a normal load. This may introduce
3907 // redundant loads, but where those can be merged the rest of the optimizer
3908 // should handle the merging, and this uncovers SSA splits which is more
3909 // important. In practice, the original loads will almost always be fully
3910 // split and removed eventually, and the splits will be merged by any
3911 // trivial CSE, including instcombine.
3912 if (LI->hasOneUse()) {
3913 assert(*LI->user_begin() == SI && "Single use isn't this store!");
3914 DeadInsts.insert(LI);
3915 }
Chandler Carruth0715cba2015-01-01 11:54:38 +00003916 DeadInsts.insert(SI);
3917 Offsets.S->kill();
3918 }
3919
Chandler Carruth24ac8302015-01-02 03:55:54 +00003920 // Remove the killed slices that have ben pre-split.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003921 AS.erase(llvm::remove_if(AS, [](const Slice &S) { return S.isDead(); }),
3922 AS.end());
Chandler Carruth0715cba2015-01-01 11:54:38 +00003923
Chandler Carruth24ac8302015-01-02 03:55:54 +00003924 // Insert our new slices. This will sort and merge them into the sorted
3925 // sequence.
Chandler Carruth0715cba2015-01-01 11:54:38 +00003926 AS.insert(NewSlices);
3927
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003928 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
Chandler Carruth0715cba2015-01-01 11:54:38 +00003929#ifndef NDEBUG
3930 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
Nicola Zaghend34e60c2018-05-14 12:53:11 +00003931 LLVM_DEBUG(AS.print(dbgs(), I, " "));
Chandler Carruth0715cba2015-01-01 11:54:38 +00003932#endif
3933
3934 // Finally, don't try to promote any allocas that new require re-splitting.
3935 // They have already been added to the worklist above.
3936 PromotableAllocas.erase(
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003937 llvm::remove_if(
David Majnemerc7004902016-08-12 04:32:37 +00003938 PromotableAllocas,
Chandler Carruth0715cba2015-01-01 11:54:38 +00003939 [&](AllocaInst *AI) { return ResplitPromotableAllocas.count(AI); }),
3940 PromotableAllocas.end());
3941
3942 return true;
3943}
3944
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003945/// Rewrite an alloca partition's users.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003946///
3947/// This routine drives both of the rewriting goals of the SROA pass. It tries
3948/// to rewrite uses of an alloca partition to be conducive for SSA value
3949/// promotion. If the partition needs a new, more refined alloca, this will
3950/// build that new alloca, preserving as much type information as possible, and
3951/// rewrite the uses of the old alloca to point at the new one and have the
3952/// appropriate new offsets. It also evaluates how successful the rewrite was
3953/// at enabling promotion and if it was successful queues the alloca to be
3954/// promoted.
Adrian Prantl565cc182015-01-20 19:42:22 +00003955AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
Chandler Carruth29a18a42015-09-12 09:09:14 +00003956 Partition &P) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003957 // Try to compute a friendly type for this partition of the alloca. This
3958 // won't always succeed, in which case we fall back to a legal integer type
3959 // or an i8 array of an appropriate size.
Craig Topperf40110f2014-04-25 05:29:35 +00003960 Type *SliceTy = nullptr;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003961 const DataLayout &DL = AI.getModule()->getDataLayout();
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00003962 if (Type *CommonUseTy = findCommonType(P.begin(), P.end(), P.endOffset()))
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003963 if (DL.getTypeAllocSize(CommonUseTy) >= P.size())
Chandler Carruth9f21fe12013-07-19 09:13:58 +00003964 SliceTy = CommonUseTy;
3965 if (!SliceTy)
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003966 if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(),
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00003967 P.beginOffset(), P.size()))
Chandler Carruth9f21fe12013-07-19 09:13:58 +00003968 SliceTy = TypePartitionTy;
3969 if ((!SliceTy || (SliceTy->isArrayTy() &&
3970 SliceTy->getArrayElementType()->isIntegerTy())) &&
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003971 DL.isLegalInteger(P.size() * 8))
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00003972 SliceTy = Type::getIntNTy(*C, P.size() * 8);
Chandler Carruth9f21fe12013-07-19 09:13:58 +00003973 if (!SliceTy)
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00003974 SliceTy = ArrayType::get(Type::getInt8Ty(*C), P.size());
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003975 assert(DL.getTypeAllocSize(SliceTy) >= P.size());
Chandler Carruthf0546402013-07-18 07:15:00 +00003976
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003977 bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL);
Chandler Carruthf0546402013-07-18 07:15:00 +00003978
Chandler Carruth2dc96822014-10-18 00:44:02 +00003979 VectorType *VecTy =
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003980 IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL);
Chandler Carruth2dc96822014-10-18 00:44:02 +00003981 if (VecTy)
3982 SliceTy = VecTy;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003983
3984 // Check for the case where we're going to rewrite to a new alloca of the
3985 // exact same type as the original, and with the same access offsets. In that
3986 // case, re-use the existing alloca, but still run through the rewriter to
Jakub Staszak086f6cd2013-02-19 22:02:21 +00003987 // perform phi and select speculation.
Hiroshi Inoue99a8faa2018-01-16 06:23:05 +00003988 // P.beginOffset() can be non-zero even with the same type in a case with
3989 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003990 AllocaInst *NewAI;
Hiroshi Inoue99a8faa2018-01-16 06:23:05 +00003991 if (SliceTy == AI.getAllocatedType() && P.beginOffset() == 0) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003992 NewAI = &AI;
Chandler Carruthf0546402013-07-18 07:15:00 +00003993 // FIXME: We should be able to bail at this point with "nothing changed".
3994 // FIXME: We might want to defer PHI speculation until after here.
Adrian Prantl565cc182015-01-20 19:42:22 +00003995 // FIXME: return nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00003996 } else {
Chandler Carruth903790e2012-09-29 10:41:21 +00003997 unsigned Alignment = AI.getAlignment();
3998 if (!Alignment) {
3999 // The minimum alignment which users can rely on when the explicit
4000 // alignment is omitted or zero is that required by the ABI for this
4001 // type.
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004002 Alignment = DL.getABITypeAlignment(AI.getAllocatedType());
Chandler Carruth903790e2012-09-29 10:41:21 +00004003 }
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004004 Alignment = MinAlign(Alignment, P.beginOffset());
Chandler Carruth903790e2012-09-29 10:41:21 +00004005 // If we will get at least this much alignment from the type alone, leave
4006 // the alloca's alignment unconstrained.
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004007 if (Alignment <= DL.getABITypeAlignment(SliceTy))
Chandler Carruth903790e2012-09-29 10:41:21 +00004008 Alignment = 0;
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004009 NewAI = new AllocaInst(
Matt Arsenault3c1fc762017-04-10 22:27:50 +00004010 SliceTy, AI.getType()->getAddressSpace(), nullptr, Alignment,
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004011 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004012 ++NumNewAllocas;
4013 }
4014
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004015 LLVM_DEBUG(dbgs() << "Rewriting alloca partition "
4016 << "[" << P.beginOffset() << "," << P.endOffset()
4017 << ") to: " << *NewAI << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004018
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004019 // Track the high watermark on the worklist as it is only relevant for
Chandler Carruthf0546402013-07-18 07:15:00 +00004020 // promoted allocas. We will reset it to this point if the alloca is not in
4021 // fact scheduled for promotion.
Chandler Carruthac8317f2012-10-04 12:33:50 +00004022 unsigned PPWOldSize = PostPromotionWorklist.size();
Chandler Carruth6c321c12013-07-19 10:57:36 +00004023 unsigned NumUses = 0;
Davide Italiano81a26da2017-04-27 23:09:01 +00004024 SmallSetVector<PHINode *, 8> PHIUsers;
4025 SmallSetVector<SelectInst *, 8> SelectUsers;
Chandler Carruth6c321c12013-07-19 10:57:36 +00004026
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004027 AllocaSliceRewriter Rewriter(DL, AS, *this, AI, *NewAI, P.beginOffset(),
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004028 P.endOffset(), IsIntegerPromotable, VecTy,
4029 PHIUsers, SelectUsers);
Chandler Carruthf0546402013-07-18 07:15:00 +00004030 bool Promotable = true;
Chandler Carruthffb7ce52014-12-24 01:48:09 +00004031 for (Slice *S : P.splitSliceTails()) {
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004032 Promotable &= Rewriter.visit(S);
Chandler Carruth6c321c12013-07-19 10:57:36 +00004033 ++NumUses;
Chandler Carruthf0546402013-07-18 07:15:00 +00004034 }
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004035 for (Slice &S : P) {
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004036 Promotable &= Rewriter.visit(&S);
Chandler Carruth6c321c12013-07-19 10:57:36 +00004037 ++NumUses;
Chandler Carruthf0546402013-07-18 07:15:00 +00004038 }
4039
Chandler Carruth6c321c12013-07-19 10:57:36 +00004040 NumAllocaPartitionUses += NumUses;
Craig Topper8a950272017-05-18 00:51:39 +00004041 MaxUsesPerAllocaPartition.updateMax(NumUses);
Chandler Carruth6c321c12013-07-19 10:57:36 +00004042
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004043 // Now that we've processed all the slices in the new partition, check if any
4044 // PHIs or Selects would block promotion.
Davide Italiano81a26da2017-04-27 23:09:01 +00004045 for (PHINode *PHI : PHIUsers)
4046 if (!isSafePHIToSpeculate(*PHI)) {
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004047 Promotable = false;
4048 PHIUsers.clear();
4049 SelectUsers.clear();
Chandler Carrutha8c4cc62014-02-25 09:45:27 +00004050 break;
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004051 }
Davide Italiano81a26da2017-04-27 23:09:01 +00004052
4053 for (SelectInst *Sel : SelectUsers)
4054 if (!isSafeSelectToSpeculate(*Sel)) {
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004055 Promotable = false;
4056 PHIUsers.clear();
4057 SelectUsers.clear();
Chandler Carrutha8c4cc62014-02-25 09:45:27 +00004058 break;
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004059 }
4060
4061 if (Promotable) {
4062 if (PHIUsers.empty() && SelectUsers.empty()) {
4063 // Promote the alloca.
4064 PromotableAllocas.push_back(NewAI);
4065 } else {
4066 // If we have either PHIs or Selects to speculate, add them to those
4067 // worklists and re-queue the new alloca so that we promote in on the
4068 // next iteration.
Chandler Carruth61747042014-10-16 21:05:14 +00004069 for (PHINode *PHIUser : PHIUsers)
4070 SpeculatablePHIs.insert(PHIUser);
4071 for (SelectInst *SelectUser : SelectUsers)
4072 SpeculatableSelects.insert(SelectUser);
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004073 Worklist.insert(NewAI);
4074 }
4075 } else {
Chandler Carruth3bf18ed2014-02-25 00:07:09 +00004076 // Drop any post-promotion work items if promotion didn't happen.
Chandler Carruthac8317f2012-10-04 12:33:50 +00004077 while (PostPromotionWorklist.size() > PPWOldSize)
4078 PostPromotionWorklist.pop_back();
David Majnemer30ffc4ce2016-04-26 01:05:00 +00004079
4080 // We couldn't promote and we didn't create a new partition, nothing
4081 // happened.
4082 if (NewAI == &AI)
4083 return nullptr;
4084
4085 // If we can't promote the alloca, iterate on it to check for new
4086 // refinements exposed by splitting the current alloca. Don't iterate on an
4087 // alloca which didn't actually change and didn't get promoted.
4088 Worklist.insert(NewAI);
Chandler Carruthf0546402013-07-18 07:15:00 +00004089 }
Chandler Carruthac8317f2012-10-04 12:33:50 +00004090
Adrian Prantl565cc182015-01-20 19:42:22 +00004091 return NewAI;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004092}
4093
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00004094/// Walks the slices of an alloca and form partitions based on them,
Chandler Carruth9f21fe12013-07-19 09:13:58 +00004095/// rewriting each of their uses.
Chandler Carruth83934062014-10-16 21:11:55 +00004096bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
4097 if (AS.begin() == AS.end())
Chandler Carruthf0546402013-07-18 07:15:00 +00004098 return false;
4099
Chandler Carruth6c321c12013-07-19 10:57:36 +00004100 unsigned NumPartitions = 0;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004101 bool Changed = false;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004102 const DataLayout &DL = AI.getModule()->getDataLayout();
Chandler Carruthf0546402013-07-18 07:15:00 +00004103
Chandler Carruth24ac8302015-01-02 03:55:54 +00004104 // First try to pre-split loads and stores.
Chandler Carruth0715cba2015-01-01 11:54:38 +00004105 Changed |= presplitLoadsAndStores(AI, AS);
4106
Hiroshi Inoue48e4c7a2017-12-01 06:05:05 +00004107 // Now that we have identified any pre-splitting opportunities,
4108 // mark loads and stores unsplittable except for the following case.
4109 // We leave a slice splittable if all other slices are disjoint or fully
4110 // included in the slice, such as whole-alloca loads and stores.
4111 // If we fail to split these during pre-splitting, we want to force them
4112 // to be rewritten into a partition.
Chandler Carruth24ac8302015-01-02 03:55:54 +00004113 bool IsSorted = true;
Hiroshi Inoue48e4c7a2017-12-01 06:05:05 +00004114
4115 uint64_t AllocaSize = DL.getTypeAllocSize(AI.getAllocatedType());
4116 const uint64_t MaxBitVectorSize = 1024;
Hiroshi Inoue99a8faa2018-01-16 06:23:05 +00004117 if (AllocaSize <= MaxBitVectorSize) {
Hiroshi Inoue48e4c7a2017-12-01 06:05:05 +00004118 // If a byte boundary is included in any load or store, a slice starting or
4119 // ending at the boundary is not splittable.
4120 SmallBitVector SplittableOffset(AllocaSize + 1, true);
4121 for (Slice &S : AS)
4122 for (unsigned O = S.beginOffset() + 1;
4123 O < S.endOffset() && O < AllocaSize; O++)
4124 SplittableOffset.reset(O);
4125
4126 for (Slice &S : AS) {
4127 if (!S.isSplittable())
4128 continue;
4129
4130 if ((S.beginOffset() > AllocaSize || SplittableOffset[S.beginOffset()]) &&
4131 (S.endOffset() > AllocaSize || SplittableOffset[S.endOffset()]))
4132 continue;
4133
4134 if (isa<LoadInst>(S.getUse()->getUser()) ||
4135 isa<StoreInst>(S.getUse()->getUser())) {
4136 S.makeUnsplittable();
4137 IsSorted = false;
4138 }
Chandler Carruth24ac8302015-01-02 03:55:54 +00004139 }
4140 }
Hiroshi Inoue48e4c7a2017-12-01 06:05:05 +00004141 else {
4142 // We only allow whole-alloca splittable loads and stores
4143 // for a large alloca to avoid creating too large BitVector.
4144 for (Slice &S : AS) {
4145 if (!S.isSplittable())
4146 continue;
4147
4148 if (S.beginOffset() == 0 && S.endOffset() >= AllocaSize)
4149 continue;
4150
4151 if (isa<LoadInst>(S.getUse()->getUser()) ||
4152 isa<StoreInst>(S.getUse()->getUser())) {
4153 S.makeUnsplittable();
4154 IsSorted = false;
4155 }
4156 }
4157 }
4158
Chandler Carruth24ac8302015-01-02 03:55:54 +00004159 if (!IsSorted)
Mandeep Singh Grang636d94d2018-04-13 19:47:57 +00004160 llvm::sort(AS.begin(), AS.end());
Chandler Carruth24ac8302015-01-02 03:55:54 +00004161
Adrian Prantl941fa752016-12-05 18:04:47 +00004162 /// Describes the allocas introduced by rewritePartition in order to migrate
4163 /// the debug info.
4164 struct Fragment {
Adrian Prantl565cc182015-01-20 19:42:22 +00004165 AllocaInst *Alloca;
4166 uint64_t Offset;
4167 uint64_t Size;
Adrian Prantl941fa752016-12-05 18:04:47 +00004168 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
Adrian Prantl565cc182015-01-20 19:42:22 +00004169 : Alloca(AI), Offset(O), Size(S) {}
4170 };
Adrian Prantl941fa752016-12-05 18:04:47 +00004171 SmallVector<Fragment, 4> Fragments;
Adrian Prantl565cc182015-01-20 19:42:22 +00004172
Chandler Carruth0715cba2015-01-01 11:54:38 +00004173 // Rewrite each partition.
Chandler Carruthe2f66ce2014-12-22 22:46:00 +00004174 for (auto &P : AS.partitions()) {
Adrian Prantl565cc182015-01-20 19:42:22 +00004175 if (AllocaInst *NewAI = rewritePartition(AI, AS, P)) {
4176 Changed = true;
Adrian Prantl34e75902015-02-09 23:57:22 +00004177 if (NewAI != &AI) {
4178 uint64_t SizeOfByte = 8;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004179 uint64_t AllocaSize = DL.getTypeSizeInBits(NewAI->getAllocatedType());
Adrian Prantl34e75902015-02-09 23:57:22 +00004180 // Don't include any padding.
4181 uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte);
Adrian Prantl941fa752016-12-05 18:04:47 +00004182 Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
Adrian Prantl34e75902015-02-09 23:57:22 +00004183 }
Adrian Prantl565cc182015-01-20 19:42:22 +00004184 }
Chandler Carruth6c321c12013-07-19 10:57:36 +00004185 ++NumPartitions;
Chandler Carruthf0546402013-07-18 07:15:00 +00004186 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004187
Chandler Carruth6c321c12013-07-19 10:57:36 +00004188 NumAllocaPartitions += NumPartitions;
Craig Topper8a950272017-05-18 00:51:39 +00004189 MaxPartitionsPerAlloca.updateMax(NumPartitions);
Chandler Carruth6c321c12013-07-19 10:57:36 +00004190
Adrian Prantl565cc182015-01-20 19:42:22 +00004191 // Migrate debug information from the old alloca to the new alloca(s)
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00004192 // and the individual partitions.
Reid Kleckner0fe506b2017-09-21 19:52:03 +00004193 TinyPtrVector<DbgInfoIntrinsic *> DbgDeclares = FindDbgAddrUses(&AI);
4194 if (!DbgDeclares.empty()) {
4195 auto *Var = DbgDeclares.front()->getVariable();
4196 auto *Expr = DbgDeclares.front()->getExpression();
Adrian Prantld7f6f162017-11-28 00:57:53 +00004197 auto VarSize = Var->getSizeInBits();
Sanjay Patelaf674fb2015-12-14 17:24:23 +00004198 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
Keno Fischerd5354fd2016-01-14 20:06:34 +00004199 uint64_t AllocaSize = DL.getTypeSizeInBits(AI.getAllocatedType());
Adrian Prantl941fa752016-12-05 18:04:47 +00004200 for (auto Fragment : Fragments) {
4201 // Create a fragment expression describing the new partition or reuse AI's
Adrian Prantl565cc182015-01-20 19:42:22 +00004202 // expression if there is only one partition.
Adrian Prantl941fa752016-12-05 18:04:47 +00004203 auto *FragmentExpr = Expr;
4204 if (Fragment.Size < AllocaSize || Expr->isFragment()) {
Adrian Prantl152ac392015-02-01 00:58:04 +00004205 // If this alloca is already a scalar replacement of a larger aggregate,
Adrian Prantl941fa752016-12-05 18:04:47 +00004206 // Fragment.Offset describes the offset inside the scalar.
Adrian Prantl49797ca2016-12-22 05:27:12 +00004207 auto ExprFragment = Expr->getFragmentInfo();
4208 uint64_t Offset = ExprFragment ? ExprFragment->OffsetInBits : 0;
Adrian Prantl941fa752016-12-05 18:04:47 +00004209 uint64_t Start = Offset + Fragment.Offset;
4210 uint64_t Size = Fragment.Size;
Adrian Prantl49797ca2016-12-22 05:27:12 +00004211 if (ExprFragment) {
Adrian Prantl941fa752016-12-05 18:04:47 +00004212 uint64_t AbsEnd =
NAKAMURA Takumia1e97a72017-08-28 06:47:47 +00004213 ExprFragment->OffsetInBits + ExprFragment->SizeInBits;
Adrian Prantl34e75902015-02-09 23:57:22 +00004214 if (Start >= AbsEnd)
4215 // No need to describe a SROAed padding.
4216 continue;
4217 Size = std::min(Size, AbsEnd - Start);
4218 }
Adrian Prantlb192b542017-08-30 20:04:17 +00004219 // The new, smaller fragment is stenciled out from the old fragment.
4220 if (auto OrigFragment = FragmentExpr->getFragmentInfo()) {
4221 assert(Start >= OrigFragment->OffsetInBits &&
4222 "new fragment is outside of original fragment");
4223 Start -= OrigFragment->OffsetInBits;
4224 }
Adrian Prantl77d90b02017-11-28 21:30:38 +00004225
4226 // The alloca may be larger than the variable.
4227 if (VarSize) {
4228 if (Size > *VarSize)
4229 Size = *VarSize;
4230 if (Size == 0 || Start + Size > *VarSize)
4231 continue;
4232 }
4233
Adrian Prantld7f6f162017-11-28 00:57:53 +00004234 // Avoid creating a fragment expression that covers the entire variable.
4235 if (!VarSize || *VarSize != Size) {
4236 if (auto E =
4237 DIExpression::createFragmentExpression(Expr, Start, Size))
4238 FragmentExpr = *E;
4239 else
4240 continue;
4241 }
Adrian Prantl152ac392015-02-01 00:58:04 +00004242 }
Adrian Prantl565cc182015-01-20 19:42:22 +00004243
Reid Kleckner0fe506b2017-09-21 19:52:03 +00004244 // Remove any existing intrinsics describing the same alloca.
4245 for (DbgInfoIntrinsic *OldDII : FindDbgAddrUses(Fragment.Alloca))
4246 OldDII->eraseFromParent();
Adrian Prantl565cc182015-01-20 19:42:22 +00004247
Adrian Prantl941fa752016-12-05 18:04:47 +00004248 DIB.insertDeclare(Fragment.Alloca, Var, FragmentExpr,
Reid Kleckner0fe506b2017-09-21 19:52:03 +00004249 DbgDeclares.front()->getDebugLoc(), &AI);
Adrian Prantl565cc182015-01-20 19:42:22 +00004250 }
4251 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004252 return Changed;
4253}
4254
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00004255/// Clobber a use with undef, deleting the used value if it becomes dead.
Chandler Carruth1bf38c62014-01-19 12:16:54 +00004256void SROA::clobberUse(Use &U) {
4257 Value *OldV = U;
4258 // Replace the use with an undef value.
4259 U = UndefValue::get(OldV->getType());
4260
4261 // Check for this making an instruction dead. We have to garbage collect
4262 // all the dead instructions to ensure the uses of any alloca end up being
4263 // minimal.
4264 if (Instruction *OldI = dyn_cast<Instruction>(OldV))
4265 if (isInstructionTriviallyDead(OldI)) {
4266 DeadInsts.insert(OldI);
4267 }
4268}
4269
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00004270/// Analyze an alloca for SROA.
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004271///
4272/// This analyzes the alloca to ensure we can reason about it, builds
Chandler Carruth9f21fe12013-07-19 09:13:58 +00004273/// the slices of the alloca, and then hands it off to be split and
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004274/// rewritten as needed.
4275bool SROA::runOnAlloca(AllocaInst &AI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004276 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004277 ++NumAllocasAnalyzed;
4278
4279 // Special case dead allocas, as they're trivial.
4280 if (AI.use_empty()) {
4281 AI.eraseFromParent();
4282 return true;
4283 }
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004284 const DataLayout &DL = AI.getModule()->getDataLayout();
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004285
4286 // Skip alloca forms that this analysis can't handle.
4287 if (AI.isArrayAllocation() || !AI.getAllocatedType()->isSized() ||
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004288 DL.getTypeAllocSize(AI.getAllocatedType()) == 0)
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004289 return false;
4290
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00004291 bool Changed = false;
4292
4293 // First, split any FCA loads and stores touching this alloca to promote
4294 // better splitting and promotion opportunities.
Benjamin Kramer6db33382015-10-15 15:08:58 +00004295 AggLoadStoreRewriter AggRewriter;
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00004296 Changed |= AggRewriter.rewrite(AI);
4297
Chandler Carruth9f21fe12013-07-19 09:13:58 +00004298 // Build the slices using a recursive instruction-visiting builder.
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004299 AllocaSlices AS(DL, AI);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004300 LLVM_DEBUG(AS.print(dbgs()));
Chandler Carruth83934062014-10-16 21:11:55 +00004301 if (AS.isEscaped())
Chandler Carruth42cb9cb2012-09-18 12:57:43 +00004302 return Changed;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004303
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004304 // Delete all the dead users of this alloca before splitting and rewriting it.
Chandler Carruth83934062014-10-16 21:11:55 +00004305 for (Instruction *DeadUser : AS.getDeadUsers()) {
Chandler Carruth1bf38c62014-01-19 12:16:54 +00004306 // Free up everything used by this instruction.
Chandler Carruth57d4cae2014-10-16 20:42:08 +00004307 for (Use &DeadOp : DeadUser->operands())
Chandler Carruth1583e992014-03-03 10:42:58 +00004308 clobberUse(DeadOp);
Chandler Carruth1bf38c62014-01-19 12:16:54 +00004309
4310 // Now replace the uses of this instruction.
Chandler Carruth57d4cae2014-10-16 20:42:08 +00004311 DeadUser->replaceAllUsesWith(UndefValue::get(DeadUser->getType()));
Chandler Carruth1bf38c62014-01-19 12:16:54 +00004312
4313 // And mark it for deletion.
Chandler Carruth57d4cae2014-10-16 20:42:08 +00004314 DeadInsts.insert(DeadUser);
Chandler Carruth1bf38c62014-01-19 12:16:54 +00004315 Changed = true;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004316 }
Chandler Carruth83934062014-10-16 21:11:55 +00004317 for (Use *DeadOp : AS.getDeadOperands()) {
Chandler Carruth57d4cae2014-10-16 20:42:08 +00004318 clobberUse(*DeadOp);
Chandler Carruth1bf38c62014-01-19 12:16:54 +00004319 Changed = true;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004320 }
4321
Chandler Carruth9f21fe12013-07-19 09:13:58 +00004322 // No slices to split. Leave the dead alloca for a later pass to clean up.
Chandler Carruth83934062014-10-16 21:11:55 +00004323 if (AS.begin() == AS.end())
Chandler Carruthe5b7a2c2012-10-05 01:29:09 +00004324 return Changed;
4325
Chandler Carruth83934062014-10-16 21:11:55 +00004326 Changed |= splitAlloca(AI, AS);
Chandler Carruthf0546402013-07-18 07:15:00 +00004327
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004328 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00004329 while (!SpeculatablePHIs.empty())
4330 speculatePHINodeLoads(*SpeculatablePHIs.pop_back_val());
4331
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004332 LLVM_DEBUG(dbgs() << " Speculating Selects\n");
Chandler Carruthf0546402013-07-18 07:15:00 +00004333 while (!SpeculatableSelects.empty())
4334 speculateSelectInstLoads(*SpeculatableSelects.pop_back_val());
4335
4336 return Changed;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004337}
4338
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00004339/// Delete the dead instructions accumulated in this run.
Chandler Carruth19450da2012-09-14 10:26:38 +00004340///
4341/// Recursively deletes the dead instructions we've accumulated. This is done
4342/// at the very end to maximize locality of the recursive delete and to
4343/// minimize the problems of invalidated instruction pointers as such pointers
4344/// are used heavily in the intermediate stages of the algorithm.
4345///
4346/// We also record the alloca instructions deleted here so that they aren't
4347/// subsequently handed to mem2reg to promote.
Teresa Johnson33090022017-11-20 18:33:38 +00004348bool SROA::deleteDeadInstructions(
Chandler Carruth113dc642014-12-20 02:39:18 +00004349 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
Teresa Johnson33090022017-11-20 18:33:38 +00004350 bool Changed = false;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004351 while (!DeadInsts.empty()) {
4352 Instruction *I = DeadInsts.pop_back_val();
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004353 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004354
Reid Kleckner0fe506b2017-09-21 19:52:03 +00004355 // If the instruction is an alloca, find the possible dbg.declare connected
4356 // to it, and remove it too. We must do this before calling RAUW or we will
4357 // not be able to find it.
4358 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
4359 DeletedAllocas.insert(AI);
4360 for (DbgInfoIntrinsic *OldDII : FindDbgAddrUses(AI))
4361 OldDII->eraseFromParent();
4362 }
4363
Chandler Carruth58d05562012-10-25 04:37:07 +00004364 I->replaceAllUsesWith(UndefValue::get(I->getType()));
4365
Chandler Carruth1583e992014-03-03 10:42:58 +00004366 for (Use &Operand : I->operands())
4367 if (Instruction *U = dyn_cast<Instruction>(Operand)) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004368 // Zero out the operand and see if it becomes trivially dead.
Craig Topperf40110f2014-04-25 05:29:35 +00004369 Operand = nullptr;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004370 if (isInstructionTriviallyDead(U))
Chandler Carruth18db7952012-11-20 01:12:50 +00004371 DeadInsts.insert(U);
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004372 }
4373
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004374 ++NumDeleted;
4375 I->eraseFromParent();
Teresa Johnson33090022017-11-20 18:33:38 +00004376 Changed = true;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004377 }
Teresa Johnson33090022017-11-20 18:33:38 +00004378 return Changed;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004379}
4380
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00004381/// Promote the allocas, using the best available technique.
Chandler Carruth70b44c52012-09-15 11:43:14 +00004382///
4383/// This attempts to promote whatever allocas have been identified as viable in
4384/// the PromotableAllocas list. If that list is empty, there is nothing to do.
Chandler Carruth748d0952015-08-26 09:09:29 +00004385/// This function returns whether any promotion occurred.
Chandler Carruth70b44c52012-09-15 11:43:14 +00004386bool SROA::promoteAllocas(Function &F) {
4387 if (PromotableAllocas.empty())
4388 return false;
4389
4390 NumPromoted += PromotableAllocas.size();
4391
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004392 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
Davide Italiano612d5a92017-04-09 20:47:14 +00004393 PromoteMemToReg(PromotableAllocas, *DT, AC);
Chandler Carruth70b44c52012-09-15 11:43:14 +00004394 PromotableAllocas.clear();
4395 return true;
4396}
4397
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004398PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT,
4399 AssumptionCache &RunAC) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +00004400 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004401 C = &F.getContext();
Chandler Carruth29a18a42015-09-12 09:09:14 +00004402 DT = &RunDT;
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004403 AC = &RunAC;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004404
4405 BasicBlock &EntryBB = F.getEntryBlock();
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00004406 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end());
Adrian Prantl565cc182015-01-20 19:42:22 +00004407 I != E; ++I) {
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004408 if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
4409 Worklist.insert(AI);
Adrian Prantl565cc182015-01-20 19:42:22 +00004410 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004411
4412 bool Changed = false;
Chandler Carruth19450da2012-09-14 10:26:38 +00004413 // A set of deleted alloca instruction pointers which should be removed from
4414 // the list of promotable allocas.
4415 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
4416
Chandler Carruthac8317f2012-10-04 12:33:50 +00004417 do {
4418 while (!Worklist.empty()) {
4419 Changed |= runOnAlloca(*Worklist.pop_back_val());
Teresa Johnson33090022017-11-20 18:33:38 +00004420 Changed |= deleteDeadInstructions(DeletedAllocas);
Chandler Carruthb09f0a32012-10-02 22:46:45 +00004421
Chandler Carruthac8317f2012-10-04 12:33:50 +00004422 // Remove the deleted allocas from various lists so that we don't try to
4423 // continue processing them.
4424 if (!DeletedAllocas.empty()) {
Chandler Carruth113dc642014-12-20 02:39:18 +00004425 auto IsInSet = [&](AllocaInst *AI) { return DeletedAllocas.count(AI); };
Benjamin Kramer3a377bc2014-03-01 11:47:00 +00004426 Worklist.remove_if(IsInSet);
4427 PostPromotionWorklist.remove_if(IsInSet);
Eugene Zelenko75075ef2017-09-01 21:37:29 +00004428 PromotableAllocas.erase(llvm::remove_if(PromotableAllocas, IsInSet),
Chandler Carruthac8317f2012-10-04 12:33:50 +00004429 PromotableAllocas.end());
4430 DeletedAllocas.clear();
4431 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004432 }
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004433
Chandler Carruthac8317f2012-10-04 12:33:50 +00004434 Changed |= promoteAllocas(F);
4435
4436 Worklist = PostPromotionWorklist;
4437 PostPromotionWorklist.clear();
4438 } while (!Worklist.empty());
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004439
Davide Italiano16e96d42016-06-07 13:21:17 +00004440 if (!Changed)
4441 return PreservedAnalyses::all();
4442
Davide Italiano16e96d42016-06-07 13:21:17 +00004443 PreservedAnalyses PA;
Chandler Carruthca68a3e2017-01-15 06:32:49 +00004444 PA.preserveSet<CFGAnalyses>();
Davide Italiano16e96d42016-06-07 13:21:17 +00004445 PA.preserve<GlobalsAA>();
4446 return PA;
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004447}
4448
Sean Silva36e0d012016-08-09 00:28:15 +00004449PreservedAnalyses SROA::run(Function &F, FunctionAnalysisManager &AM) {
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004450 return runImpl(F, AM.getResult<DominatorTreeAnalysis>(F),
4451 AM.getResult<AssumptionAnalysis>(F));
Chandler Carruth1b398ae2012-09-14 09:22:59 +00004452}
Chandler Carruth29a18a42015-09-12 09:09:14 +00004453
4454/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
4455///
4456/// This is in the llvm namespace purely to allow it to be a friend of the \c
4457/// SROA pass.
4458class llvm::sroa::SROALegacyPass : public FunctionPass {
4459 /// The SROA implementation.
4460 SROA Impl;
4461
4462public:
Eugene Zelenko75075ef2017-09-01 21:37:29 +00004463 static char ID;
4464
Chandler Carruth29a18a42015-09-12 09:09:14 +00004465 SROALegacyPass() : FunctionPass(ID) {
4466 initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
4467 }
Eugene Zelenko75075ef2017-09-01 21:37:29 +00004468
Chandler Carruth29a18a42015-09-12 09:09:14 +00004469 bool runOnFunction(Function &F) override {
Andrew Kayloraa641a52016-04-22 22:06:11 +00004470 if (skipFunction(F))
Chandler Carruth29a18a42015-09-12 09:09:14 +00004471 return false;
4472
4473 auto PA = Impl.runImpl(
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004474 F, getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
4475 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F));
Chandler Carruth29a18a42015-09-12 09:09:14 +00004476 return !PA.areAllPreserved();
4477 }
Eugene Zelenko75075ef2017-09-01 21:37:29 +00004478
Chandler Carruth29a18a42015-09-12 09:09:14 +00004479 void getAnalysisUsage(AnalysisUsage &AU) const override {
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004480 AU.addRequired<AssumptionCacheTracker>();
Chandler Carruth29a18a42015-09-12 09:09:14 +00004481 AU.addRequired<DominatorTreeWrapperPass>();
4482 AU.addPreserved<GlobalsAAWrapperPass>();
4483 AU.setPreservesCFG();
4484 }
4485
Mehdi Amini117296c2016-10-01 02:56:57 +00004486 StringRef getPassName() const override { return "SROA"; }
Chandler Carruth29a18a42015-09-12 09:09:14 +00004487};
4488
4489char SROALegacyPass::ID = 0;
4490
4491FunctionPass *llvm::createSROAPass() { return new SROALegacyPass(); }
4492
4493INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
4494 "Scalar Replacement Of Aggregates", false, false)
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004495INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
Chandler Carruth29a18a42015-09-12 09:09:14 +00004496INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
4497INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
4498 false, false)