blob: acc9eae3701eeb15b0351bf882294b3395858609 [file] [log] [blame]
Nicolas Capens2ae9d742016-11-24 14:43:05 -05001// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "Optimizer.hpp"
16
17#include "src/IceCfg.h"
18#include "src/IceCfgNode.h"
19
Nicolas Capens2ae9d742016-11-24 14:43:05 -050020#include <vector>
21
Nicolas Capens157ba262019-12-10 17:49:14 -050022namespace {
23
24class Optimizer
Nicolas Capens2ae9d742016-11-24 14:43:05 -050025{
Nicolas Capens157ba262019-12-10 17:49:14 -050026public:
Nicolas Capens54313fb2021-02-19 14:26:27 -050027 Optimizer(rr::Nucleus::OptimizerReport *report)
28 : report(report)
29 {
30 }
31
Nicolas Capens157ba262019-12-10 17:49:14 -050032 void run(Ice::Cfg *function);
33
34private:
35 void analyzeUses(Ice::Cfg *function);
Nicolas Capens0b506e12021-02-04 17:00:08 -050036
Nicolas Capens157ba262019-12-10 17:49:14 -050037 void eliminateDeadCode();
Nicolas Capens0b506e12021-02-04 17:00:08 -050038 void propagateAlloca();
Nicolas Capens0cfc0432021-02-05 15:18:42 -050039 void performScalarReplacementOfAggregates();
Nicolas Capens157ba262019-12-10 17:49:14 -050040 void eliminateUnitializedLoads();
41 void eliminateLoadsFollowingSingleStore();
42 void optimizeStoresInSingleBasicBlock();
43
44 void replace(Ice::Inst *instruction, Ice::Operand *newValue);
45 void deleteInstruction(Ice::Inst *instruction);
46 bool isDead(Ice::Inst *instruction);
Nicolas Capens0cfc0432021-02-05 15:18:42 -050047 bool isStaticallyIndexedArray(Ice::Operand *allocaAddress);
Nicolas Capens157ba262019-12-10 17:49:14 -050048
Nicolas Capens33a77f72021-02-08 15:04:38 -050049 static const Ice::InstIntrinsic *asLoadSubVector(const Ice::Inst *instruction);
50 static const Ice::InstIntrinsic *asStoreSubVector(const Ice::Inst *instruction);
Nicolas Capens157ba262019-12-10 17:49:14 -050051 static bool isLoad(const Ice::Inst &instruction);
52 static bool isStore(const Ice::Inst &instruction);
Nicolas Capens157ba262019-12-10 17:49:14 -050053 static std::size_t storeSize(const Ice::Inst *instruction);
54 static bool loadTypeMatchesStore(const Ice::Inst *load, const Ice::Inst *store);
55
Nicolas Capens54313fb2021-02-19 14:26:27 -050056 void collectDiagnostics();
57
Nicolas Capens157ba262019-12-10 17:49:14 -050058 Ice::Cfg *function;
59 Ice::GlobalContext *context;
60
Ben Clayton713b8d32019-12-17 20:37:56 +000061 struct Uses : std::vector<Ice::Inst *>
Nicolas Capens2ae9d742016-11-24 14:43:05 -050062 {
Nicolas Capens157ba262019-12-10 17:49:14 -050063 bool areOnlyLoadStore() const;
64 void insert(Ice::Operand *value, Ice::Inst *instruction);
65 void erase(Ice::Inst *instruction);
Nicolas Capens2ae9d742016-11-24 14:43:05 -050066
Ben Clayton713b8d32019-12-17 20:37:56 +000067 std::vector<Ice::Inst *> loads;
68 std::vector<Ice::Inst *> stores;
Nicolas Capens2ae9d742016-11-24 14:43:05 -050069 };
70
Nicolas Capens157ba262019-12-10 17:49:14 -050071 struct LoadStoreInst
Nicolas Capens2ae9d742016-11-24 14:43:05 -050072 {
Ben Clayton713b8d32019-12-17 20:37:56 +000073 LoadStoreInst(Ice::Inst *inst, bool isStore)
74 : inst(inst)
Nicolas Capens673a7fe2021-02-05 15:03:22 -050075 , address(isStore ? inst->getStoreAddress() : inst->getLoadAddress())
Ben Clayton713b8d32019-12-17 20:37:56 +000076 , isStore(isStore)
Alexis Hetu932640b2018-06-20 15:35:53 -040077 {
Alexis Hetu932640b2018-06-20 15:35:53 -040078 }
Nicolas Capens2ae9d742016-11-24 14:43:05 -050079
Ben Clayton713b8d32019-12-17 20:37:56 +000080 Ice::Inst *inst;
Nicolas Capens157ba262019-12-10 17:49:14 -050081 Ice::Operand *address;
82 bool isStore;
83 };
84
Ben Clayton713b8d32019-12-17 20:37:56 +000085 Optimizer::Uses *getUses(Ice::Operand *);
86 void setUses(Ice::Operand *, Optimizer::Uses *);
87 bool hasUses(Ice::Operand *) const;
Nicolas Capens157ba262019-12-10 17:49:14 -050088
Ben Clayton713b8d32019-12-17 20:37:56 +000089 Ice::CfgNode *getNode(Ice::Inst *);
90 void setNode(Ice::Inst *, Ice::CfgNode *);
Nicolas Capens157ba262019-12-10 17:49:14 -050091
Ben Clayton713b8d32019-12-17 20:37:56 +000092 Ice::Inst *getDefinition(Ice::Variable *);
93 void setDefinition(Ice::Variable *, Ice::Inst *);
Nicolas Capens157ba262019-12-10 17:49:14 -050094
Ben Clayton713b8d32019-12-17 20:37:56 +000095 const std::vector<LoadStoreInst> &getLoadStoreInsts(Ice::CfgNode *);
96 void setLoadStoreInsts(Ice::CfgNode *, std::vector<LoadStoreInst> *);
97 bool hasLoadStoreInsts(Ice::CfgNode *node) const;
Nicolas Capens157ba262019-12-10 17:49:14 -050098
Antonio Maiorano5ba2a5b2020-01-17 15:29:37 -050099 std::vector<Ice::Operand *> operandsWithUses;
Nicolas Capens54313fb2021-02-19 14:26:27 -0500100
101 rr::Nucleus::OptimizerReport *report = nullptr;
Nicolas Capens157ba262019-12-10 17:49:14 -0500102};
103
104void Optimizer::run(Ice::Cfg *function)
105{
106 this->function = function;
107 this->context = function->getContext();
108
109 analyzeUses(function);
110
Nicolas Capens0b506e12021-02-04 17:00:08 -0500111 // Start by eliminating any dead code, to avoid redundant work for the
112 // subsequent optimization passes.
Nicolas Capens157ba262019-12-10 17:49:14 -0500113 eliminateDeadCode();
Nicolas Capens0b506e12021-02-04 17:00:08 -0500114
115 // Eliminate allocas which store the address of other allocas.
116 propagateAlloca();
117
Nicolas Capens0cfc0432021-02-05 15:18:42 -0500118 // Replace arrays with individual elements if only statically indexed.
119 performScalarReplacementOfAggregates();
120
Nicolas Capens157ba262019-12-10 17:49:14 -0500121 eliminateUnitializedLoads();
122 eliminateLoadsFollowingSingleStore();
123 optimizeStoresInSingleBasicBlock();
124 eliminateDeadCode();
125
Antonio Maiorano5ba2a5b2020-01-17 15:29:37 -0500126 for(auto operand : operandsWithUses)
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500127 {
Antonio Maiorano614a4d42020-01-29 13:33:02 -0500128 // Deletes the Uses instance on the operand
129 setUses(operand, nullptr);
Nicolas Capens157ba262019-12-10 17:49:14 -0500130 }
Antonio Maiorano5ba2a5b2020-01-17 15:29:37 -0500131 operandsWithUses.clear();
Nicolas Capens54313fb2021-02-19 14:26:27 -0500132
133 collectDiagnostics();
Nicolas Capens157ba262019-12-10 17:49:14 -0500134}
135
Nicolas Capens0b506e12021-02-04 17:00:08 -0500136// Eliminates allocas which store the address of other allocas.
137void Optimizer::propagateAlloca()
138{
139 Ice::CfgNode *entryBlock = function->getEntryNode();
140 Ice::InstList &instList = entryBlock->getInsts();
141
142 for(Ice::Inst &inst : instList)
143 {
144 if(inst.isDeleted())
145 {
146 continue;
147 }
148
149 auto *alloca = llvm::dyn_cast<Ice::InstAlloca>(&inst);
150
151 if(!alloca)
152 {
153 break; // Allocas are all at the top
154 }
155
156 // Look for stores of this alloca's address.
157 Ice::Operand *address = alloca->getDest();
158 Uses uses = *getUses(address); // Hard copy
159
160 for(auto *store : uses)
161 {
162 if(isStore(*store) && store->getData() == address)
163 {
164 Ice::Operand *dest = store->getStoreAddress();
165 Ice::Variable *destVar = llvm::dyn_cast<Ice::Variable>(dest);
166 Ice::Inst *def = destVar ? getDefinition(destVar) : nullptr;
167
168 // If the address is stored into another stack variable, eliminate the latter.
169 if(def && def->getKind() == Ice::Inst::Alloca)
170 {
171 Uses destUses = *getUses(dest); // Hard copy
172
173 // Make sure the only store into the stack variable is this address, and that all of its other uses are loads.
174 // This prevents dynamic array loads/stores to be replaced by a scalar.
175 if((destUses.stores.size() == 1) && (destUses.loads.size() == destUses.size() - 1))
176 {
177 for(auto *load : destUses.loads)
178 {
179 replace(load, address);
180 }
181
182 // The address is now only stored, never loaded, so the store can be eliminated, together with its alloca.
183 assert(getUses(dest)->size() == 1);
184 deleteInstruction(store);
185 assert(def->isDeleted());
186 }
187 }
188 }
189 }
190 }
191}
192
Nicolas Capens0cfc0432021-02-05 15:18:42 -0500193Ice::Type pointerType()
194{
195 if(sizeof(void *) == 8)
196 {
197 return Ice::IceType_i64;
198 }
199 else
200 {
201 return Ice::IceType_i32;
202 }
203}
204
205// Replace arrays with individual elements if only statically indexed.
206void Optimizer::performScalarReplacementOfAggregates()
207{
208 std::vector<Ice::InstAlloca *> newAllocas;
209
210 Ice::CfgNode *entryBlock = function->getEntryNode();
211 Ice::InstList &instList = entryBlock->getInsts();
212
213 for(Ice::Inst &inst : instList)
214 {
215 if(inst.isDeleted())
216 {
217 continue;
218 }
219
220 auto *alloca = llvm::dyn_cast<Ice::InstAlloca>(&inst);
221
222 if(!alloca)
223 {
224 break; // Allocas are all at the top
225 }
226
227 uint32_t sizeInBytes = llvm::cast<Ice::ConstantInteger32>(alloca->getSizeInBytes())->getValue();
228 uint32_t alignInBytes = alloca->getAlignInBytes();
229
230 // This pass relies on array elements to be naturally aligned (i.e. matches the type size).
231 assert(sizeInBytes >= alignInBytes);
232 assert(sizeInBytes % alignInBytes == 0);
233 uint32_t elementCount = sizeInBytes / alignInBytes;
234
235 Ice::Operand *address = alloca->getDest();
236
237 if(isStaticallyIndexedArray(address))
238 {
239 // Delete the old array.
240 alloca->setDeleted();
241
242 // Allocate new stack slots for each element.
243 std::vector<Ice::Variable *> newAddress(elementCount);
244 auto *bytes = Ice::ConstantInteger32::create(context, Ice::IceType_i32, alignInBytes);
245
246 for(uint32_t i = 0; i < elementCount; i++)
247 {
248 newAddress[i] = function->makeVariable(pointerType());
249 auto *alloca = Ice::InstAlloca::create(function, newAddress[i], bytes, alignInBytes);
250 setDefinition(newAddress[i], alloca);
251
252 newAllocas.push_back(alloca);
253 }
254
255 Uses uses = *getUses(address); // Hard copy
256
257 for(auto *use : uses)
258 {
259 assert(!use->isDeleted());
260
261 if(isLoad(*use)) // Direct use of base address
262 {
263 use->replaceSource(asLoadSubVector(use) ? 1 : 0, newAddress[0]);
264 getUses(newAddress[0])->insert(newAddress[0], use);
265 }
266 else if(isStore(*use)) // Direct use of base address
267 {
268 use->replaceSource(asStoreSubVector(use) ? 2 : 1, newAddress[0]);
269 getUses(newAddress[0])->insert(newAddress[0], use);
270 }
271 else // Statically indexed use
272 {
273 auto *arithmetic = llvm::cast<Ice::InstArithmetic>(use);
274
275 if(arithmetic->getOp() == Ice::InstArithmetic::Add)
276 {
277 auto *rhs = arithmetic->getSrc(1);
278 int32_t offset = llvm::cast<Ice::ConstantInteger32>(rhs)->getValue();
279
280 assert(offset % alignInBytes == 0);
281 int32_t index = offset / alignInBytes;
282 assert(static_cast<uint32_t>(index) < elementCount);
283
284 replace(arithmetic, newAddress[index]);
285 }
286 else
287 assert(false && "Mismatch between isStaticallyIndexedArray() and scalarReplacementOfAggregates()");
288 }
289 }
290 }
291 }
292
293 // After looping over all the old allocas, add any new ones that replace them.
294 // They're added to the front in reverse order, to retain their original order.
295 for(size_t i = newAllocas.size(); i-- != 0;)
296 {
297 if(!isDead(newAllocas[i]))
298 {
299 instList.push_front(newAllocas[i]);
300 }
301 }
302}
303
Nicolas Capens157ba262019-12-10 17:49:14 -0500304void Optimizer::eliminateDeadCode()
305{
306 bool modified;
307 do
308 {
309 modified = false;
310 for(Ice::CfgNode *basicBlock : function->getNodes())
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500311 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500312 for(Ice::Inst &inst : Ice::reverse_range(basicBlock->getInsts()))
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500313 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500314 if(inst.isDeleted())
315 {
316 continue;
317 }
318
319 if(isDead(&inst))
320 {
321 deleteInstruction(&inst);
322 modified = true;
323 }
324 }
325 }
Ben Clayton713b8d32019-12-17 20:37:56 +0000326 } while(modified);
Nicolas Capens157ba262019-12-10 17:49:14 -0500327}
328
329void Optimizer::eliminateUnitializedLoads()
330{
331 Ice::CfgNode *entryBlock = function->getEntryNode();
332
333 for(Ice::Inst &alloca : entryBlock->getInsts())
334 {
335 if(alloca.isDeleted())
336 {
337 continue;
338 }
339
340 if(!llvm::isa<Ice::InstAlloca>(alloca))
341 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000342 break; // Allocas are all at the top
Nicolas Capens157ba262019-12-10 17:49:14 -0500343 }
344
345 Ice::Operand *address = alloca.getDest();
346
347 if(!hasUses(address))
348 {
349 continue;
350 }
351
352 const auto &addressUses = *getUses(address);
353
354 if(!addressUses.areOnlyLoadStore())
355 {
356 continue;
357 }
358
359 if(addressUses.stores.empty())
360 {
361 for(Ice::Inst *load : addressUses.loads)
362 {
363 Ice::Variable *loadData = load->getDest();
364
365 if(hasUses(loadData))
366 {
367 for(Ice::Inst *use : *getUses(loadData))
368 {
369 for(Ice::SizeT i = 0; i < use->getSrcSize(); i++)
370 {
371 if(use->getSrc(i) == loadData)
372 {
373 auto *undef = context->getConstantUndef(loadData->getType());
374
375 use->replaceSource(i, undef);
376 }
377 }
378 }
379
380 setUses(loadData, nullptr);
381 }
382
383 load->setDeleted();
384 }
385
386 alloca.setDeleted();
387 setUses(address, nullptr);
388 }
389 }
390}
391
392void Optimizer::eliminateLoadsFollowingSingleStore()
393{
394 Ice::CfgNode *entryBlock = function->getEntryNode();
395
396 for(Ice::Inst &alloca : entryBlock->getInsts())
397 {
398 if(alloca.isDeleted())
399 {
400 continue;
401 }
402
403 if(!llvm::isa<Ice::InstAlloca>(alloca))
404 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000405 break; // Allocas are all at the top
Nicolas Capens157ba262019-12-10 17:49:14 -0500406 }
407
408 Ice::Operand *address = alloca.getDest();
409
410 if(!hasUses(address))
411 {
412 continue;
413 }
414
415 auto &addressUses = *getUses(address);
416
417 if(!addressUses.areOnlyLoadStore())
418 {
419 continue;
420 }
421
422 if(addressUses.stores.size() == 1)
423 {
424 Ice::Inst *store = addressUses.stores[0];
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500425 Ice::Operand *storeValue = store->getData();
Nicolas Capens157ba262019-12-10 17:49:14 -0500426
Nicolas Capens3ab17bd2021-02-10 16:41:31 -0500427 auto instIterator = store->getIterator();
428 auto basicBlockEnd = getNode(store)->getInsts().end();
429
430 while(++instIterator != basicBlockEnd)
Nicolas Capens157ba262019-12-10 17:49:14 -0500431 {
Nicolas Capens3ab17bd2021-02-10 16:41:31 -0500432 Ice::Inst *load = &*instIterator;
433
Nicolas Capens157ba262019-12-10 17:49:14 -0500434 if(load->isDeleted() || !isLoad(*load))
435 {
436 continue;
437 }
438
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500439 if(load->getLoadAddress() != address)
Nicolas Capens157ba262019-12-10 17:49:14 -0500440 {
441 continue;
442 }
443
444 if(!loadTypeMatchesStore(load, store))
445 {
446 continue;
447 }
448
449 replace(load, storeValue);
450
451 for(size_t i = 0; i < addressUses.loads.size(); i++)
452 {
453 if(addressUses.loads[i] == load)
454 {
455 addressUses.loads[i] = addressUses.loads.back();
456 addressUses.loads.pop_back();
457 break;
458 }
459 }
460
461 for(size_t i = 0; i < addressUses.size(); i++)
462 {
463 if(addressUses[i] == load)
464 {
465 addressUses[i] = addressUses.back();
466 addressUses.pop_back();
467 break;
468 }
469 }
470
471 if(addressUses.size() == 1)
472 {
473 assert(addressUses[0] == store);
474
475 alloca.setDeleted();
476 store->setDeleted();
477 setUses(address, nullptr);
478
479 if(hasUses(storeValue))
480 {
481 auto &valueUses = *getUses(storeValue);
482
483 for(size_t i = 0; i < valueUses.size(); i++)
484 {
485 if(valueUses[i] == store)
486 {
487 valueUses[i] = valueUses.back();
488 valueUses.pop_back();
489 break;
490 }
491 }
492
493 if(valueUses.empty())
494 {
495 setUses(storeValue, nullptr);
496 }
497 }
498
499 break;
500 }
501 }
502 }
503 }
504}
505
506void Optimizer::optimizeStoresInSingleBasicBlock()
507{
508 Ice::CfgNode *entryBlock = function->getEntryNode();
509
Ben Clayton713b8d32019-12-17 20:37:56 +0000510 std::vector<std::vector<LoadStoreInst> *> allocatedVectors;
Nicolas Capens157ba262019-12-10 17:49:14 -0500511
512 for(Ice::Inst &alloca : entryBlock->getInsts())
513 {
514 if(alloca.isDeleted())
515 {
516 continue;
517 }
518
519 if(!llvm::isa<Ice::InstAlloca>(alloca))
520 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000521 break; // Allocas are all at the top
Nicolas Capens157ba262019-12-10 17:49:14 -0500522 }
523
524 Ice::Operand *address = alloca.getDest();
525
526 if(!hasUses(address))
527 {
528 continue;
529 }
530
531 const auto &addressUses = *getUses(address);
532
533 if(!addressUses.areOnlyLoadStore())
534 {
535 continue;
536 }
537
538 Ice::CfgNode *singleBasicBlock = getNode(addressUses.stores[0]);
539
540 for(size_t i = 1; i < addressUses.stores.size(); i++)
541 {
542 Ice::Inst *store = addressUses.stores[i];
543 if(getNode(store) != singleBasicBlock)
544 {
545 singleBasicBlock = nullptr;
546 break;
547 }
548 }
549
550 if(singleBasicBlock)
551 {
552 if(!hasLoadStoreInsts(singleBasicBlock))
553 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000554 std::vector<LoadStoreInst> *loadStoreInstVector = new std::vector<LoadStoreInst>();
Nicolas Capens157ba262019-12-10 17:49:14 -0500555 setLoadStoreInsts(singleBasicBlock, loadStoreInstVector);
556 allocatedVectors.push_back(loadStoreInstVector);
557 for(Ice::Inst &inst : singleBasicBlock->getInsts())
Nicolas Capensc9d70d52016-12-12 15:07:39 -0500558 {
559 if(inst.isDeleted())
560 {
561 continue;
562 }
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500563
Nicolas Capens157ba262019-12-10 17:49:14 -0500564 bool isStoreInst = isStore(inst);
565 bool isLoadInst = isLoad(inst);
566
567 if(isStoreInst || isLoadInst)
Nicolas Capensc9d70d52016-12-12 15:07:39 -0500568 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500569 loadStoreInstVector->push_back(LoadStoreInst(&inst, isStoreInst));
Nicolas Capensc9d70d52016-12-12 15:07:39 -0500570 }
571 }
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500572 }
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500573
Nicolas Capens157ba262019-12-10 17:49:14 -0500574 Ice::Inst *store = nullptr;
575 Ice::Operand *storeValue = nullptr;
576 bool unmatchedLoads = false;
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500577
Ben Clayton713b8d32019-12-17 20:37:56 +0000578 for(auto &loadStoreInst : getLoadStoreInsts(singleBasicBlock))
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500579 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000580 Ice::Inst *inst = loadStoreInst.inst;
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500581
Nicolas Capens157ba262019-12-10 17:49:14 -0500582 if((loadStoreInst.address != address) || inst->isDeleted())
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500583 {
584 continue;
585 }
586
Nicolas Capens157ba262019-12-10 17:49:14 -0500587 if(loadStoreInst.isStore)
Alexis Hetu932640b2018-06-20 15:35:53 -0400588 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500589 // New store found. If we had a previous one, try to eliminate it.
590 if(store && !unmatchedLoads)
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500591 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500592 // If the previous store is wider than the new one, we can't eliminate it
593 // because there could be a wide load reading its non-overwritten data.
594 if(storeSize(inst) >= storeSize(store))
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500595 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500596 deleteInstruction(store);
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500597 }
598 }
599
Nicolas Capens157ba262019-12-10 17:49:14 -0500600 store = inst;
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500601 storeValue = store->getData();
Nicolas Capens157ba262019-12-10 17:49:14 -0500602 unmatchedLoads = false;
603 }
604 else
605 {
606 if(!loadTypeMatchesStore(inst, store))
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500607 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500608 unmatchedLoads = true;
609 continue;
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500610 }
Nicolas Capens157ba262019-12-10 17:49:14 -0500611
612 replace(inst, storeValue);
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500613 }
614 }
615 }
616 }
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500617
Nicolas Capens157ba262019-12-10 17:49:14 -0500618 for(auto loadStoreInstVector : allocatedVectors)
Nicolas Capense205c3d2016-11-30 15:14:28 -0500619 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500620 delete loadStoreInstVector;
621 }
622}
Nicolas Capense205c3d2016-11-30 15:14:28 -0500623
Nicolas Capens157ba262019-12-10 17:49:14 -0500624void Optimizer::analyzeUses(Ice::Cfg *function)
625{
626 for(Ice::CfgNode *basicBlock : function->getNodes())
627 {
628 for(Ice::Inst &instruction : basicBlock->getInsts())
Nicolas Capense205c3d2016-11-30 15:14:28 -0500629 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500630 if(instruction.isDeleted())
Nicolas Capense205c3d2016-11-30 15:14:28 -0500631 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500632 continue;
633 }
Alexis Hetu932640b2018-06-20 15:35:53 -0400634
Nicolas Capens157ba262019-12-10 17:49:14 -0500635 setNode(&instruction, basicBlock);
636 if(instruction.getDest())
637 {
638 setDefinition(instruction.getDest(), &instruction);
639 }
640
641 for(Ice::SizeT i = 0; i < instruction.getSrcSize(); i++)
642 {
643 Ice::SizeT unique = 0;
644 for(; unique < i; unique++)
Nicolas Capense205c3d2016-11-30 15:14:28 -0500645 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500646 if(instruction.getSrc(i) == instruction.getSrc(unique))
Alexis Hetu932640b2018-06-20 15:35:53 -0400647 {
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500648 break;
649 }
650 }
651
Nicolas Capens157ba262019-12-10 17:49:14 -0500652 if(i == unique)
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500653 {
Nicolas Capens157ba262019-12-10 17:49:14 -0500654 Ice::Operand *src = instruction.getSrc(i);
655 getUses(src)->insert(src, &instruction);
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500656 }
Nicolas Capensf4452fc2016-12-12 13:08:06 -0500657 }
658 }
659 }
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500660}
661
Nicolas Capens157ba262019-12-10 17:49:14 -0500662void Optimizer::replace(Ice::Inst *instruction, Ice::Operand *newValue)
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500663{
Nicolas Capens157ba262019-12-10 17:49:14 -0500664 Ice::Variable *oldValue = instruction->getDest();
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500665
Nicolas Capens157ba262019-12-10 17:49:14 -0500666 if(!newValue)
667 {
668 newValue = context->getConstantUndef(oldValue->getType());
Nicolas Capens2ae9d742016-11-24 14:43:05 -0500669 }
Nicolas Capens157ba262019-12-10 17:49:14 -0500670
671 if(hasUses(oldValue))
672 {
673 for(Ice::Inst *use : *getUses(oldValue))
674 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000675 assert(!use->isDeleted()); // Should have been removed from uses already
Nicolas Capens157ba262019-12-10 17:49:14 -0500676
677 for(Ice::SizeT i = 0; i < use->getSrcSize(); i++)
678 {
679 if(use->getSrc(i) == oldValue)
680 {
681 use->replaceSource(i, newValue);
682 }
683 }
684
685 getUses(newValue)->insert(newValue, use);
686 }
687
688 setUses(oldValue, nullptr);
689 }
690
691 deleteInstruction(instruction);
692}
693
694void Optimizer::deleteInstruction(Ice::Inst *instruction)
695{
696 if(!instruction || instruction->isDeleted())
697 {
698 return;
699 }
700
Nicolas Capens0cfc0432021-02-05 15:18:42 -0500701 assert(!instruction->getDest() || getUses(instruction->getDest())->empty());
Nicolas Capens157ba262019-12-10 17:49:14 -0500702 instruction->setDeleted();
703
704 for(Ice::SizeT i = 0; i < instruction->getSrcSize(); i++)
705 {
706 Ice::Operand *src = instruction->getSrc(i);
707
708 if(hasUses(src))
709 {
710 auto &srcUses = *getUses(src);
711
712 srcUses.erase(instruction);
713
714 if(srcUses.empty())
715 {
716 setUses(src, nullptr);
717
718 if(Ice::Variable *var = llvm::dyn_cast<Ice::Variable>(src))
719 {
720 deleteInstruction(getDefinition(var));
721 }
722 }
723 }
724 }
725}
726
727bool Optimizer::isDead(Ice::Inst *instruction)
728{
729 Ice::Variable *dest = instruction->getDest();
730
731 if(dest)
732 {
733 return (!hasUses(dest) || getUses(dest)->empty()) && !instruction->hasSideEffects();
734 }
735 else if(isStore(*instruction))
736 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500737 if(Ice::Variable *address = llvm::dyn_cast<Ice::Variable>(instruction->getStoreAddress()))
Nicolas Capens157ba262019-12-10 17:49:14 -0500738 {
739 Ice::Inst *def = getDefinition(address);
740
741 if(def && llvm::isa<Ice::InstAlloca>(def))
742 {
743 if(hasUses(address))
744 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000745 Optimizer::Uses *uses = getUses(address);
746 return uses->size() == uses->stores.size(); // Dead if all uses are stores
Nicolas Capens157ba262019-12-10 17:49:14 -0500747 }
748 else
749 {
Ben Clayton713b8d32019-12-17 20:37:56 +0000750 return true; // No uses
Nicolas Capens157ba262019-12-10 17:49:14 -0500751 }
752 }
753 }
754 }
755
756 return false;
757}
758
Nicolas Capens0cfc0432021-02-05 15:18:42 -0500759bool Optimizer::isStaticallyIndexedArray(Ice::Operand *allocaAddress)
760{
761 auto &uses = *getUses(allocaAddress);
762
763 for(auto *use : uses)
764 {
765 // Direct load from base address.
766 if(isLoad(*use) && use->getLoadAddress() == allocaAddress)
767 {
768 continue; // This is fine.
769 }
770
771 if(isStore(*use))
772 {
773 // Can either be the address we're storing to, or the data we're storing.
774 if(use->getStoreAddress() == allocaAddress)
775 {
776 continue;
777 }
778 else
779 {
780 // propagateAlloca() eliminates most of the stores of the address itself.
781 // For the cases it doesn't handle, assume SRoA is not feasible.
782 return false;
783 }
784 }
785
786 // Pointer arithmetic is fine if it only uses constants.
787 auto *arithmetic = llvm::dyn_cast<Ice::InstArithmetic>(use);
788 if(arithmetic && arithmetic->getOp() == Ice::InstArithmetic::Add)
789 {
790 auto *rhs = arithmetic->getSrc(1);
791
792 if(llvm::isa<Ice::Constant>(rhs))
793 {
794 continue;
795 }
796 }
797
798 // If there's any other type of use, bail out.
799 return false;
800 }
801
802 return true;
803}
804
Nicolas Capens33a77f72021-02-08 15:04:38 -0500805const Ice::InstIntrinsic *Optimizer::asLoadSubVector(const Ice::Inst *instruction)
Nicolas Capens157ba262019-12-10 17:49:14 -0500806{
Nicolas Capens33a77f72021-02-08 15:04:38 -0500807 if(auto *instrinsic = llvm::dyn_cast<Ice::InstIntrinsic>(instruction))
Nicolas Capens157ba262019-12-10 17:49:14 -0500808 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500809 if(instrinsic->getIntrinsicID() == Ice::Intrinsics::LoadSubVector)
Nicolas Capens157ba262019-12-10 17:49:14 -0500810 {
811 return instrinsic;
812 }
813 }
814
815 return nullptr;
816}
817
Nicolas Capens33a77f72021-02-08 15:04:38 -0500818const Ice::InstIntrinsic *Optimizer::asStoreSubVector(const Ice::Inst *instruction)
Nicolas Capens157ba262019-12-10 17:49:14 -0500819{
Nicolas Capens33a77f72021-02-08 15:04:38 -0500820 if(auto *instrinsic = llvm::dyn_cast<Ice::InstIntrinsic>(instruction))
Nicolas Capens157ba262019-12-10 17:49:14 -0500821 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500822 if(instrinsic->getIntrinsicID() == Ice::Intrinsics::StoreSubVector)
Nicolas Capens157ba262019-12-10 17:49:14 -0500823 {
824 return instrinsic;
825 }
826 }
827
828 return nullptr;
829}
830
831bool Optimizer::isLoad(const Ice::Inst &instruction)
832{
833 if(llvm::isa<Ice::InstLoad>(&instruction))
834 {
835 return true;
836 }
837
838 return asLoadSubVector(&instruction) != nullptr;
839}
840
841bool Optimizer::isStore(const Ice::Inst &instruction)
842{
843 if(llvm::isa<Ice::InstStore>(&instruction))
844 {
845 return true;
846 }
847
848 return asStoreSubVector(&instruction) != nullptr;
849}
850
Nicolas Capens157ba262019-12-10 17:49:14 -0500851std::size_t Optimizer::storeSize(const Ice::Inst *store)
852{
853 assert(isStore(*store));
854
855 if(auto *instStore = llvm::dyn_cast<Ice::InstStore>(store))
856 {
857 return Ice::typeWidthInBytes(instStore->getData()->getType());
858 }
859
860 if(auto *storeSubVector = asStoreSubVector(store))
861 {
862 return llvm::cast<Ice::ConstantInteger32>(storeSubVector->getSrc(3))->getValue();
863 }
864
865 return 0;
866}
867
868bool Optimizer::loadTypeMatchesStore(const Ice::Inst *load, const Ice::Inst *store)
869{
870 if(!load || !store)
871 {
872 return false;
873 }
874
875 assert(isLoad(*load) && isStore(*store));
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500876 assert(load->getLoadAddress() == store->getStoreAddress());
Nicolas Capens157ba262019-12-10 17:49:14 -0500877
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500878 if(store->getData()->getType() != load->getDest()->getType())
Nicolas Capens157ba262019-12-10 17:49:14 -0500879 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500880 return false;
Nicolas Capens157ba262019-12-10 17:49:14 -0500881 }
882
883 if(auto *storeSubVector = asStoreSubVector(store))
884 {
885 if(auto *loadSubVector = asLoadSubVector(load))
886 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500887 // Check for matching sub-vector width.
888 return llvm::cast<Ice::ConstantInteger32>(storeSubVector->getSrc(2))->getValue() ==
889 llvm::cast<Ice::ConstantInteger32>(loadSubVector->getSrc(1))->getValue();
Nicolas Capens157ba262019-12-10 17:49:14 -0500890 }
891 }
892
Nicolas Capens673a7fe2021-02-05 15:03:22 -0500893 return true;
Nicolas Capens157ba262019-12-10 17:49:14 -0500894}
895
Nicolas Capens54313fb2021-02-19 14:26:27 -0500896void Optimizer::collectDiagnostics()
897{
898 if(report)
899 {
900 *report = {};
901
902 for(auto *basicBlock : function->getNodes())
903 {
904 for(auto &inst : basicBlock->getInsts())
905 {
906 if(inst.isDeleted())
907 {
908 continue;
909 }
910
911 if(llvm::isa<Ice::InstAlloca>(inst))
912 {
913 report->allocas++;
914 }
915 else if(isLoad(inst))
916 {
917 report->loads++;
918 }
919 else if(isStore(inst))
920 {
921 report->stores++;
922 }
923 }
924 }
925 }
926}
927
Ben Clayton713b8d32019-12-17 20:37:56 +0000928Optimizer::Uses *Optimizer::getUses(Ice::Operand *operand)
Nicolas Capens157ba262019-12-10 17:49:14 -0500929{
Ben Clayton713b8d32019-12-17 20:37:56 +0000930 Optimizer::Uses *uses = (Optimizer::Uses *)operand->Ice::Operand::getExternalData();
Nicolas Capens157ba262019-12-10 17:49:14 -0500931 if(!uses)
932 {
933 uses = new Optimizer::Uses;
934 setUses(operand, uses);
Antonio Maiorano5ba2a5b2020-01-17 15:29:37 -0500935 operandsWithUses.push_back(operand);
Nicolas Capens157ba262019-12-10 17:49:14 -0500936 }
937 return uses;
938}
939
Ben Clayton713b8d32019-12-17 20:37:56 +0000940void Optimizer::setUses(Ice::Operand *operand, Optimizer::Uses *uses)
Nicolas Capens157ba262019-12-10 17:49:14 -0500941{
Antonio Maiorano614a4d42020-01-29 13:33:02 -0500942 if(auto *oldUses = reinterpret_cast<Optimizer::Uses *>(operand->Ice::Operand::getExternalData()))
943 {
944 delete oldUses;
945 }
946
Nicolas Capens157ba262019-12-10 17:49:14 -0500947 operand->Ice::Operand::setExternalData(uses);
948}
949
Ben Clayton713b8d32019-12-17 20:37:56 +0000950bool Optimizer::hasUses(Ice::Operand *operand) const
Nicolas Capens157ba262019-12-10 17:49:14 -0500951{
952 return operand->Ice::Operand::getExternalData() != nullptr;
953}
954
Ben Clayton713b8d32019-12-17 20:37:56 +0000955Ice::CfgNode *Optimizer::getNode(Ice::Inst *inst)
Nicolas Capens157ba262019-12-10 17:49:14 -0500956{
Ben Clayton713b8d32019-12-17 20:37:56 +0000957 return (Ice::CfgNode *)inst->Ice::Inst::getExternalData();
Nicolas Capens157ba262019-12-10 17:49:14 -0500958}
959
Ben Clayton713b8d32019-12-17 20:37:56 +0000960void Optimizer::setNode(Ice::Inst *inst, Ice::CfgNode *node)
Nicolas Capens157ba262019-12-10 17:49:14 -0500961{
962 inst->Ice::Inst::setExternalData(node);
963}
964
Ben Clayton713b8d32019-12-17 20:37:56 +0000965Ice::Inst *Optimizer::getDefinition(Ice::Variable *var)
Nicolas Capens157ba262019-12-10 17:49:14 -0500966{
Ben Clayton713b8d32019-12-17 20:37:56 +0000967 return (Ice::Inst *)var->Ice::Variable::getExternalData();
Nicolas Capens157ba262019-12-10 17:49:14 -0500968}
969
Ben Clayton713b8d32019-12-17 20:37:56 +0000970void Optimizer::setDefinition(Ice::Variable *var, Ice::Inst *inst)
Nicolas Capens157ba262019-12-10 17:49:14 -0500971{
972 var->Ice::Variable::setExternalData(inst);
973}
974
Ben Clayton713b8d32019-12-17 20:37:56 +0000975const std::vector<Optimizer::LoadStoreInst> &Optimizer::getLoadStoreInsts(Ice::CfgNode *node)
Nicolas Capens157ba262019-12-10 17:49:14 -0500976{
Ben Clayton713b8d32019-12-17 20:37:56 +0000977 return *((const std::vector<LoadStoreInst> *)node->Ice::CfgNode::getExternalData());
Nicolas Capens157ba262019-12-10 17:49:14 -0500978}
979
Ben Clayton713b8d32019-12-17 20:37:56 +0000980void Optimizer::setLoadStoreInsts(Ice::CfgNode *node, std::vector<LoadStoreInst> *insts)
Nicolas Capens157ba262019-12-10 17:49:14 -0500981{
982 node->Ice::CfgNode::setExternalData(insts);
983}
984
Ben Clayton713b8d32019-12-17 20:37:56 +0000985bool Optimizer::hasLoadStoreInsts(Ice::CfgNode *node) const
Nicolas Capens157ba262019-12-10 17:49:14 -0500986{
987 return node->Ice::CfgNode::getExternalData() != nullptr;
988}
989
990bool Optimizer::Uses::areOnlyLoadStore() const
991{
992 return size() == (loads.size() + stores.size());
993}
994
995void Optimizer::Uses::insert(Ice::Operand *value, Ice::Inst *instruction)
996{
997 push_back(instruction);
998
999 if(isLoad(*instruction))
1000 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -05001001 if(value == instruction->getLoadAddress())
Nicolas Capens157ba262019-12-10 17:49:14 -05001002 {
1003 loads.push_back(instruction);
1004 }
1005 }
1006 else if(isStore(*instruction))
1007 {
Nicolas Capens673a7fe2021-02-05 15:03:22 -05001008 if(value == instruction->getStoreAddress())
Nicolas Capens157ba262019-12-10 17:49:14 -05001009 {
1010 stores.push_back(instruction);
1011 }
1012 }
1013}
1014
1015void Optimizer::Uses::erase(Ice::Inst *instruction)
1016{
1017 auto &uses = *this;
1018
1019 for(size_t i = 0; i < uses.size(); i++)
1020 {
1021 if(uses[i] == instruction)
1022 {
1023 uses[i] = back();
1024 pop_back();
1025
1026 for(size_t i = 0; i < loads.size(); i++)
1027 {
1028 if(loads[i] == instruction)
1029 {
1030 loads[i] = loads.back();
1031 loads.pop_back();
1032 break;
1033 }
1034 }
1035
1036 for(size_t i = 0; i < stores.size(); i++)
1037 {
1038 if(stores[i] == instruction)
1039 {
1040 stores[i] = stores.back();
1041 stores.pop_back();
1042 break;
1043 }
1044 }
1045
1046 break;
1047 }
1048 }
1049}
1050
Ben Clayton713b8d32019-12-17 20:37:56 +00001051} // anonymous namespace
Nicolas Capens157ba262019-12-10 17:49:14 -05001052
1053namespace rr {
1054
Nicolas Capens54313fb2021-02-19 14:26:27 -05001055void optimize(Ice::Cfg *function, Nucleus::OptimizerReport *report)
Nicolas Capens157ba262019-12-10 17:49:14 -05001056{
Nicolas Capens54313fb2021-02-19 14:26:27 -05001057 Optimizer optimizer(report);
Nicolas Capens157ba262019-12-10 17:49:14 -05001058
1059 optimizer.run(function);
1060}
1061
Antonio Maiorano614a4d42020-01-29 13:33:02 -05001062} // namespace rr