Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 1 | // Copyright (c) 2017 The Khronos Group Inc. |
| 2 | // Copyright (c) 2017 Valve Corporation |
| 3 | // Copyright (c) 2017 LunarG Inc. |
| 4 | // |
| 5 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | // you may not use this file except in compliance with the License. |
| 7 | // You may obtain a copy of the License at |
| 8 | // |
| 9 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | // |
| 11 | // Unless required by applicable law or agreed to in writing, software |
| 12 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | // See the License for the specific language governing permissions and |
| 15 | // limitations under the License. |
| 16 | |
| 17 | #include "inline_pass.h" |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 18 | |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 19 | #include "cfa.h" |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 20 | |
| 21 | // Indices of operands in SPIR-V instructions |
| 22 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 23 | static const int kSpvFunctionCallFunctionId = 2; |
| 24 | static const int kSpvFunctionCallArgumentId = 3; |
| 25 | static const int kSpvReturnValueId = 0; |
| 26 | static const int kSpvTypePointerStorageClass = 1; |
| 27 | static const int kSpvTypePointerTypeId = 2; |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 28 | static const int kSpvLoopMergeMergeBlockId = 0; |
| 29 | static const int kSpvSelectionMergeMergeBlockId = 0; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 30 | |
| 31 | namespace spvtools { |
| 32 | namespace opt { |
| 33 | |
| 34 | uint32_t InlinePass::FindPointerToType(uint32_t type_id, |
| 35 | SpvStorageClass storage_class) { |
| 36 | ir::Module::inst_iterator type_itr = module_->types_values_begin(); |
| 37 | for (; type_itr != module_->types_values_end(); ++type_itr) { |
| 38 | const ir::Instruction* type_inst = &*type_itr; |
| 39 | if (type_inst->opcode() == SpvOpTypePointer && |
| 40 | type_inst->GetSingleWordOperand(kSpvTypePointerTypeId) == type_id && |
| 41 | type_inst->GetSingleWordOperand(kSpvTypePointerStorageClass) == |
| 42 | storage_class) |
| 43 | return type_inst->result_id(); |
| 44 | } |
| 45 | return 0; |
| 46 | } |
| 47 | |
| 48 | uint32_t InlinePass::AddPointerToType(uint32_t type_id, |
| 49 | SpvStorageClass storage_class) { |
| 50 | uint32_t resultId = TakeNextId(); |
| 51 | std::unique_ptr<ir::Instruction> type_inst(new ir::Instruction( |
| 52 | SpvOpTypePointer, 0, resultId, |
| 53 | {{spv_operand_type_t::SPV_OPERAND_TYPE_STORAGE_CLASS, |
| 54 | {uint32_t(storage_class)}}, |
| 55 | {spv_operand_type_t::SPV_OPERAND_TYPE_ID, {type_id}}})); |
| 56 | module_->AddType(std::move(type_inst)); |
| 57 | return resultId; |
| 58 | } |
| 59 | |
| 60 | void InlinePass::AddBranch(uint32_t label_id, |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 61 | std::unique_ptr<ir::BasicBlock>* block_ptr) { |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 62 | std::unique_ptr<ir::Instruction> newBranch(new ir::Instruction( |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 63 | SpvOpBranch, 0, 0, |
| 64 | {{spv_operand_type_t::SPV_OPERAND_TYPE_ID, {label_id}}})); |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 65 | (*block_ptr)->AddInstruction(std::move(newBranch)); |
| 66 | } |
| 67 | |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 68 | void InlinePass::AddBranchCond(uint32_t cond_id, uint32_t true_id, |
| 69 | uint32_t false_id, std::unique_ptr<ir::BasicBlock>* block_ptr) { |
| 70 | std::unique_ptr<ir::Instruction> newBranch(new ir::Instruction( |
| 71 | SpvOpBranchConditional, 0, 0, |
| 72 | {{spv_operand_type_t::SPV_OPERAND_TYPE_ID, {cond_id}}, |
| 73 | {spv_operand_type_t::SPV_OPERAND_TYPE_ID, {true_id}}, |
| 74 | {spv_operand_type_t::SPV_OPERAND_TYPE_ID, {false_id}}})); |
| 75 | (*block_ptr)->AddInstruction(std::move(newBranch)); |
| 76 | } |
| 77 | |
| 78 | void InlinePass::AddLoopMerge(uint32_t merge_id, uint32_t continue_id, |
| 79 | std::unique_ptr<ir::BasicBlock>* block_ptr) { |
| 80 | std::unique_ptr<ir::Instruction> newLoopMerge(new ir::Instruction( |
| 81 | SpvOpLoopMerge, 0, 0, |
| 82 | {{spv_operand_type_t::SPV_OPERAND_TYPE_ID, {merge_id}}, |
| 83 | {spv_operand_type_t::SPV_OPERAND_TYPE_ID, {continue_id}}, |
| 84 | {spv_operand_type_t::SPV_OPERAND_TYPE_LOOP_CONTROL, {0}}})); |
| 85 | (*block_ptr)->AddInstruction(std::move(newLoopMerge)); |
| 86 | } |
| 87 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 88 | void InlinePass::AddStore(uint32_t ptr_id, uint32_t val_id, |
| 89 | std::unique_ptr<ir::BasicBlock>* block_ptr) { |
| 90 | std::unique_ptr<ir::Instruction> newStore(new ir::Instruction( |
| 91 | SpvOpStore, 0, 0, {{spv_operand_type_t::SPV_OPERAND_TYPE_ID, {ptr_id}}, |
| 92 | {spv_operand_type_t::SPV_OPERAND_TYPE_ID, {val_id}}})); |
| 93 | (*block_ptr)->AddInstruction(std::move(newStore)); |
| 94 | } |
| 95 | |
| 96 | void InlinePass::AddLoad(uint32_t type_id, uint32_t resultId, uint32_t ptr_id, |
| 97 | std::unique_ptr<ir::BasicBlock>* block_ptr) { |
| 98 | std::unique_ptr<ir::Instruction> newLoad(new ir::Instruction( |
| 99 | SpvOpLoad, type_id, resultId, |
| 100 | {{spv_operand_type_t::SPV_OPERAND_TYPE_ID, {ptr_id}}})); |
| 101 | (*block_ptr)->AddInstruction(std::move(newLoad)); |
| 102 | } |
| 103 | |
| 104 | std::unique_ptr<ir::Instruction> InlinePass::NewLabel(uint32_t label_id) { |
| 105 | std::unique_ptr<ir::Instruction> newLabel( |
| 106 | new ir::Instruction(SpvOpLabel, 0, label_id, {})); |
| 107 | return newLabel; |
| 108 | } |
| 109 | |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 110 | uint32_t InlinePass::GetFalseId() { |
| 111 | if (false_id_ != 0) |
| 112 | return false_id_; |
| 113 | false_id_ = module_->GetGlobalValue(SpvOpConstantFalse); |
| 114 | if (false_id_ != 0) |
| 115 | return false_id_; |
| 116 | uint32_t boolId = module_->GetGlobalValue(SpvOpTypeBool); |
| 117 | if (boolId == 0) { |
| 118 | boolId = TakeNextId(); |
| 119 | module_->AddGlobalValue(SpvOpTypeBool, boolId, 0); |
| 120 | } |
| 121 | false_id_ = TakeNextId(); |
| 122 | module_->AddGlobalValue(SpvOpConstantFalse, false_id_, boolId); |
| 123 | return false_id_; |
| 124 | } |
| 125 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 126 | void InlinePass::MapParams( |
| 127 | ir::Function* calleeFn, |
| 128 | ir::UptrVectorIterator<ir::Instruction> call_inst_itr, |
| 129 | std::unordered_map<uint32_t, uint32_t>* callee2caller) { |
| 130 | int param_idx = 0; |
| 131 | calleeFn->ForEachParam( |
| 132 | [&call_inst_itr, ¶m_idx, &callee2caller](const ir::Instruction* cpi) { |
| 133 | const uint32_t pid = cpi->result_id(); |
| 134 | (*callee2caller)[pid] = call_inst_itr->GetSingleWordOperand( |
| 135 | kSpvFunctionCallArgumentId + param_idx); |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 136 | ++param_idx; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 137 | }); |
| 138 | } |
| 139 | |
| 140 | void InlinePass::CloneAndMapLocals( |
| 141 | ir::Function* calleeFn, |
| 142 | std::vector<std::unique_ptr<ir::Instruction>>* new_vars, |
| 143 | std::unordered_map<uint32_t, uint32_t>* callee2caller) { |
| 144 | auto callee_block_itr = calleeFn->begin(); |
| 145 | auto callee_var_itr = callee_block_itr->begin(); |
| 146 | while (callee_var_itr->opcode() == SpvOp::SpvOpVariable) { |
| 147 | std::unique_ptr<ir::Instruction> var_inst( |
| 148 | new ir::Instruction(*callee_var_itr)); |
| 149 | uint32_t newId = TakeNextId(); |
| 150 | var_inst->SetResultId(newId); |
| 151 | (*callee2caller)[callee_var_itr->result_id()] = newId; |
| 152 | new_vars->push_back(std::move(var_inst)); |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 153 | ++callee_var_itr; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 154 | } |
| 155 | } |
| 156 | |
| 157 | uint32_t InlinePass::CreateReturnVar( |
| 158 | ir::Function* calleeFn, |
| 159 | std::vector<std::unique_ptr<ir::Instruction>>* new_vars) { |
| 160 | uint32_t returnVarId = 0; |
| 161 | const uint32_t calleeTypeId = calleeFn->type_id(); |
| 162 | const ir::Instruction* calleeType = |
| 163 | def_use_mgr_->id_to_defs().find(calleeTypeId)->second; |
| 164 | if (calleeType->opcode() != SpvOpTypeVoid) { |
| 165 | // Find or create ptr to callee return type. |
| 166 | uint32_t returnVarTypeId = |
| 167 | FindPointerToType(calleeTypeId, SpvStorageClassFunction); |
| 168 | if (returnVarTypeId == 0) |
| 169 | returnVarTypeId = AddPointerToType(calleeTypeId, SpvStorageClassFunction); |
| 170 | // Add return var to new function scope variables. |
| 171 | returnVarId = TakeNextId(); |
| 172 | std::unique_ptr<ir::Instruction> var_inst(new ir::Instruction( |
| 173 | SpvOpVariable, returnVarTypeId, returnVarId, |
| 174 | {{spv_operand_type_t::SPV_OPERAND_TYPE_STORAGE_CLASS, |
| 175 | {SpvStorageClassFunction}}})); |
| 176 | new_vars->push_back(std::move(var_inst)); |
| 177 | } |
| 178 | return returnVarId; |
| 179 | } |
| 180 | |
| 181 | bool InlinePass::IsSameBlockOp(const ir::Instruction* inst) const { |
| 182 | return inst->opcode() == SpvOpSampledImage || inst->opcode() == SpvOpImage; |
| 183 | } |
| 184 | |
| 185 | void InlinePass::CloneSameBlockOps( |
| 186 | std::unique_ptr<ir::Instruction>* inst, |
| 187 | std::unordered_map<uint32_t, uint32_t>* postCallSB, |
| 188 | std::unordered_map<uint32_t, ir::Instruction*>* preCallSB, |
| 189 | std::unique_ptr<ir::BasicBlock>* block_ptr) { |
| 190 | (*inst) |
| 191 | ->ForEachInId([&postCallSB, &preCallSB, &block_ptr, this](uint32_t* iid) { |
| 192 | const auto mapItr = (*postCallSB).find(*iid); |
| 193 | if (mapItr == (*postCallSB).end()) { |
| 194 | const auto mapItr2 = (*preCallSB).find(*iid); |
| 195 | if (mapItr2 != (*preCallSB).end()) { |
| 196 | // Clone pre-call same-block ops, map result id. |
| 197 | const ir::Instruction* inInst = mapItr2->second; |
| 198 | std::unique_ptr<ir::Instruction> sb_inst( |
| 199 | new ir::Instruction(*inInst)); |
| 200 | CloneSameBlockOps(&sb_inst, postCallSB, preCallSB, block_ptr); |
| 201 | const uint32_t rid = sb_inst->result_id(); |
| 202 | const uint32_t nid = this->TakeNextId(); |
| 203 | sb_inst->SetResultId(nid); |
| 204 | (*postCallSB)[rid] = nid; |
| 205 | *iid = nid; |
| 206 | (*block_ptr)->AddInstruction(std::move(sb_inst)); |
| 207 | } |
| 208 | } else { |
| 209 | // Reset same-block op operand. |
| 210 | *iid = mapItr->second; |
| 211 | } |
| 212 | }); |
| 213 | } |
| 214 | |
| 215 | void InlinePass::GenInlineCode( |
| 216 | std::vector<std::unique_ptr<ir::BasicBlock>>* new_blocks, |
| 217 | std::vector<std::unique_ptr<ir::Instruction>>* new_vars, |
| 218 | ir::UptrVectorIterator<ir::Instruction> call_inst_itr, |
| 219 | ir::UptrVectorIterator<ir::BasicBlock> call_block_itr) { |
| 220 | // Map from all ids in the callee to their equivalent id in the caller |
| 221 | // as callee instructions are copied into caller. |
| 222 | std::unordered_map<uint32_t, uint32_t> callee2caller; |
| 223 | // Pre-call same-block insts |
| 224 | std::unordered_map<uint32_t, ir::Instruction*> preCallSB; |
| 225 | // Post-call same-block op ids |
| 226 | std::unordered_map<uint32_t, uint32_t> postCallSB; |
| 227 | |
| 228 | ir::Function* calleeFn = id2function_[call_inst_itr->GetSingleWordOperand( |
| 229 | kSpvFunctionCallFunctionId)]; |
| 230 | |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 231 | // Check for multiple returns in the callee. |
| 232 | auto fi = multi_return_funcs_.find(calleeFn->result_id()); |
| 233 | const bool multiReturn = fi != multi_return_funcs_.end(); |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 234 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 235 | // Map parameters to actual arguments. |
| 236 | MapParams(calleeFn, call_inst_itr, &callee2caller); |
| 237 | |
| 238 | // Define caller local variables for all callee variables and create map to |
| 239 | // them. |
| 240 | CloneAndMapLocals(calleeFn, new_vars, &callee2caller); |
| 241 | |
| 242 | // Create return var if needed. |
| 243 | uint32_t returnVarId = CreateReturnVar(calleeFn, new_vars); |
| 244 | |
GregF | a699d1a | 2017-08-29 18:35:05 -0600 | [diff] [blame] | 245 | // Create set of callee result ids. Used to detect forward references |
| 246 | std::unordered_set<uint32_t> callee_result_ids; |
| 247 | calleeFn->ForEachInst([&callee_result_ids]( |
| 248 | const ir::Instruction* cpi) { |
| 249 | const uint32_t rid = cpi->result_id(); |
| 250 | if (rid != 0) |
| 251 | callee_result_ids.insert(rid); |
| 252 | }); |
| 253 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 254 | // Clone and map callee code. Copy caller block code to beginning of |
| 255 | // first block and end of last block. |
| 256 | bool prevInstWasReturn = false; |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 257 | uint32_t singleTripLoopHeaderId = 0; |
| 258 | uint32_t singleTripLoopContinueId = 0; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 259 | uint32_t returnLabelId = 0; |
| 260 | bool multiBlocks = false; |
| 261 | const uint32_t calleeTypeId = calleeFn->type_id(); |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 262 | // new_blk_ptr is a new basic block in the caller. New instructions are |
| 263 | // written to it. It is created when we encounter the OpLabel |
| 264 | // of the first callee block. |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 265 | std::unique_ptr<ir::BasicBlock> new_blk_ptr; |
| 266 | calleeFn->ForEachInst([&new_blocks, &callee2caller, &call_block_itr, |
| 267 | &call_inst_itr, &new_blk_ptr, &prevInstWasReturn, |
| 268 | &returnLabelId, &returnVarId, &calleeTypeId, |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 269 | &multiBlocks, &postCallSB, &preCallSB, multiReturn, |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 270 | &singleTripLoopHeaderId, &singleTripLoopContinueId, |
GregF | a699d1a | 2017-08-29 18:35:05 -0600 | [diff] [blame] | 271 | &callee_result_ids, this]( |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 272 | const ir::Instruction* cpi) { |
| 273 | switch (cpi->opcode()) { |
| 274 | case SpvOpFunction: |
| 275 | case SpvOpFunctionParameter: |
| 276 | case SpvOpVariable: |
| 277 | // Already processed |
| 278 | break; |
| 279 | case SpvOpLabel: { |
| 280 | // If previous instruction was early return, insert branch |
| 281 | // instruction to return block. |
| 282 | if (prevInstWasReturn) { |
| 283 | if (returnLabelId == 0) returnLabelId = this->TakeNextId(); |
| 284 | AddBranch(returnLabelId, &new_blk_ptr); |
| 285 | prevInstWasReturn = false; |
| 286 | } |
| 287 | // Finish current block (if it exists) and get label for next block. |
| 288 | uint32_t labelId; |
| 289 | bool firstBlock = false; |
| 290 | if (new_blk_ptr != nullptr) { |
| 291 | new_blocks->push_back(std::move(new_blk_ptr)); |
| 292 | // If result id is already mapped, use it, otherwise get a new |
| 293 | // one. |
| 294 | const uint32_t rid = cpi->result_id(); |
| 295 | const auto mapItr = callee2caller.find(rid); |
| 296 | labelId = (mapItr != callee2caller.end()) ? mapItr->second |
| 297 | : this->TakeNextId(); |
| 298 | } else { |
| 299 | // First block needs to use label of original block |
| 300 | // but map callee label in case of phi reference. |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 301 | labelId = call_block_itr->id(); |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 302 | callee2caller[cpi->result_id()] = labelId; |
| 303 | firstBlock = true; |
| 304 | } |
| 305 | // Create first/next block. |
| 306 | new_blk_ptr.reset(new ir::BasicBlock(NewLabel(labelId))); |
| 307 | if (firstBlock) { |
| 308 | // Copy contents of original caller block up to call instruction. |
| 309 | for (auto cii = call_block_itr->begin(); cii != call_inst_itr; |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 310 | ++cii) { |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 311 | std::unique_ptr<ir::Instruction> cp_inst(new ir::Instruction(*cii)); |
| 312 | // Remember same-block ops for possible regeneration. |
| 313 | if (IsSameBlockOp(&*cp_inst)) { |
| 314 | auto* sb_inst_ptr = cp_inst.get(); |
| 315 | preCallSB[cp_inst->result_id()] = sb_inst_ptr; |
| 316 | } |
| 317 | new_blk_ptr->AddInstruction(std::move(cp_inst)); |
| 318 | } |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 319 | // If callee has multiple returns, insert header block for |
| 320 | // single-trip loop that will encompass callee code. Start postheader |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 321 | // block. |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 322 | if (multiReturn) { |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 323 | singleTripLoopHeaderId = this->TakeNextId(); |
| 324 | AddBranch(singleTripLoopHeaderId, &new_blk_ptr); |
| 325 | new_blocks->push_back(std::move(new_blk_ptr)); |
| 326 | new_blk_ptr.reset(new ir::BasicBlock(NewLabel( |
| 327 | singleTripLoopHeaderId))); |
| 328 | returnLabelId = this->TakeNextId(); |
| 329 | singleTripLoopContinueId = this->TakeNextId(); |
| 330 | AddLoopMerge(returnLabelId, singleTripLoopContinueId, &new_blk_ptr); |
| 331 | uint32_t postHeaderId = this->TakeNextId(); |
| 332 | AddBranch(postHeaderId, &new_blk_ptr); |
| 333 | new_blocks->push_back(std::move(new_blk_ptr)); |
| 334 | new_blk_ptr.reset(new ir::BasicBlock(NewLabel(postHeaderId))); |
| 335 | multiBlocks = true; |
| 336 | } |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 337 | } else { |
| 338 | multiBlocks = true; |
| 339 | } |
| 340 | } break; |
| 341 | case SpvOpReturnValue: { |
| 342 | // Store return value to return variable. |
| 343 | assert(returnVarId != 0); |
| 344 | uint32_t valId = cpi->GetInOperand(kSpvReturnValueId).words[0]; |
| 345 | const auto mapItr = callee2caller.find(valId); |
| 346 | if (mapItr != callee2caller.end()) { |
| 347 | valId = mapItr->second; |
| 348 | } |
| 349 | AddStore(returnVarId, valId, &new_blk_ptr); |
| 350 | |
| 351 | // Remember we saw a return; if followed by a label, will need to |
| 352 | // insert branch. |
| 353 | prevInstWasReturn = true; |
| 354 | } break; |
| 355 | case SpvOpReturn: { |
| 356 | // Remember we saw a return; if followed by a label, will need to |
| 357 | // insert branch. |
| 358 | prevInstWasReturn = true; |
| 359 | } break; |
| 360 | case SpvOpFunctionEnd: { |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 361 | // If there was an early return, we generated a return label id |
| 362 | // for it. Now we have to generate the return block with that Id. |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 363 | if (returnLabelId != 0) { |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 364 | // If previous instruction was return, insert branch instruction |
| 365 | // to return block. |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 366 | if (prevInstWasReturn) AddBranch(returnLabelId, &new_blk_ptr); |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 367 | if (multiReturn) { |
| 368 | // If we generated a loop header to for the single-trip loop |
| 369 | // to accommodate multiple returns, insert the continue |
| 370 | // target block now, with a false branch back to the loop header. |
| 371 | new_blocks->push_back(std::move(new_blk_ptr)); |
| 372 | new_blk_ptr.reset( |
| 373 | new ir::BasicBlock(NewLabel(singleTripLoopContinueId))); |
| 374 | AddBranchCond(GetFalseId(), singleTripLoopHeaderId, returnLabelId, |
| 375 | &new_blk_ptr); |
| 376 | } |
| 377 | // Generate the return block. |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 378 | new_blocks->push_back(std::move(new_blk_ptr)); |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 379 | new_blk_ptr.reset(new ir::BasicBlock(NewLabel(returnLabelId))); |
| 380 | multiBlocks = true; |
| 381 | } |
| 382 | // Load return value into result id of call, if it exists. |
| 383 | if (returnVarId != 0) { |
| 384 | const uint32_t resId = call_inst_itr->result_id(); |
| 385 | assert(resId != 0); |
| 386 | AddLoad(calleeTypeId, resId, returnVarId, &new_blk_ptr); |
| 387 | } |
| 388 | // Copy remaining instructions from caller block. |
| 389 | auto cii = call_inst_itr; |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 390 | for (++cii; cii != call_block_itr->end(); ++cii) { |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 391 | std::unique_ptr<ir::Instruction> cp_inst(new ir::Instruction(*cii)); |
| 392 | // If multiple blocks generated, regenerate any same-block |
| 393 | // instruction that has not been seen in this last block. |
| 394 | if (multiBlocks) { |
| 395 | CloneSameBlockOps(&cp_inst, &postCallSB, &preCallSB, &new_blk_ptr); |
| 396 | // Remember same-block ops in this block. |
| 397 | if (IsSameBlockOp(&*cp_inst)) { |
| 398 | const uint32_t rid = cp_inst->result_id(); |
| 399 | postCallSB[rid] = rid; |
| 400 | } |
| 401 | } |
| 402 | new_blk_ptr->AddInstruction(std::move(cp_inst)); |
| 403 | } |
| 404 | // Finalize inline code. |
| 405 | new_blocks->push_back(std::move(new_blk_ptr)); |
| 406 | } break; |
| 407 | default: { |
| 408 | // Copy callee instruction and remap all input Ids. |
| 409 | std::unique_ptr<ir::Instruction> cp_inst(new ir::Instruction(*cpi)); |
GregF | a699d1a | 2017-08-29 18:35:05 -0600 | [diff] [blame] | 410 | cp_inst->ForEachInId([&callee2caller, &cpi, &callee_result_ids, this] |
| 411 | (uint32_t* iid) { |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 412 | const auto mapItr = callee2caller.find(*iid); |
| 413 | if (mapItr != callee2caller.end()) { |
| 414 | *iid = mapItr->second; |
GregF | a699d1a | 2017-08-29 18:35:05 -0600 | [diff] [blame] | 415 | } else if (callee_result_ids.find(*iid) != callee_result_ids.end()) { |
| 416 | // Forward reference. Allocate a new id, map it, |
| 417 | // use it and check for it when remapping result ids |
| 418 | const uint32_t nid = this->TakeNextId(); |
| 419 | callee2caller[*iid] = nid; |
| 420 | *iid = nid; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 421 | } |
| 422 | }); |
GregF | a699d1a | 2017-08-29 18:35:05 -0600 | [diff] [blame] | 423 | // If result id is non-zero, remap it. If already mapped, use mapped |
| 424 | // value, else use next id. |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 425 | const uint32_t rid = cp_inst->result_id(); |
| 426 | if (rid != 0) { |
GregF | a699d1a | 2017-08-29 18:35:05 -0600 | [diff] [blame] | 427 | const auto mapItr = callee2caller.find(rid); |
| 428 | uint32_t nid; |
| 429 | if (mapItr != callee2caller.end()) { |
| 430 | nid = mapItr->second; |
| 431 | } |
| 432 | else { |
| 433 | nid = this->TakeNextId(); |
| 434 | callee2caller[rid] = nid; |
| 435 | } |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 436 | cp_inst->SetResultId(nid); |
| 437 | } |
| 438 | new_blk_ptr->AddInstruction(std::move(cp_inst)); |
| 439 | } break; |
| 440 | } |
| 441 | }); |
| 442 | // Update block map given replacement blocks. |
| 443 | for (auto& blk : *new_blocks) { |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 444 | id2block_[blk->id()] = &*blk; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 445 | } |
| 446 | } |
| 447 | |
David Neto | ceb1d4f | 2017-03-31 10:36:58 -0400 | [diff] [blame] | 448 | bool InlinePass::IsInlinableFunctionCall(const ir::Instruction* inst) { |
| 449 | if (inst->opcode() != SpvOp::SpvOpFunctionCall) return false; |
GregF | a107d34 | 2017-04-25 13:57:20 -0600 | [diff] [blame] | 450 | const uint32_t calleeFnId = |
| 451 | inst->GetSingleWordOperand(kSpvFunctionCallFunctionId); |
| 452 | const auto ci = inlinable_.find(calleeFnId); |
| 453 | return ci != inlinable_.cend(); |
David Neto | ceb1d4f | 2017-03-31 10:36:58 -0400 | [diff] [blame] | 454 | } |
| 455 | |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 456 | void InlinePass::UpdateSucceedingPhis( |
| 457 | std::vector<std::unique_ptr<ir::BasicBlock>>& new_blocks) { |
| 458 | const auto firstBlk = new_blocks.begin(); |
| 459 | const auto lastBlk = new_blocks.end() - 1; |
| 460 | const uint32_t firstId = (*firstBlk)->id(); |
| 461 | const uint32_t lastId = (*lastBlk)->id(); |
| 462 | (*lastBlk)->ForEachSuccessorLabel( |
| 463 | [&firstId, &lastId, this](uint32_t succ) { |
| 464 | ir::BasicBlock* sbp = this->id2block_[succ]; |
| 465 | sbp->ForEachPhiInst([&firstId, &lastId](ir::Instruction* phi) { |
| 466 | phi->ForEachInId([&firstId, &lastId](uint32_t* id) { |
| 467 | if (*id == firstId) *id = lastId; |
| 468 | }); |
| 469 | }); |
| 470 | }); |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 471 | } |
| 472 | |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 473 | bool InlinePass::HasMultipleReturns(ir::Function* func) { |
| 474 | bool seenReturn = false; |
| 475 | bool multipleReturns = false; |
| 476 | for (auto& blk : *func) { |
| 477 | auto terminal_ii = blk.cend(); |
| 478 | --terminal_ii; |
| 479 | if (terminal_ii->opcode() == SpvOpReturn || |
| 480 | terminal_ii->opcode() == SpvOpReturnValue) { |
| 481 | if (seenReturn) { |
| 482 | multipleReturns = true; |
| 483 | break; |
| 484 | } |
| 485 | seenReturn = true; |
| 486 | } |
| 487 | } |
| 488 | return multipleReturns; |
| 489 | } |
| 490 | |
| 491 | uint32_t InlinePass::MergeBlockIdIfAny(const ir::BasicBlock& blk) { |
| 492 | auto merge_ii = blk.cend(); |
| 493 | --merge_ii; |
| 494 | uint32_t mbid = 0; |
| 495 | if (merge_ii != blk.cbegin()) { |
| 496 | --merge_ii; |
| 497 | if (merge_ii->opcode() == SpvOpLoopMerge) |
| 498 | mbid = merge_ii->GetSingleWordOperand(kSpvLoopMergeMergeBlockId); |
| 499 | else if (merge_ii->opcode() == SpvOpSelectionMerge) |
| 500 | mbid = merge_ii->GetSingleWordOperand(kSpvSelectionMergeMergeBlockId); |
| 501 | } |
| 502 | return mbid; |
| 503 | } |
| 504 | |
| 505 | void InlinePass::ComputeStructuredSuccessors(ir::Function* func) { |
| 506 | // If header, make merge block first successor. |
| 507 | for (auto& blk : *func) { |
| 508 | uint32_t mbid = MergeBlockIdIfAny(blk); |
| 509 | if (mbid != 0) |
| 510 | block2structured_succs_[&blk].push_back(id2block_[mbid]); |
| 511 | // add true successors |
| 512 | blk.ForEachSuccessorLabel([&blk, this](uint32_t sbid) { |
| 513 | block2structured_succs_[&blk].push_back(id2block_[sbid]); |
| 514 | }); |
| 515 | } |
| 516 | } |
| 517 | |
| 518 | InlinePass::GetBlocksFunction InlinePass::StructuredSuccessorsFunction() { |
| 519 | return [this](const ir::BasicBlock* block) { |
| 520 | return &(block2structured_succs_[block]); |
| 521 | }; |
| 522 | } |
| 523 | |
| 524 | bool InlinePass::HasNoReturnInLoop(ir::Function* func) { |
| 525 | // If control not structured, do not do loop/return analysis |
| 526 | // TODO: Analyze returns in non-structured control flow |
| 527 | if (!module_->HasCapability(SpvCapabilityShader)) |
| 528 | return false; |
| 529 | // Compute structured block order. This order has the property |
| 530 | // that dominators are before all blocks they dominate and merge blocks |
| 531 | // are after all blocks that are in the control constructs of their header. |
| 532 | ComputeStructuredSuccessors(func); |
| 533 | auto ignore_block = [](cbb_ptr) {}; |
| 534 | auto ignore_edge = [](cbb_ptr, cbb_ptr) {}; |
| 535 | std::list<const ir::BasicBlock*> structuredOrder; |
| 536 | spvtools::CFA<ir::BasicBlock>::DepthFirstTraversal( |
| 537 | &*func->begin(), StructuredSuccessorsFunction(), ignore_block, |
| 538 | [&](cbb_ptr b) { structuredOrder.push_front(b); }, ignore_edge); |
| 539 | // Search for returns in loops. Only need to track outermost loop |
| 540 | bool return_in_loop = false; |
| 541 | uint32_t outerLoopMergeId = 0; |
| 542 | for (auto& blk : structuredOrder) { |
| 543 | // Exiting current outer loop |
| 544 | if (blk->id() == outerLoopMergeId) |
| 545 | outerLoopMergeId = 0; |
| 546 | // Return block |
| 547 | auto terminal_ii = blk->cend(); |
| 548 | --terminal_ii; |
| 549 | if (terminal_ii->opcode() == SpvOpReturn || |
| 550 | terminal_ii->opcode() == SpvOpReturnValue) { |
| 551 | if (outerLoopMergeId != 0) { |
| 552 | return_in_loop = true; |
| 553 | break; |
| 554 | } |
| 555 | } |
| 556 | else if (terminal_ii != blk->cbegin()) { |
| 557 | auto merge_ii = terminal_ii; |
| 558 | --merge_ii; |
| 559 | // Entering outermost loop |
| 560 | if (merge_ii->opcode() == SpvOpLoopMerge && outerLoopMergeId == 0) |
| 561 | outerLoopMergeId = merge_ii->GetSingleWordOperand( |
| 562 | kSpvLoopMergeMergeBlockId); |
| 563 | } |
| 564 | } |
| 565 | return !return_in_loop; |
| 566 | } |
| 567 | |
| 568 | void InlinePass::AnalyzeReturns(ir::Function* func) { |
| 569 | // Look for multiple returns |
| 570 | if (!HasMultipleReturns(func)) { |
| 571 | no_return_in_loop_.insert(func->result_id()); |
| 572 | return; |
| 573 | } |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 574 | multi_return_funcs_.insert(func->result_id()); |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 575 | // If multiple returns, see if any are in a loop |
| 576 | if (HasNoReturnInLoop(func)) |
| 577 | no_return_in_loop_.insert(func->result_id()); |
| 578 | } |
| 579 | |
| 580 | bool InlinePass::IsInlinableFunction(ir::Function* func) { |
GregF | a107d34 | 2017-04-25 13:57:20 -0600 | [diff] [blame] | 581 | // We can only inline a function if it has blocks. |
| 582 | if (func->cbegin() == func->cend()) |
| 583 | return false; |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 584 | // Do not inline functions with returns in loops. Currently early return |
| 585 | // functions are inlined by wrapping them in a one trip loop and implementing |
| 586 | // the returns as a branch to the loop's merge block. However, this can only |
| 587 | // done validly if the return was not in a loop in the original function. |
| 588 | // Also remember functions with multiple (early) returns. |
| 589 | AnalyzeReturns(func); |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 590 | return no_return_in_loop_.find(func->result_id()) != |
| 591 | no_return_in_loop_.cend(); |
GregF | a107d34 | 2017-04-25 13:57:20 -0600 | [diff] [blame] | 592 | } |
| 593 | |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 594 | void InlinePass::InitializeInline(ir::Module* module) { |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 595 | def_use_mgr_.reset(new analysis::DefUseManager(consumer(), module)); |
| 596 | |
| 597 | // Initialize next unused Id. |
| 598 | next_id_ = module->id_bound(); |
| 599 | |
| 600 | // Save module. |
| 601 | module_ = module; |
| 602 | |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 603 | false_id_ = 0; |
| 604 | |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 605 | // clear collections |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 606 | id2function_.clear(); |
| 607 | id2block_.clear(); |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 608 | block2structured_succs_.clear(); |
GregF | a107d34 | 2017-04-25 13:57:20 -0600 | [diff] [blame] | 609 | inlinable_.clear(); |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 610 | no_return_in_loop_.clear(); |
David Neto | 2a1014b | 2017-08-09 14:59:04 -0400 | [diff] [blame] | 611 | multi_return_funcs_.clear(); |
GregF | e28bd39 | 2017-08-01 17:20:13 -0600 | [diff] [blame] | 612 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 613 | for (auto& fn : *module_) { |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 614 | // Initialize function and block maps. |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 615 | id2function_[fn.result_id()] = &fn; |
| 616 | for (auto& blk : fn) { |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 617 | id2block_[blk.id()] = &blk; |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 618 | } |
Greg Fischer | bba812f | 2017-05-04 20:55:53 -0600 | [diff] [blame] | 619 | // Compute inlinability |
GregF | a107d34 | 2017-04-25 13:57:20 -0600 | [diff] [blame] | 620 | if (IsInlinableFunction(&fn)) |
| 621 | inlinable_.insert(fn.result_id()); |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 622 | } |
| 623 | }; |
| 624 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 625 | |
| 626 | InlinePass::InlinePass() |
| 627 | : module_(nullptr), def_use_mgr_(nullptr), next_id_(0) {} |
| 628 | |
Greg Fischer | 04fcc66 | 2016-11-10 10:11:50 -0700 | [diff] [blame] | 629 | } // namespace opt |
| 630 | } // namespace spvtools |