blob: eef434ac64cb2aef9c478b3b4d96a4d6c624391c [file] [log] [blame]
John Zulauf9cb530d2019-09-30 14:14:10 -06001/* Copyright (c) 2019 The Khronos Group Inc.
2 * Copyright (c) 2019 Valve Corporation
3 * Copyright (c) 2019 LunarG, Inc.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 *
17 * Author: John Zulauf <jzulauf@lunarg.com>
18 */
19
20#include <limits>
21#include <vector>
22#include "synchronization_validation.h"
23
24static const char *string_SyncHazardVUID(SyncHazard hazard) {
25 switch (hazard) {
26 case SyncHazard::NONE:
John Zulauf2f952d22020-02-10 11:34:51 -070027 return "SYNC-HAZARD-NONE";
John Zulauf9cb530d2019-09-30 14:14:10 -060028 break;
29 case SyncHazard::READ_AFTER_WRITE:
30 return "SYNC-HAZARD-READ_AFTER_WRITE";
31 break;
32 case SyncHazard::WRITE_AFTER_READ:
33 return "SYNC-HAZARD-WRITE_AFTER_READ";
34 break;
35 case SyncHazard::WRITE_AFTER_WRITE:
36 return "SYNC-HAZARD-WRITE_AFTER_WRITE";
37 break;
John Zulauf2f952d22020-02-10 11:34:51 -070038 case SyncHazard::READ_RACING_WRITE:
39 return "SYNC-HAZARD-READ-RACING-WRITE";
40 break;
41 case SyncHazard::WRITE_RACING_WRITE:
42 return "SYNC-HAZARD-WRITE-RACING-WRITE";
43 break;
44 case SyncHazard::WRITE_RACING_READ:
45 return "SYNC-HAZARD-WRITE-RACING-READ";
46 break;
John Zulauf9cb530d2019-09-30 14:14:10 -060047 default:
48 assert(0);
49 }
50 return "SYNC-HAZARD-INVALID";
51}
52
53static const char *string_SyncHazard(SyncHazard hazard) {
54 switch (hazard) {
55 case SyncHazard::NONE:
56 return "NONR";
57 break;
58 case SyncHazard::READ_AFTER_WRITE:
59 return "READ_AFTER_WRITE";
60 break;
61 case SyncHazard::WRITE_AFTER_READ:
62 return "WRITE_AFTER_READ";
63 break;
64 case SyncHazard::WRITE_AFTER_WRITE:
65 return "WRITE_AFTER_WRITE";
66 break;
John Zulauf2f952d22020-02-10 11:34:51 -070067 case SyncHazard::READ_RACING_WRITE:
68 return "READ_RACING_WRITE";
69 break;
70 case SyncHazard::WRITE_RACING_WRITE:
71 return "WRITE_RACING_WRITE";
72 break;
73 case SyncHazard::WRITE_RACING_READ:
74 return "WRITE_RACING_READ";
75 break;
John Zulauf9cb530d2019-09-30 14:14:10 -060076 default:
77 assert(0);
78 }
79 return "INVALID HAZARD";
80}
81
John Zulauf0cb5be22020-01-23 12:18:22 -070082// Expand the pipeline stage without regard to whether the are valid w.r.t. queue or extension
83VkPipelineStageFlags ExpandPipelineStages(VkQueueFlags queue_flags, VkPipelineStageFlags stage_mask) {
84 VkPipelineStageFlags expanded = stage_mask;
85 if (VK_PIPELINE_STAGE_ALL_COMMANDS_BIT & stage_mask) {
86 expanded = expanded & ~VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
87 for (const auto &all_commands : syncAllCommandStagesByQueueFlags) {
88 if (all_commands.first & queue_flags) {
89 expanded |= all_commands.second;
90 }
91 }
92 }
93 if (VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT & stage_mask) {
94 expanded = expanded & ~VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT;
95 expanded |= syncAllCommandStagesByQueueFlags.at(VK_QUEUE_GRAPHICS_BIT) & ~VK_PIPELINE_STAGE_HOST_BIT;
96 }
97 return expanded;
98}
99
John Zulauf36bcf6a2020-02-03 15:12:52 -0700100VkPipelineStageFlags RelatedPipelineStages(VkPipelineStageFlags stage_mask,
101 std::map<VkPipelineStageFlagBits, VkPipelineStageFlags> &map) {
102 VkPipelineStageFlags unscanned = stage_mask;
103 VkPipelineStageFlags related = 0;
104 for (const auto entry : map) {
105 const auto stage = entry.first;
106 if (stage & unscanned) {
107 related = related | entry.second;
108 unscanned = unscanned & ~stage;
109 if (!unscanned) break;
110 }
111 }
112 return related;
113}
114
115VkPipelineStageFlags WithEarlierPipelineStages(VkPipelineStageFlags stage_mask) {
116 return stage_mask | RelatedPipelineStages(stage_mask, syncLogicallyEarlierStages);
117}
118
119VkPipelineStageFlags WithLaterPipelineStages(VkPipelineStageFlags stage_mask) {
120 return stage_mask | RelatedPipelineStages(stage_mask, syncLogicallyLaterStages);
121}
122
John Zulauf5c5e88d2019-12-26 11:22:02 -0700123static const ResourceAccessRange full_range(std::numeric_limits<VkDeviceSize>::min(), std::numeric_limits<VkDeviceSize>::max());
124static ResourceAccessRange MakeMemoryAccessRange(const BUFFER_STATE &buffer, VkDeviceSize offset, VkDeviceSize size) {
John Zulauf9cb530d2019-09-30 14:14:10 -0600125 assert(!buffer.sparse);
126 const auto base = offset + buffer.binding.offset;
John Zulauf5c5e88d2019-12-26 11:22:02 -0700127 return ResourceAccessRange(base, base + size);
128}
129
John Zulauf3d84f1b2020-03-09 13:33:25 -0600130AccessTrackerContext::AccessTrackerContext(uint32_t subpass, VkQueueFlags queue_flags,
131 const std::vector<SubpassDependencyGraphNode> &dependencies,
132 const std::vector<AccessTrackerContext> &contexts,
133 AccessTrackerContext *external_context) {
134 Reset();
135 const auto &subpass_dep = dependencies[subpass];
136 prev_.reserve(subpass_dep.prev.size());
137 for (const auto &prev_dep : subpass_dep.prev) {
138 assert(prev_dep.dependency);
139 const auto dep = *prev_dep.dependency;
140 prev_.emplace_back(const_cast<AccessTrackerContext *>(&contexts[dep.srcSubpass]), queue_flags, dep);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700141 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600142
143 async_.reserve(subpass_dep.async.size());
144 for (const auto async_subpass : subpass_dep.async) {
145 async_.emplace_back(const_cast<AccessTrackerContext *>(&contexts[async_subpass]));
146 }
147 if (subpass_dep.barrier_from_external)
148 external_ = TrackBack(external_context, queue_flags, *subpass_dep.barrier_from_external);
149 else {
150 external_ = TrackBack();
151 }
John Zulauf5c5e88d2019-12-26 11:22:02 -0700152}
153
John Zulauf3d84f1b2020-03-09 13:33:25 -0600154// A recursive range walker for hazard detection, first for the current context and the (DetectHazardRecur) to walk
155// the DAG of the contexts (for example subpasses)
156template <typename Detector>
157HazardResult AccessTrackerContext::DetectHazard(const VulkanTypedHandle &handle, const Detector &detector,
158 const ResourceAccessRange &range, SyncBarrierStack *barrier_stack) const {
159 const auto access_tracker = GetAccessTracker(handle);
160 bool whole_range_gap = true;
161 HazardResult hazard;
162 if (access_tracker) {
163 const auto &accesses = access_tracker->GetCurrentAccessMap();
164 const auto from = accesses.lower_bound(range);
165 if (from != accesses.end() && from->first.intersects(range)) {
166 whole_range_gap = false;
167 const auto to = accesses.upper_bound(range);
168 ResourceAccessRange gap = {range.begin, range.begin};
169 for (auto pos = from; pos != to; ++pos) {
170 hazard = detector.Detect(pos, barrier_stack);
171 if (hazard.hazard) return hazard;
172
173 // Check for short circuiting recursion over pos->first
174 // If we've just checked against a write we don't have to looke any further...
175 // however even if the current usage is write and there is not hazard with a stored read, we still have to
176 // recur to ensure it's safe against *all* prior/parallel accesses.
177 auto upper_bound = std::min(range.end, pos->first.end);
178 if (detector.ShortCircuit(pos, barrier_stack)) {
179 gap.end = pos->first.begin;
180 } else {
181 // make sure we don't go past range
182 gap.end = upper_bound;
183 }
184 if (!gap.empty()) {
185 // Must recur on all gaps
186 hazard = DetectHazardRecur(handle, detector, gap, barrier_stack);
187 if (hazard.hazard) return hazard;
188 }
189 gap.begin = upper_bound;
190 }
191 }
192 }
193
194 if (whole_range_gap) { // There's no accesses recorded for this range for this context and resource
195 hazard = DetectHazardRecur(handle, detector, range, barrier_stack);
196 }
197 // We only check the async list for top level contexts (empty stack)
198 // Async check are not done at recursive levels, as the async lists are exhaustive for the top-level context
199 if (barrier_stack->size() == 0) {
200 for (const auto &async_context : async_) {
201 hazard = async_context->DetectAsyncHazard(handle, detector, range);
202 if (hazard.hazard) return hazard;
203 }
204 }
205
206 return hazard;
207}
208
209// A non recursive range walker for the asynchronous contexts (those we have no barriers with)
210template <typename Detector>
211HazardResult AccessTrackerContext::DetectAsyncHazard(const VulkanTypedHandle &handle, const Detector &detector,
212 const ResourceAccessRange &range) const {
213 const auto access_tracker = GetAccessTracker(handle);
214 HazardResult hazard;
215 if (access_tracker) {
216 auto accesses = access_tracker->GetCurrentAccessMap();
217 const auto from = accesses.lower_bound(range);
218 const auto to = accesses.upper_bound(range);
219 for (auto pos = from; pos != to; ++pos) {
220 hazard = detector.DetectAsync(pos);
221 if (hazard.hazard) break;
222 }
223 }
224 return hazard;
225}
226
227// Walk the contents or the context's predecessors, checking for hazards invoking the recursive and non-recursive range walkers
228// as appropriate for the type of predecessor
229template <typename Detector>
230HazardResult AccessTrackerContext::DetectHazardRecur(const VulkanTypedHandle &handle, const Detector &detector,
231 const ResourceAccessRange &range, SyncBarrierStack *barrier_stack) const {
232 HazardResult hazard;
233
234 assert(barrier_stack); // Barrier stack is a pointer only by convention (non-const objects to be passed by pointer)
235 for (const auto &prev_dep : prev_) {
236 barrier_stack->push_back(&prev_dep.barrier);
237 hazard = prev_dep.context->DetectHazard(handle, detector, range, barrier_stack);
238 barrier_stack->pop_back();
239 if (hazard.hazard) return hazard;
240 }
241 if (external_.context) {
242 barrier_stack->push_back(&external_.barrier);
243 hazard = external_.context->DetectHazard(handle, detector, range, barrier_stack);
244 barrier_stack->pop_back();
245 if (hazard.hazard) return hazard;
246 }
247
248 return hazard;
249}
250
251template <typename Detector>
252HazardResult AccessTrackerContext::DetectHazard(const VulkanTypedHandle &handle, const Detector &detector,
253 const ResourceAccessRange &range) const {
254 SyncBarrierStack barrier_stack;
255 return DetectHazard(handle, detector, range, &barrier_stack);
256}
257
258class HazardDetector {
259 SyncStageAccessIndex usage_index_;
260
261 public:
262 HazardResult Detect(const ResourceAccessRangeMap::const_iterator &pos, SyncBarrierStack *barrier_stack) const {
263 return pos->second.DetectHazard(usage_index_, barrier_stack);
264 }
265 bool ShortCircuit(const ResourceAccessRangeMap::const_iterator &pos, SyncBarrierStack *barrier_stack) const {
266 // Check for short circuiting recursion over pos->first
267 // If we've just checked against a write we don't have to looke any further...
268 // however even if the current usage is write and there is not hazard with a stored read, we still have to
269 // recur to ensure it's safe against *all* prior/parallel accesses.
270 return pos->second.HasWriteOp();
271 }
272 HazardResult DetectAsync(const ResourceAccessRangeMap::const_iterator &pos) const {
273 return pos->second.DetectAsyncHazard(usage_index_);
274 }
275 HazardDetector(SyncStageAccessIndex usage) : usage_index_(usage) {}
276};
277
278HazardResult AccessTrackerContext::DetectHazard(const VulkanTypedHandle &handle, SyncStageAccessIndex usage_index,
279 const ResourceAccessRange &range) const {
280 HazardDetector detector(usage_index);
281 return DetectHazard(handle, detector, range);
282}
283
284void CommandBufferAccessContext::BeginRenderPass(const RENDER_PASS_STATE &rp_state) {
285 // Create an access context for the first subpass and add it to the command buffers collection
286 render_pass_contexts_.emplace_back(queue_flags_, &rp_state.subpass_dependencies, &cb_tracker_context_);
287 current_renderpass_context_ = &render_pass_contexts_.back();
288 current_context_ = &current_renderpass_context_->CurrentContext();
289}
290
291void CommandBufferAccessContext::NextRenderPass(const RENDER_PASS_STATE &rp_state) {
292 assert(current_renderpass_context_);
293 current_renderpass_context_->NextSubpass(queue_flags_, &cb_tracker_context_);
294 current_context_ = &current_renderpass_context_->CurrentContext();
295}
296
297HazardResult AccessTrackerContext::DetectHazard(const IMAGE_STATE &image, SyncStageAccessIndex current_usage,
298 const VkImageSubresourceLayers &subresource, const VkOffset3D &offset,
299 const VkExtent3D &extent) const {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700300 // TODO: replace the encoder/generator with offset3D/extent3D aware versions
301 VkImageSubresourceRange subresource_range = {subresource.aspectMask, subresource.mipLevel, 1, subresource.baseArrayLayer,
302 subresource.layerCount};
303 subresource_adapter::RangeGenerator range_gen(image.range_encoder, subresource_range);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600304 VulkanTypedHandle image_handle(image.image, kVulkanObjectTypeImage);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700305 for (; range_gen->non_empty(); ++range_gen) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600306 HazardResult hazard = DetectHazard(image_handle, current_usage, *range_gen);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700307 if (hazard.hazard) return hazard;
308 }
309 return HazardResult();
John Zulauf9cb530d2019-09-30 14:14:10 -0600310}
311
John Zulauf3d84f1b2020-03-09 13:33:25 -0600312class BarrierHazardDetector {
313 public:
314 BarrierHazardDetector(SyncStageAccessIndex usage_index, VkPipelineStageFlags src_exec_scope,
315 SyncStageAccessFlags src_access_scope)
316 : usage_index_(usage_index), src_exec_scope_(src_exec_scope), src_access_scope_(src_access_scope) {}
317
318 HazardResult Detect(const ResourceAccessRangeMap::const_iterator &pos, SyncBarrierStack *barrier_stack) const {
319 return pos->second.DetectBarrierHazard(usage_index_, src_exec_scope_, src_access_scope_, barrier_stack);
John Zulauf0cb5be22020-01-23 12:18:22 -0700320 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600321 bool ShortCircuit(const ResourceAccessRangeMap::const_iterator &pos, SyncBarrierStack *barrier_stack) const { return false; }
322 HazardResult DetectAsync(const ResourceAccessRangeMap::const_iterator &pos) const {
323 // Async barrier hazard detection can use the same path as the usage index is not IsRead, but is IsWrite
324 return pos->second.DetectAsyncHazard(usage_index_);
325 }
326
327 private:
328 SyncStageAccessIndex usage_index_;
329 VkPipelineStageFlags src_exec_scope_;
330 SyncStageAccessFlags src_access_scope_;
331};
332
333HazardResult AccessTrackerContext::DetectBarrierHazard(const VulkanTypedHandle &handle, SyncStageAccessIndex current_usage,
334 VkPipelineStageFlags src_exec_scope, SyncStageAccessFlags src_access_scope,
335 const ResourceAccessRange &range) const {
336 BarrierHazardDetector detector(current_usage, src_exec_scope, src_access_scope);
337 return DetectHazard(handle, detector, range);
John Zulauf0cb5be22020-01-23 12:18:22 -0700338}
339
John Zulauf3d84f1b2020-03-09 13:33:25 -0600340HazardResult DetectImageBarrierHazard(const AccessTrackerContext &context, const IMAGE_STATE &image,
John Zulauf36bcf6a2020-02-03 15:12:52 -0700341 VkPipelineStageFlags src_exec_scope, SyncStageAccessFlags src_stage_accesses,
John Zulauf0cb5be22020-01-23 12:18:22 -0700342 const VkImageMemoryBarrier &barrier) {
John Zulauf0cb5be22020-01-23 12:18:22 -0700343 auto subresource_range = NormalizeSubresourceRange(image.createInfo, barrier.subresourceRange);
344 subresource_adapter::RangeGenerator range_gen(image.range_encoder, subresource_range);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600345 const VulkanTypedHandle image_handle(image.image, kVulkanObjectTypeImage);
John Zulauf36bcf6a2020-02-03 15:12:52 -0700346 const auto src_access_scope = SyncStageAccess::AccessScope(src_stage_accesses, barrier.srcAccessMask);
John Zulauf0cb5be22020-01-23 12:18:22 -0700347 for (; range_gen->non_empty(); ++range_gen) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600348 HazardResult hazard = context.DetectBarrierHazard(image_handle, SyncStageAccessIndex::SYNC_IMAGE_LAYOUT_TRANSITION,
349 src_exec_scope, src_access_scope, *range_gen);
John Zulauf0cb5be22020-01-23 12:18:22 -0700350 if (hazard.hazard) return hazard;
351 }
352 return HazardResult();
353}
354
John Zulauf9cb530d2019-09-30 14:14:10 -0600355template <typename Flags, typename Map>
356SyncStageAccessFlags AccessScopeImpl(Flags flag_mask, const Map &map) {
357 SyncStageAccessFlags scope = 0;
358 for (const auto &bit_scope : map) {
359 if (flag_mask < bit_scope.first) break;
360
361 if (flag_mask & bit_scope.first) {
362 scope |= bit_scope.second;
363 }
364 }
365 return scope;
366}
367
368SyncStageAccessFlags SyncStageAccess::AccessScopeByStage(VkPipelineStageFlags stages) {
369 return AccessScopeImpl(stages, syncStageAccessMaskByStageBit);
370}
371
372SyncStageAccessFlags SyncStageAccess::AccessScopeByAccess(VkAccessFlags accesses) {
373 return AccessScopeImpl(accesses, syncStageAccessMaskByAccessBit);
374}
375
376// Getting from stage mask and access mask to stage/acess masks is something we need to be good at...
377SyncStageAccessFlags SyncStageAccess::AccessScope(VkPipelineStageFlags stages, VkAccessFlags accesses) {
378 // The access scope is the intersection of all stage/access types possible for the enabled stages and the enables accesses
379 // (after doing a couple factoring of common terms the union of stage/access intersections is the intersections of the
380 // union of all stage/access types for all the stages and the same unions for the access mask...
381 return AccessScopeByStage(stages) & AccessScopeByAccess(accesses);
382}
383
384template <typename Action>
John Zulauf5c5e88d2019-12-26 11:22:02 -0700385void UpdateMemoryAccessState(ResourceAccessRangeMap *accesses, const ResourceAccessRange &range, const Action &action) {
John Zulauf9cb530d2019-09-30 14:14:10 -0600386 // TODO -- region/mem-range accuracte update
387 auto pos = accesses->lower_bound(range);
388 if (pos == accesses->end() || !pos->first.intersects(range)) {
389 // The range is empty, fill it with a default value.
390 pos = action.Infill(accesses, pos, range);
391 } else if (range.begin < pos->first.begin) {
392 // Leading empty space, infill
John Zulauf5c5e88d2019-12-26 11:22:02 -0700393 pos = action.Infill(accesses, pos, ResourceAccessRange(range.begin, pos->first.begin));
John Zulauf9cb530d2019-09-30 14:14:10 -0600394 } else if (pos->first.begin < range.begin) {
395 // Trim the beginning if needed
396 pos = accesses->split(pos, range.begin, sparse_container::split_op_keep_both());
397 ++pos;
398 }
399
400 const auto the_end = accesses->end();
401 while ((pos != the_end) && pos->first.intersects(range)) {
402 if (pos->first.end > range.end) {
403 pos = accesses->split(pos, range.end, sparse_container::split_op_keep_both());
404 }
405
406 pos = action(accesses, pos);
407 if (pos == the_end) break;
408
409 auto next = pos;
410 ++next;
411 if ((pos->first.end < range.end) && (next != the_end) && !next->first.is_subsequent_to(pos->first)) {
412 // Need to infill if next is disjoint
413 VkDeviceSize limit = (next == the_end) ? range.end : std::min(range.end, next->first.begin);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700414 ResourceAccessRange new_range(pos->first.end, limit);
John Zulauf9cb530d2019-09-30 14:14:10 -0600415 next = action.Infill(accesses, next, new_range);
416 }
417 pos = next;
418 }
419}
420
421struct UpdateMemoryAccessStateFunctor {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700422 using Iterator = ResourceAccessRangeMap::iterator;
423 Iterator Infill(ResourceAccessRangeMap *accesses, Iterator pos, ResourceAccessRange range) const {
John Zulauf9cb530d2019-09-30 14:14:10 -0600424 return accesses->insert(pos, std::make_pair(range, ResourceAccessState()));
425 }
John Zulauf5c5e88d2019-12-26 11:22:02 -0700426 Iterator operator()(ResourceAccessRangeMap *accesses, Iterator pos) const {
John Zulauf9cb530d2019-09-30 14:14:10 -0600427 auto &access_state = pos->second;
428 access_state.Update(usage, tag);
429 return pos;
430 }
431
432 UpdateMemoryAccessStateFunctor(SyncStageAccessIndex usage_, const ResourceUsageTag &tag_) : usage(usage_), tag(tag_) {}
433 SyncStageAccessIndex usage;
434 const ResourceUsageTag &tag;
435};
436
437struct ApplyMemoryAccessBarrierFunctor {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700438 using Iterator = ResourceAccessRangeMap::iterator;
439 inline Iterator Infill(ResourceAccessRangeMap *accesses, Iterator pos, ResourceAccessRange range) const { return pos; }
John Zulauf9cb530d2019-09-30 14:14:10 -0600440
John Zulauf5c5e88d2019-12-26 11:22:02 -0700441 Iterator operator()(ResourceAccessRangeMap *accesses, Iterator pos) const {
John Zulauf9cb530d2019-09-30 14:14:10 -0600442 auto &access_state = pos->second;
John Zulauf36bcf6a2020-02-03 15:12:52 -0700443 access_state.ApplyMemoryAccessBarrier(src_exec_scope, src_access_scope, dst_exec_scope, dst_access_scope);
John Zulauf9cb530d2019-09-30 14:14:10 -0600444 return pos;
445 }
446
John Zulauf36bcf6a2020-02-03 15:12:52 -0700447 ApplyMemoryAccessBarrierFunctor(VkPipelineStageFlags src_exec_scope_, SyncStageAccessFlags src_access_scope_,
448 VkPipelineStageFlags dst_exec_scope_, SyncStageAccessFlags dst_access_scope_)
449 : src_exec_scope(src_exec_scope_),
450 src_access_scope(src_access_scope_),
451 dst_exec_scope(dst_exec_scope_),
452 dst_access_scope(dst_access_scope_) {}
John Zulauf9cb530d2019-09-30 14:14:10 -0600453
John Zulauf36bcf6a2020-02-03 15:12:52 -0700454 VkPipelineStageFlags src_exec_scope;
455 SyncStageAccessFlags src_access_scope;
456 VkPipelineStageFlags dst_exec_scope;
457 SyncStageAccessFlags dst_access_scope;
John Zulauf9cb530d2019-09-30 14:14:10 -0600458};
459
460struct ApplyGlobalBarrierFunctor {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700461 using Iterator = ResourceAccessRangeMap::iterator;
462 inline Iterator Infill(ResourceAccessRangeMap *accesses, Iterator pos, ResourceAccessRange range) const { return pos; }
John Zulauf9cb530d2019-09-30 14:14:10 -0600463
John Zulauf5c5e88d2019-12-26 11:22:02 -0700464 Iterator operator()(ResourceAccessRangeMap *accesses, Iterator pos) const {
John Zulauf9cb530d2019-09-30 14:14:10 -0600465 auto &access_state = pos->second;
John Zulauf36bcf6a2020-02-03 15:12:52 -0700466 access_state.ApplyExecutionBarrier(src_exec_scope, dst_exec_scope);
John Zulauf9cb530d2019-09-30 14:14:10 -0600467
468 for (const auto &functor : barrier_functor) {
469 functor(accesses, pos);
470 }
471 return pos;
472 }
473
John Zulauf36bcf6a2020-02-03 15:12:52 -0700474 ApplyGlobalBarrierFunctor(VkPipelineStageFlags src_exec_scope, VkPipelineStageFlags dst_exec_scope,
475 SyncStageAccessFlags src_stage_accesses, SyncStageAccessFlags dst_stage_accesses,
John Zulauf9cb530d2019-09-30 14:14:10 -0600476 uint32_t memoryBarrierCount, const VkMemoryBarrier *pMemoryBarriers)
John Zulauf36bcf6a2020-02-03 15:12:52 -0700477 : src_exec_scope(src_exec_scope), dst_exec_scope(dst_exec_scope) {
John Zulauf9cb530d2019-09-30 14:14:10 -0600478 // Don't want to create this per tracked item, but don't want to loop through all tracked items per barrier...
479 barrier_functor.reserve(memoryBarrierCount);
480 for (uint32_t barrier_index = 0; barrier_index < memoryBarrierCount; barrier_index++) {
481 const auto &barrier = pMemoryBarriers[barrier_index];
John Zulauf36bcf6a2020-02-03 15:12:52 -0700482 barrier_functor.emplace_back(src_exec_scope, SyncStageAccess::AccessScope(src_stage_accesses, barrier.srcAccessMask),
483 dst_exec_scope, SyncStageAccess::AccessScope(dst_stage_accesses, barrier.dstAccessMask));
John Zulauf9cb530d2019-09-30 14:14:10 -0600484 }
485 }
486
John Zulauf36bcf6a2020-02-03 15:12:52 -0700487 const VkPipelineStageFlags src_exec_scope;
488 const VkPipelineStageFlags dst_exec_scope;
John Zulauf9cb530d2019-09-30 14:14:10 -0600489 std::vector<ApplyMemoryAccessBarrierFunctor> barrier_functor;
490};
491
John Zulauf3d84f1b2020-03-09 13:33:25 -0600492void AccessTracker::UpdateAccessState(SyncStageAccessIndex current_usage, const ResourceAccessRange &range,
493 const ResourceUsageTag &tag) {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700494 UpdateMemoryAccessStateFunctor action(current_usage, tag);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600495 UpdateMemoryAccessState(&accesses_, range, action);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700496}
497
John Zulauf3d84f1b2020-03-09 13:33:25 -0600498void AccessTrackerContext::UpdateAccessState(const VulkanTypedHandle &handle, SyncStageAccessIndex current_usage,
499 const ResourceAccessRange &range, const ResourceUsageTag &tag) {
500 auto *tracker = GetAccessTracker(handle);
501 assert(tracker);
502 tracker->UpdateAccessState(current_usage, range, tag);
503}
504
505void AccessTracker::UpdateAccessState(const IMAGE_STATE &image, SyncStageAccessIndex current_usage,
506 const VkImageSubresourceLayers &subresource, const VkOffset3D &offset,
507 const VkExtent3D &extent, const ResourceUsageTag &tag) {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700508 // TODO: replace the encoder/generator with offset3D aware versions
509 VkImageSubresourceRange subresource_range = {subresource.aspectMask, subresource.mipLevel, 1, subresource.baseArrayLayer,
510 subresource.layerCount};
511 subresource_adapter::RangeGenerator range_gen(image.range_encoder, subresource_range);
512 for (; range_gen->non_empty(); ++range_gen) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600513 UpdateAccessState(current_usage, *range_gen, tag);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700514 }
515}
516
John Zulauf3d84f1b2020-03-09 13:33:25 -0600517void AccessTrackerContext::UpdateAccessState(const IMAGE_STATE &image, SyncStageAccessIndex current_usage,
518 const VkImageSubresourceLayers &subresource, const VkOffset3D &offset,
519 const VkExtent3D &extent, const ResourceUsageTag &tag) {
520 const VulkanTypedHandle handle(image.image, kVulkanObjectTypeImage);
521 auto *tracker = GetAccessTracker(handle);
522 assert(tracker);
523 tracker->UpdateAccessState(image, current_usage, subresource, offset, extent, tag);
524}
525
526SyncBarrier::SyncBarrier(VkQueueFlags queue_flags, const VkSubpassDependency2 &barrier) {
527 const auto src_stage_mask = ExpandPipelineStages(queue_flags, barrier.srcStageMask);
528 src_exec_scope = WithEarlierPipelineStages(src_stage_mask);
529 src_access_scope = SyncStageAccess::AccessScope(src_stage_mask, barrier.srcAccessMask);
530 const auto dst_stage_mask = ExpandPipelineStages(queue_flags, barrier.dstStageMask);
531 dst_exec_scope = WithLaterPipelineStages(dst_stage_mask);
532 dst_access_scope = SyncStageAccess::AccessScope(dst_stage_mask, barrier.dstAccessMask);
533}
534
535void ResourceAccessState::ApplyBarrier(const SyncBarrier &barrier) {
536 ApplyExecutionBarrier(barrier.src_exec_scope, barrier.dst_exec_scope);
537 ApplyMemoryAccessBarrier(barrier.src_exec_scope, barrier.src_access_scope, barrier.dst_exec_scope, barrier.dst_access_scope);
538}
539
540ResourceAccessState ResourceAccessState::ApplyBarrierStack(const ResourceAccessState &that, const SyncBarrierStack &barrier_stack) {
541 ResourceAccessState copy = that;
542 for (auto barrier = barrier_stack.begin(); barrier != barrier_stack.end(); ++barrier) {
543 assert(*barrier);
544 copy.ApplyBarrier(*(*barrier));
545 }
546 return copy;
547}
548
549HazardResult ResourceAccessState::DetectHazard(SyncStageAccessIndex usage_index, SyncBarrierStack *barrier_stack) const {
550 if (barrier_stack) {
551 return ApplyBarrierStack(*this, *barrier_stack).DetectHazard(usage_index);
552 }
553 return DetectHazard(usage_index);
554}
555
John Zulauf9cb530d2019-09-30 14:14:10 -0600556HazardResult ResourceAccessState::DetectHazard(SyncStageAccessIndex usage_index) const {
557 HazardResult hazard;
558 auto usage = FlagBit(usage_index);
559 if (IsRead(usage)) {
560 if (IsWriteHazard(usage)) {
561 hazard.Set(READ_AFTER_WRITE, write_tag);
562 }
563 } else {
564 // Assume write
565 // TODO determine what to do with READ-WRITE usage states if any
566 // Write-After-Write check -- if we have a previous write to test against
567 if (last_write && IsWriteHazard(usage)) {
568 hazard.Set(WRITE_AFTER_WRITE, write_tag);
569 } else {
570 // Only look for casus belli for WAR
571 const auto usage_stage = PipelineStageBit(usage_index);
572 for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
573 if (IsReadHazard(usage_stage, last_reads[read_index])) {
574 hazard.Set(WRITE_AFTER_READ, last_reads[read_index].tag);
575 break;
576 }
577 }
578 }
579 }
580 return hazard;
581}
582
John Zulauf2f952d22020-02-10 11:34:51 -0700583// Asynchronous Hazards occur between subpasses with no connection through the DAG
John Zulauf3d84f1b2020-03-09 13:33:25 -0600584HazardResult ResourceAccessState::DetectAsyncHazard(SyncStageAccessIndex usage_index) const {
John Zulauf2f952d22020-02-10 11:34:51 -0700585 HazardResult hazard;
586 auto usage = FlagBit(usage_index);
587 if (IsRead(usage)) {
588 if (last_write != 0) {
589 hazard.Set(READ_RACING_WRITE, write_tag);
590 }
591 } else {
592 if (last_write != 0) {
593 hazard.Set(WRITE_RACING_WRITE, write_tag);
594 } else if (last_read_count > 0) {
595 hazard.Set(WRITE_RACING_READ, last_reads[0].tag);
596 }
597 }
598 return hazard;
599}
600
John Zulauf36bcf6a2020-02-03 15:12:52 -0700601HazardResult ResourceAccessState::DetectBarrierHazard(SyncStageAccessIndex usage_index, VkPipelineStageFlags src_exec_scope,
John Zulauf3d84f1b2020-03-09 13:33:25 -0600602 SyncStageAccessFlags src_access_scope,
603 SyncBarrierStack *barrier_stack) const {
604 if (barrier_stack) {
605 return ApplyBarrierStack(*this, *barrier_stack).DetectBarrierHazard(usage_index, src_exec_scope, src_access_scope);
606 }
607 return DetectBarrierHazard(usage_index, src_exec_scope, src_access_scope);
608}
609
610HazardResult ResourceAccessState::DetectBarrierHazard(SyncStageAccessIndex usage_index, VkPipelineStageFlags src_exec_scope,
John Zulauf36bcf6a2020-02-03 15:12:52 -0700611 SyncStageAccessFlags src_access_scope) const {
John Zulauf0cb5be22020-01-23 12:18:22 -0700612 // Only supporting image layout transitions for now
613 assert(usage_index == SyncStageAccessIndex::SYNC_IMAGE_LAYOUT_TRANSITION);
614 HazardResult hazard;
615 if (last_write) {
616 // If the previous write is *not* in the 1st access scope
617 // *AND* the current barrier is not in the dependency chain
618 // *AND* the there is no prior memory barrier for the previous write in the dependency chain
619 // then the barrier access is unsafe (R/W after W)
John Zulauf36bcf6a2020-02-03 15:12:52 -0700620 if (((last_write & src_access_scope) == 0) && ((src_exec_scope & write_dependency_chain) == 0) && (write_barriers == 0)) {
John Zulauf0cb5be22020-01-23 12:18:22 -0700621 // TODO: Do we need a difference hazard name for this?
622 hazard.Set(WRITE_AFTER_WRITE, write_tag);
623 }
624 } else {
625 // Look at the reads
626 for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
John Zulauf36bcf6a2020-02-03 15:12:52 -0700627 const auto &read_access = last_reads[read_index];
628 // If the read stage is not in the src sync sync
629 // *AND* not execution chained with an existing sync barrier (that's the or)
630 // then the barrier access is unsafe (R/W after R)
631 if ((src_exec_scope & (read_access.stage | read_access.barriers)) == 0) {
632 hazard.Set(WRITE_AFTER_READ, read_access.tag);
John Zulauf0cb5be22020-01-23 12:18:22 -0700633 break;
634 }
635 }
636 }
637 return hazard;
638}
639
John Zulauf9cb530d2019-09-30 14:14:10 -0600640void ResourceAccessState::Update(SyncStageAccessIndex usage_index, const ResourceUsageTag &tag) {
641 // Move this logic in the ResourceStateTracker as methods, thereof (or we'll repeat it for every flavor of resource...
642 const auto usage_bit = FlagBit(usage_index);
643 if (IsRead(usage_index)) {
644 // Mulitple outstanding reads may be of interest and do dependency chains independently
645 // However, for purposes of barrier tracking, only one read per pipeline stage matters
646 const auto usage_stage = PipelineStageBit(usage_index);
647 if (usage_stage & last_read_stages) {
648 for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
649 ReadState &access = last_reads[read_index];
650 if (access.stage == usage_stage) {
651 access.barriers = 0;
652 access.tag = tag;
653 break;
654 }
655 }
656 } else {
657 // We don't have this stage in the list yet...
658 assert(last_read_count < last_reads.size());
659 ReadState &access = last_reads[last_read_count++];
660 access.stage = usage_stage;
661 access.barriers = 0;
662 access.tag = tag;
663 last_read_stages |= usage_stage;
664 }
665 } else {
666 // Assume write
667 // TODO determine what to do with READ-WRITE operations if any
668 // Clobber last read and both sets of barriers... because all we have is DANGER, DANGER, WILL ROBINSON!!!
669 // if the last_reads/last_write were unsafe, we've reported them,
670 // in either case the prior access is irrelevant, we can overwrite them as *this* write is now after them
671 last_read_count = 0;
672 last_read_stages = 0;
673
674 write_barriers = 0;
675 write_dependency_chain = 0;
676 write_tag = tag;
677 last_write = usage_bit;
678 }
679}
680void ResourceAccessState::ApplyExecutionBarrier(VkPipelineStageFlags srcStageMask, VkPipelineStageFlags dstStageMask) {
681 // Execution Barriers only protect read operations
682 for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
683 ReadState &access = last_reads[read_index];
684 // The | implements the "dependency chain" logic for this access, as the barriers field stores the second sync scope
685 if (srcStageMask & (access.stage | access.barriers)) {
686 access.barriers |= dstStageMask;
687 }
688 }
689 if (write_dependency_chain & srcStageMask) write_dependency_chain |= dstStageMask;
690}
691
John Zulauf36bcf6a2020-02-03 15:12:52 -0700692void ResourceAccessState::ApplyMemoryAccessBarrier(VkPipelineStageFlags src_exec_scope, SyncStageAccessFlags src_access_scope,
693 VkPipelineStageFlags dst_exec_scope, SyncStageAccessFlags dst_access_scope) {
John Zulauf9cb530d2019-09-30 14:14:10 -0600694 // Assuming we've applied the execution side of this barrier, we update just the write
695 // The || implements the "dependency chain" logic for this barrier
John Zulauf36bcf6a2020-02-03 15:12:52 -0700696 if ((src_access_scope & last_write) || (write_dependency_chain & src_exec_scope)) {
697 write_barriers |= dst_access_scope;
698 write_dependency_chain |= dst_exec_scope;
John Zulauf9cb530d2019-09-30 14:14:10 -0600699 }
700}
701
702void SyncValidator::ResetCommandBuffer(VkCommandBuffer command_buffer) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600703 auto *access_context = GetAccessContextNoInsert(command_buffer);
704 if (access_context) {
705 access_context->Reset();
John Zulauf9cb530d2019-09-30 14:14:10 -0600706 }
707}
708
John Zulauf3d84f1b2020-03-09 13:33:25 -0600709void SyncValidator::ApplyGlobalBarriers(AccessTrackerContext *context, VkPipelineStageFlags srcStageMask,
John Zulauf36bcf6a2020-02-03 15:12:52 -0700710 VkPipelineStageFlags dstStageMask, SyncStageAccessFlags src_access_scope,
711 SyncStageAccessFlags dst_access_scope, uint32_t memoryBarrierCount,
John Zulauf9cb530d2019-09-30 14:14:10 -0600712 const VkMemoryBarrier *pMemoryBarriers) {
713 // TODO: Implement this better (maybe some delayed/on-demand integration).
John Zulauf36bcf6a2020-02-03 15:12:52 -0700714 ApplyGlobalBarrierFunctor barriers_functor(srcStageMask, dstStageMask, src_access_scope, dst_access_scope, memoryBarrierCount,
John Zulauf9cb530d2019-09-30 14:14:10 -0600715 pMemoryBarriers);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600716 for (auto &handle_tracker_pair : context->GetAccessTrackerMap()) {
717 UpdateMemoryAccessState(&handle_tracker_pair.second.GetCurrentAccessMap(), full_range, barriers_functor);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700718 }
John Zulauf9cb530d2019-09-30 14:14:10 -0600719}
720
John Zulauf3d84f1b2020-03-09 13:33:25 -0600721void SyncValidator::ApplyBufferBarriers(AccessTrackerContext *context, VkPipelineStageFlags src_exec_scope,
John Zulauf36bcf6a2020-02-03 15:12:52 -0700722 SyncStageAccessFlags src_stage_accesses, VkPipelineStageFlags dst_exec_scope,
723 SyncStageAccessFlags dst_stage_accesses, uint32_t barrier_count,
John Zulauf9cb530d2019-09-30 14:14:10 -0600724 const VkBufferMemoryBarrier *barriers) {
725 // TODO Implement this at subresource/memory_range accuracy
726 for (uint32_t index = 0; index < barrier_count; index++) {
727 const auto &barrier = barriers[index];
728 const auto *buffer = Get<BUFFER_STATE>(barrier.buffer);
729 if (!buffer) continue;
John Zulauf3d84f1b2020-03-09 13:33:25 -0600730 auto *tracker = context->GetAccessTracker(VulkanTypedHandle(buffer->binding.mem_state->mem, kVulkanObjectTypeDeviceMemory));
731 if (!tracker) continue;
John Zulauf5c5e88d2019-12-26 11:22:02 -0700732 ResourceAccessRange range = MakeMemoryAccessRange(*buffer, barrier.offset, barrier.size);
John Zulauf9cb530d2019-09-30 14:14:10 -0600733 UpdateMemoryAccessState(
John Zulauf3d84f1b2020-03-09 13:33:25 -0600734 &tracker->GetCurrentAccessMap(), range,
John Zulauf36bcf6a2020-02-03 15:12:52 -0700735 ApplyMemoryAccessBarrierFunctor(src_exec_scope, AccessScope(src_stage_accesses, barrier.srcAccessMask), dst_exec_scope,
736 AccessScope(dst_stage_accesses, barrier.dstAccessMask)));
John Zulauf9cb530d2019-09-30 14:14:10 -0600737 }
738}
739
John Zulauf3d84f1b2020-03-09 13:33:25 -0600740void SyncValidator::ApplyImageBarriers(AccessTrackerContext *context, VkPipelineStageFlags src_exec_scope,
John Zulauf36bcf6a2020-02-03 15:12:52 -0700741 SyncStageAccessFlags src_stage_accesses, VkPipelineStageFlags dst_exec_scope,
742 SyncStageAccessFlags dst_stage_accesses, uint32_t barrier_count,
John Zulauf5c5e88d2019-12-26 11:22:02 -0700743 const VkImageMemoryBarrier *barriers) {
744 for (uint32_t index = 0; index < barrier_count; index++) {
745 const auto &barrier = barriers[index];
746 const auto *image = Get<IMAGE_STATE>(barrier.image);
747 if (!image) continue;
John Zulauf3d84f1b2020-03-09 13:33:25 -0600748 auto tracker = context->GetAccessTrackerNoInsert(VulkanTypedHandle(barrier.image, kVulkanObjectTypeImage));
749 if (!tracker) continue;
750 auto *accesses = &tracker->GetCurrentAccessMap();
751
John Zulauf5c5e88d2019-12-26 11:22:02 -0700752 auto subresource_range = NormalizeSubresourceRange(image->createInfo, barrier.subresourceRange);
753 subresource_adapter::RangeGenerator range_gen(image->range_encoder, subresource_range);
John Zulauf36bcf6a2020-02-03 15:12:52 -0700754 const ApplyMemoryAccessBarrierFunctor barrier_action(src_exec_scope, AccessScope(src_stage_accesses, barrier.srcAccessMask),
755 dst_exec_scope,
756 AccessScope(dst_stage_accesses, barrier.dstAccessMask));
John Zulauf5c5e88d2019-12-26 11:22:02 -0700757 for (; range_gen->non_empty(); ++range_gen) {
758 UpdateMemoryAccessState(accesses, *range_gen, barrier_action);
759 }
John Zulauf9cb530d2019-09-30 14:14:10 -0600760 }
John Zulauf9cb530d2019-09-30 14:14:10 -0600761}
762
763bool SyncValidator::PreCallValidateCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer,
764 uint32_t regionCount, const VkBufferCopy *pRegions) const {
765 bool skip = false;
John Zulauf3d84f1b2020-03-09 13:33:25 -0600766 const auto *cb_context = GetAccessContext(commandBuffer);
767 assert(cb_context);
768 if (!cb_context) return skip;
769 const auto *context = cb_context->GetCurrentAccessContext();
John Zulauf9cb530d2019-09-30 14:14:10 -0600770
John Zulauf3d84f1b2020-03-09 13:33:25 -0600771 // If we have no previous accesses, we have no hazards
772 // TODO: make this sub-resource capable
773 // TODO: make this general, and stuff it into templates/utility functions
774 const auto *src_buffer = Get<BUFFER_STATE>(srcBuffer);
775 const auto src_mem = (src_buffer && !src_buffer->sparse) ? src_buffer->binding.mem_state->mem : VK_NULL_HANDLE;
776 const auto *dst_buffer = Get<BUFFER_STATE>(dstBuffer);
777 const auto dst_mem = (dst_buffer && !dst_buffer->sparse) ? dst_buffer->binding.mem_state->mem : VK_NULL_HANDLE;
778
779 for (uint32_t region = 0; region < regionCount; region++) {
780 const auto &copy_region = pRegions[region];
781 if (src_mem != VK_NULL_HANDLE) {
782 ResourceAccessRange src_range = MakeMemoryAccessRange(*src_buffer, copy_region.srcOffset, copy_region.size);
783 auto hazard = context->DetectHazard(VulkanTypedHandle(src_mem, kVulkanObjectTypeDeviceMemory),
784 SYNC_TRANSFER_TRANSFER_READ, src_range);
785 if (hazard.hazard) {
786 // TODO -- add tag information to log msg when useful.
787 skip |= LogError(srcBuffer, string_SyncHazardVUID(hazard.hazard), "Hazard %s for srcBuffer %s, region %" PRIu32,
788 string_SyncHazard(hazard.hazard), report_data->FormatHandle(srcBuffer).c_str(), region);
John Zulauf9cb530d2019-09-30 14:14:10 -0600789 }
John Zulauf9cb530d2019-09-30 14:14:10 -0600790 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600791 if ((dst_mem != VK_NULL_HANDLE) && !skip) {
792 ResourceAccessRange dst_range = MakeMemoryAccessRange(*dst_buffer, copy_region.dstOffset, copy_region.size);
793 auto hazard = context->DetectHazard(VulkanTypedHandle(dst_mem, kVulkanObjectTypeDeviceMemory),
794 SYNC_TRANSFER_TRANSFER_WRITE, dst_range);
795 if (hazard.hazard) {
796 skip |= LogError(dstBuffer, string_SyncHazardVUID(hazard.hazard), "Hazard %s for dstBuffer %s, region %" PRIu32,
797 string_SyncHazard(hazard.hazard), report_data->FormatHandle(dstBuffer).c_str(), region);
798 }
799 }
800 if (skip) break;
John Zulauf9cb530d2019-09-30 14:14:10 -0600801 }
802 return skip;
803}
804
805void SyncValidator::PreCallRecordCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer,
806 uint32_t regionCount, const VkBufferCopy *pRegions) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600807 auto *cb_context = GetAccessContext(commandBuffer);
808 assert(cb_context);
809 auto *context = cb_context->GetCurrentAccessContext();
810
John Zulauf9cb530d2019-09-30 14:14:10 -0600811 const auto *src_buffer = Get<BUFFER_STATE>(srcBuffer);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600812 const auto src_mem = (src_buffer && !src_buffer->sparse) ? src_buffer->binding.mem_state->mem : VK_NULL_HANDLE;
813 const VulkanTypedHandle src_handle(src_mem, kVulkanObjectTypeDeviceMemory);
814 AccessTracker *src_tracker = src_mem ? context->GetAccessTracker(src_handle) : nullptr;
815
John Zulauf9cb530d2019-09-30 14:14:10 -0600816 const auto *dst_buffer = Get<BUFFER_STATE>(dstBuffer);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600817 const auto dst_mem = (dst_buffer && !dst_buffer->sparse) ? dst_buffer->binding.mem_state->mem : VK_NULL_HANDLE;
818 const VulkanTypedHandle dst_handle(dst_mem, kVulkanObjectTypeDeviceMemory);
819 AccessTracker *dst_tracker = dst_mem ? context->GetAccessTracker(dst_handle) : nullptr;
John Zulauf9cb530d2019-09-30 14:14:10 -0600820
821 for (uint32_t region = 0; region < regionCount; region++) {
822 const auto &copy_region = pRegions[region];
John Zulauf3d84f1b2020-03-09 13:33:25 -0600823 if (src_mem) {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700824 ResourceAccessRange src_range = MakeMemoryAccessRange(*src_buffer, copy_region.srcOffset, copy_region.size);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600825 src_tracker->UpdateAccessState(SYNC_TRANSFER_TRANSFER_READ, src_range, tag);
John Zulauf9cb530d2019-09-30 14:14:10 -0600826 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600827 if (dst_mem) {
John Zulauf5c5e88d2019-12-26 11:22:02 -0700828 ResourceAccessRange dst_range = MakeMemoryAccessRange(*dst_buffer, copy_region.dstOffset, copy_region.size);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600829 dst_tracker->UpdateAccessState(SYNC_TRANSFER_TRANSFER_WRITE, dst_range, tag);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700830 }
831 }
832}
833
834bool SyncValidator::PreCallValidateCmdCopyImage(VkCommandBuffer commandBuffer, VkImage srcImage, VkImageLayout srcImageLayout,
835 VkImage dstImage, VkImageLayout dstImageLayout, uint32_t regionCount,
836 const VkImageCopy *pRegions) const {
837 bool skip = false;
John Zulauf3d84f1b2020-03-09 13:33:25 -0600838 const auto *cb_access_context = GetAccessContext(commandBuffer);
839 assert(cb_access_context);
840 if (!cb_access_context) return skip;
John Zulauf5c5e88d2019-12-26 11:22:02 -0700841
John Zulauf3d84f1b2020-03-09 13:33:25 -0600842 const auto *context = cb_access_context->GetCurrentAccessContext();
843 assert(context);
844 if (!context) return skip;
845
846 const auto *src_image = Get<IMAGE_STATE>(srcImage);
847 const auto *dst_image = Get<IMAGE_STATE>(dstImage);
848
849 for (uint32_t region = 0; region < regionCount; region++) {
850 const auto &copy_region = pRegions[region];
851 if (src_image) {
852 auto hazard = context->DetectHazard(*src_image, SYNC_TRANSFER_TRANSFER_READ, copy_region.srcSubresource,
853 copy_region.srcOffset, copy_region.extent);
854 if (hazard.hazard) {
855 skip |= LogError(srcImage, string_SyncHazardVUID(hazard.hazard), "Hazard %s for srcImage %s, region %" PRIu32,
856 string_SyncHazard(hazard.hazard), report_data->FormatHandle(srcImage).c_str(), region);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700857 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600858 }
859
860 if (dst_image) {
locke-lunarg1df1f882020-03-02 16:42:08 -0700861 VkExtent3D dst_copy_extent =
862 GetAdjustedDestImageExtent(src_image->createInfo.format, dst_image->createInfo.format, copy_region.extent);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600863 auto hazard = context->DetectHazard(*dst_image, SYNC_TRANSFER_TRANSFER_WRITE, copy_region.dstSubresource,
locke-lunarg1df1f882020-03-02 16:42:08 -0700864 copy_region.dstOffset, dst_copy_extent);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600865 if (hazard.hazard) {
866 skip |= LogError(dstImage, string_SyncHazardVUID(hazard.hazard), "Hazard %s for dstImage %s, region %" PRIu32,
867 string_SyncHazard(hazard.hazard), report_data->FormatHandle(dstImage).c_str(), region);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700868 }
869 }
870 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600871
John Zulauf5c5e88d2019-12-26 11:22:02 -0700872 return skip;
873}
874
875void SyncValidator::PreCallRecordCmdCopyImage(VkCommandBuffer commandBuffer, VkImage srcImage, VkImageLayout srcImageLayout,
876 VkImage dstImage, VkImageLayout dstImageLayout, uint32_t regionCount,
877 const VkImageCopy *pRegions) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600878 auto *cb_access_context = GetAccessContext(commandBuffer);
879 assert(cb_access_context);
880 auto *context = cb_access_context->GetCurrentAccessContext();
881 assert(context);
882
John Zulauf5c5e88d2019-12-26 11:22:02 -0700883 auto *src_image = Get<IMAGE_STATE>(srcImage);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600884 auto *src_tracker = context->GetAccessTracker(VulkanTypedHandle(srcImage, kVulkanObjectTypeImage));
John Zulauf5c5e88d2019-12-26 11:22:02 -0700885 auto *dst_image = Get<IMAGE_STATE>(dstImage);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600886 auto *dst_tracker = context->GetAccessTracker(VulkanTypedHandle(dstImage, kVulkanObjectTypeImage));
John Zulauf5c5e88d2019-12-26 11:22:02 -0700887
888 for (uint32_t region = 0; region < regionCount; region++) {
889 const auto &copy_region = pRegions[region];
John Zulauf3d84f1b2020-03-09 13:33:25 -0600890 if (src_image) {
891 src_tracker->UpdateAccessState(*src_image, SYNC_TRANSFER_TRANSFER_READ, copy_region.srcSubresource,
892 copy_region.srcOffset, copy_region.extent, tag);
John Zulauf5c5e88d2019-12-26 11:22:02 -0700893 }
John Zulauf3d84f1b2020-03-09 13:33:25 -0600894 if (dst_image) {
locke-lunarg1df1f882020-03-02 16:42:08 -0700895 VkExtent3D dst_copy_extent =
896 GetAdjustedDestImageExtent(src_image->createInfo.format, dst_image->createInfo.format, copy_region.extent);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600897 dst_tracker->UpdateAccessState(*dst_image, SYNC_TRANSFER_TRANSFER_WRITE, copy_region.dstSubresource,
locke-lunarg1df1f882020-03-02 16:42:08 -0700898 copy_region.dstOffset, dst_copy_extent, tag);
John Zulauf9cb530d2019-09-30 14:14:10 -0600899 }
900 }
901}
902
903bool SyncValidator::PreCallValidateCmdPipelineBarrier(VkCommandBuffer commandBuffer, VkPipelineStageFlags srcStageMask,
904 VkPipelineStageFlags dstStageMask, VkDependencyFlags dependencyFlags,
905 uint32_t memoryBarrierCount, const VkMemoryBarrier *pMemoryBarriers,
906 uint32_t bufferMemoryBarrierCount,
907 const VkBufferMemoryBarrier *pBufferMemoryBarriers,
908 uint32_t imageMemoryBarrierCount,
909 const VkImageMemoryBarrier *pImageMemoryBarriers) const {
910 bool skip = false;
John Zulauf3d84f1b2020-03-09 13:33:25 -0600911 const auto *cb_access_context = GetAccessContext(commandBuffer);
912 assert(cb_access_context);
913 if (!cb_access_context) return skip;
John Zulauf0cb5be22020-01-23 12:18:22 -0700914
John Zulauf3d84f1b2020-03-09 13:33:25 -0600915 const auto *context = cb_access_context->GetCurrentAccessContext();
916 assert(context);
917 if (!context) return skip;
John Zulauf0cb5be22020-01-23 12:18:22 -0700918
John Zulauf3d84f1b2020-03-09 13:33:25 -0600919 const auto src_stage_mask = ExpandPipelineStages(cb_access_context->GetQueueFlags(), srcStageMask);
John Zulauf36bcf6a2020-02-03 15:12:52 -0700920 const auto src_exec_scope = WithEarlierPipelineStages(src_stage_mask);
921 auto src_stage_accesses = AccessScopeByStage(src_stage_mask);
John Zulauf0cb5be22020-01-23 12:18:22 -0700922 // Validate Image Layout transitions
923 for (uint32_t index = 0; index < imageMemoryBarrierCount; index++) {
924 const auto &barrier = pImageMemoryBarriers[index];
925 if (barrier.newLayout == barrier.oldLayout) continue; // Only interested in layout transitions at this point.
926 const auto *image_state = Get<IMAGE_STATE>(barrier.image);
927 if (!image_state) continue;
John Zulauf3d84f1b2020-03-09 13:33:25 -0600928 const auto hazard = DetectImageBarrierHazard(*context, *image_state, src_exec_scope, src_stage_accesses, barrier);
John Zulauf0cb5be22020-01-23 12:18:22 -0700929 if (hazard.hazard) {
930 // TODO -- add tag information to log msg when useful.
931 skip |= LogError(barrier.image, string_SyncHazardVUID(hazard.hazard), "Hazard %s for image barrier %" PRIu32 " %s",
932 string_SyncHazard(hazard.hazard), index, report_data->FormatHandle(barrier.image).c_str());
933 }
934 }
John Zulauf9cb530d2019-09-30 14:14:10 -0600935
936 return skip;
937}
938
939void SyncValidator::PreCallRecordCmdPipelineBarrier(VkCommandBuffer commandBuffer, VkPipelineStageFlags srcStageMask,
940 VkPipelineStageFlags dstStageMask, VkDependencyFlags dependencyFlags,
941 uint32_t memoryBarrierCount, const VkMemoryBarrier *pMemoryBarriers,
942 uint32_t bufferMemoryBarrierCount,
943 const VkBufferMemoryBarrier *pBufferMemoryBarriers,
944 uint32_t imageMemoryBarrierCount,
945 const VkImageMemoryBarrier *pImageMemoryBarriers) {
John Zulauf3d84f1b2020-03-09 13:33:25 -0600946 auto *cb_access_context = GetAccessContext(commandBuffer);
947 assert(cb_access_context);
948 if (!cb_access_context) return;
949 auto access_context = cb_access_context->GetCurrentAccessContext();
950 assert(access_context);
951 if (!access_context) return;
John Zulauf9cb530d2019-09-30 14:14:10 -0600952
John Zulauf3d84f1b2020-03-09 13:33:25 -0600953 const auto src_stage_mask = ExpandPipelineStages(cb_access_context->GetQueueFlags(), srcStageMask);
John Zulauf36bcf6a2020-02-03 15:12:52 -0700954 auto src_stage_accesses = AccessScopeByStage(src_stage_mask);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600955 const auto dst_stage_mask = ExpandPipelineStages(cb_access_context->GetQueueFlags(), dstStageMask);
John Zulauf36bcf6a2020-02-03 15:12:52 -0700956 auto dst_stage_accesses = AccessScopeByStage(dst_stage_mask);
957 const auto src_exec_scope = WithEarlierPipelineStages(src_stage_mask);
958 const auto dst_exec_scope = WithLaterPipelineStages(dst_stage_mask);
John Zulauf3d84f1b2020-03-09 13:33:25 -0600959 ApplyBufferBarriers(access_context, src_exec_scope, src_stage_accesses, dst_exec_scope, dst_stage_accesses,
960 bufferMemoryBarrierCount, pBufferMemoryBarriers);
961 ApplyImageBarriers(access_context, src_exec_scope, src_stage_accesses, dst_exec_scope, dst_stage_accesses,
962 imageMemoryBarrierCount, pImageMemoryBarriers);
John Zulauf9cb530d2019-09-30 14:14:10 -0600963
964 // Apply these last in-case there operation is a superset of the other two and would clean them up...
John Zulauf3d84f1b2020-03-09 13:33:25 -0600965 ApplyGlobalBarriers(access_context, src_exec_scope, dst_exec_scope, src_stage_accesses, dst_stage_accesses, memoryBarrierCount,
John Zulauf0cb5be22020-01-23 12:18:22 -0700966 pMemoryBarriers);
John Zulauf9cb530d2019-09-30 14:14:10 -0600967}
968
969void SyncValidator::PostCallRecordCreateDevice(VkPhysicalDevice gpu, const VkDeviceCreateInfo *pCreateInfo,
970 const VkAllocationCallbacks *pAllocator, VkDevice *pDevice, VkResult result) {
971 // The state tracker sets up the device state
972 StateTracker::PostCallRecordCreateDevice(gpu, pCreateInfo, pAllocator, pDevice, result);
973
974 // Add the callback hooks for the functions that are either broadly or deeply used and that the ValidationStateTracker refactor
975 // would be messier without.
976 // TODO: Find a good way to do this hooklessly.
977 ValidationObject *device_object = GetLayerDataPtr(get_dispatch_key(*pDevice), layer_data_map);
978 ValidationObject *validation_data = GetValidationObject(device_object->object_dispatch, LayerObjectTypeSyncValidation);
979 SyncValidator *sync_device_state = static_cast<SyncValidator *>(validation_data);
980
981 sync_device_state->SetCommandBufferResetCallback(
982 [sync_device_state](VkCommandBuffer command_buffer) -> void { sync_device_state->ResetCommandBuffer(command_buffer); });
983}
John Zulauf3d84f1b2020-03-09 13:33:25 -0600984
985void SyncValidator::PostCallRecordBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo,
986 VkResult result) {
987 // The state tracker sets up the command buffer state
988 StateTracker::PostCallRecordBeginCommandBuffer(commandBuffer, pBeginInfo, result);
989
990 // Create/initialize the structure that trackers accesses at the command buffer scope.
991 auto cb_access_context = GetAccessContext(commandBuffer);
992 assert(cb_access_context);
993 cb_access_context->Reset();
994}
995
996void SyncValidator::RecordCmdBeginRenderPass(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo *pRenderPassBegin,
997 const VkSubpassBeginInfo *pSubpassBeginInfo) {
998 const auto rp_state = Get<RENDER_PASS_STATE>(pRenderPassBegin->renderPass);
999 auto cb_context = GetAccessContext(commandBuffer);
1000 if (rp_state && cb_context) {
1001 cb_context->BeginRenderPass(*rp_state);
1002 }
1003}
1004
1005void SyncValidator::PostCallRecordCmdBeginRenderPass(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo *pRenderPassBegin,
1006 VkSubpassContents contents) {
1007 StateTracker::PostCallRecordCmdBeginRenderPass(commandBuffer, pRenderPassBegin, contents);
1008 auto subpass_begin_info = lvl_init_struct<VkSubpassBeginInfo>();
1009 subpass_begin_info.contents = contents;
1010 RecordCmdBeginRenderPass(commandBuffer, pRenderPassBegin, &subpass_begin_info);
1011}
1012
1013void SyncValidator::PostCallRecordCmdBeginRenderPass2(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo *pRenderPassBegin,
1014 const VkSubpassBeginInfo *pSubpassBeginInfo) {
1015 StateTracker::PostCallRecordCmdBeginRenderPass2(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
1016 RecordCmdBeginRenderPass(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
1017}
1018
1019void SyncValidator::PostCallRecordCmdBeginRenderPass2KHR(VkCommandBuffer commandBuffer,
1020 const VkRenderPassBeginInfo *pRenderPassBegin,
1021 const VkSubpassBeginInfo *pSubpassBeginInfo) {
1022 StateTracker::PostCallRecordCmdBeginRenderPass2KHR(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
1023 RecordCmdBeginRenderPass(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
1024}
1025
1026void SyncValidator::RecordCmdNextSubpass(VkCommandBuffer commandBuffer, const VkSubpassBeginInfo *pSubpassBeginInfo,
1027 const VkSubpassEndInfo *pSubpassEndInfo) {
1028 auto cb_context = GetAccessContext(commandBuffer);
1029 assert(cb_context);
1030 auto cb_state = cb_context->GetCommandBufferState();
1031 if (!cb_state) return;
1032
1033 auto rp_state = cb_state->activeRenderPass;
1034 if (!rp_state) return;
1035
1036 cb_context->NextRenderPass(*rp_state);
1037}
1038
1039void SyncValidator::PostCallRecordCmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents) {
1040 StateTracker::PostCallRecordCmdNextSubpass(commandBuffer, contents);
1041 auto subpass_begin_info = lvl_init_struct<VkSubpassBeginInfo>();
1042 subpass_begin_info.contents = contents;
1043 RecordCmdNextSubpass(commandBuffer, &subpass_begin_info, nullptr);
1044}
1045
1046void SyncValidator::PostCallRecordCmdNextSubpass2(VkCommandBuffer commandBuffer, const VkSubpassBeginInfo *pSubpassBeginInfo,
1047 const VkSubpassEndInfo *pSubpassEndInfo) {
1048 StateTracker::PostCallRecordCmdNextSubpass2(commandBuffer, pSubpassBeginInfo, pSubpassEndInfo);
1049 RecordCmdNextSubpass(commandBuffer, pSubpassBeginInfo, pSubpassEndInfo);
1050}
1051
1052void SyncValidator::PostCallRecordCmdNextSubpass2KHR(VkCommandBuffer commandBuffer, const VkSubpassBeginInfo *pSubpassBeginInfo,
1053 const VkSubpassEndInfo *pSubpassEndInfo) {
1054 StateTracker::PostCallRecordCmdNextSubpass2KHR(commandBuffer, pSubpassBeginInfo, pSubpassEndInfo);
1055 RecordCmdNextSubpass(commandBuffer, pSubpassBeginInfo, pSubpassEndInfo);
1056}
1057
1058void SyncValidator::RecordCmdEndRenderPass(VkCommandBuffer commandBuffer, const VkSubpassEndInfo *pSubpassEndInfo) {}
1059
1060void SyncValidator::PostCallRecordCmdEndRenderPass(VkCommandBuffer commandBuffer) {
1061 StateTracker::PostCallRecordCmdEndRenderPass(commandBuffer);
1062 RecordCmdEndRenderPass(commandBuffer, nullptr);
1063}
1064
1065void SyncValidator::PostCallRecordCmdEndRenderPass2(VkCommandBuffer commandBuffer, const VkSubpassEndInfo *pSubpassEndInfo) {
1066 StateTracker::PostCallRecordCmdEndRenderPass2(commandBuffer, pSubpassEndInfo);
1067 RecordCmdEndRenderPass(commandBuffer, pSubpassEndInfo);
1068}
1069
1070void SyncValidator::PostCallRecordCmdEndRenderPass2KHR(VkCommandBuffer commandBuffer, const VkSubpassEndInfo *pSubpassEndInfo) {
1071 StateTracker::PostCallRecordCmdEndRenderPass2KHR(commandBuffer, pSubpassEndInfo);
1072 RecordCmdEndRenderPass(commandBuffer, pSubpassEndInfo);
1073}