layers/synchronization_validation.cpp

/* Copyright (c) 2019 The Khronos Group Inc.
 * Copyright (c) 2019 Valve Corporation
 * Copyright (c) 2019 LunarG, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Author: John Zulauf <jzulauf@lunarg.com>
 */

#include <limits>
#include <vector>
#include "synchronization_validation.h"

static const char *string_SyncHazardVUID(SyncHazard hazard) {
    switch (hazard) {
        case SyncHazard::NONE:
            return "SYNC-NONE";
            break;
        case SyncHazard::READ_AFTER_WRITE:
            return "SYNC-HAZARD-READ_AFTER_WRITE";
            break;
        case SyncHazard::WRITE_AFTER_READ:
            return "SYNC-HAZARD-WRITE_AFTER_READ";
            break;
        case SyncHazard::WRITE_AFTER_WRITE:
            return "SYNC-HAZARD-WRITE_AFTER_WRITE";
            break;
        default:
            assert(0);
    }
    return "SYNC-HAZARD-INVALID";
}

static const char *string_SyncHazard(SyncHazard hazard) {
    switch (hazard) {
        case SyncHazard::NONE:
            return "NONR";
            break;
        case SyncHazard::READ_AFTER_WRITE:
            return "READ_AFTER_WRITE";
            break;
        case SyncHazard::WRITE_AFTER_READ:
            return "WRITE_AFTER_READ";
            break;
        case SyncHazard::WRITE_AFTER_WRITE:
            return "WRITE_AFTER_WRITE";
            break;
        default:
            assert(0);
    }
    return "INVALID HAZARD";
}

static const MemoryAccessRange full_range(std::numeric_limits<VkDeviceSize>::min(), std::numeric_limits<VkDeviceSize>::max());
static MemoryAccessRange MakeMemoryAccessRange(const BUFFER_STATE &buffer, VkDeviceSize offset, VkDeviceSize size) {
    assert(!buffer.sparse);
    const auto base = offset + buffer.binding.offset;
    return MemoryAccessRange(base, base + size);
}

template <typename Flags, typename Map>
SyncStageAccessFlags AccessScopeImpl(Flags flag_mask, const Map &map) {
    SyncStageAccessFlags scope = 0;
    for (const auto &bit_scope : map) {
        if (flag_mask < bit_scope.first) break;

        if (flag_mask & bit_scope.first) {
            scope |= bit_scope.second;
        }
    }
    return scope;
}

SyncStageAccessFlags SyncStageAccess::AccessScopeByStage(VkPipelineStageFlags stages) {
    return AccessScopeImpl(stages, syncStageAccessMaskByStageBit);
}

SyncStageAccessFlags SyncStageAccess::AccessScopeByAccess(VkAccessFlags accesses) {
    return AccessScopeImpl(accesses, syncStageAccessMaskByAccessBit);
}

// Getting from stage mask and access mask to stage/acess masks is something we need to be good at...
SyncStageAccessFlags SyncStageAccess::AccessScope(VkPipelineStageFlags stages, VkAccessFlags accesses) {
    // The access scope is the intersection of all stage/access types possible for the enabled stages and the enables accesses
    // (after doing a couple factoring of common terms the union of stage/access intersections is the intersections of the
    // union of all stage/access types for all the stages and the same unions for the access mask...
    return AccessScopeByStage(stages) & AccessScopeByAccess(accesses);
}

template <typename Action>
void UpdateMemoryAccessState(MemoryAccessRangeMap *accesses, const MemoryAccessRange &range, const Action &action) {
    // TODO -- region/mem-range accuracte update
    auto pos = accesses->lower_bound(range);
    if (pos == accesses->end() || !pos->first.intersects(range)) {
        // The range is empty, fill it with a default value.
        pos = action.Infill(accesses, pos, range);
    } else if (range.begin < pos->first.begin) {
        // Leading empty space, infill
        pos = action.Infill(accesses, pos, MemoryAccessRange(range.begin, pos->first.begin));
    } else if (pos->first.begin < range.begin) {
        // Trim the beginning if needed
        pos = accesses->split(pos, range.begin, sparse_container::split_op_keep_both());
        ++pos;
    }

    const auto the_end = accesses->end();
    while ((pos != the_end) && pos->first.intersects(range)) {
        if (pos->first.end > range.end) {
            pos = accesses->split(pos, range.end, sparse_container::split_op_keep_both());
        }

        pos = action(accesses, pos);
        if (pos == the_end) break;

        auto next = pos;
        ++next;
        if ((pos->first.end < range.end) && (next != the_end) && !next->first.is_subsequent_to(pos->first)) {
            // Need to infill if next is disjoint
            VkDeviceSize limit = (next == the_end) ? range.end : std::min(range.end, next->first.begin);
            MemoryAccessRange new_range(pos->first.end, limit);
            next = action.Infill(accesses, next, new_range);
        }
        pos = next;
    }
}

struct UpdateMemoryAccessStateFunctor {
    using Iterator = MemoryAccessRangeMap::iterator;
    Iterator Infill(MemoryAccessRangeMap *accesses, Iterator pos, MemoryAccessRange range) const {
        return accesses->insert(pos, std::make_pair(range, ResourceAccessState()));
    }
    Iterator operator()(MemoryAccessRangeMap *accesses, Iterator pos) const {
        auto &access_state = pos->second;
        access_state.Update(usage, tag);
        return pos;
    }

    UpdateMemoryAccessStateFunctor(SyncStageAccessIndex usage_, const ResourceUsageTag &tag_) : usage(usage_), tag(tag_) {}
    SyncStageAccessIndex usage;
    const ResourceUsageTag &tag;
};

struct ApplyMemoryAccessBarrierFunctor {
    using Iterator = MemoryAccessRangeMap::iterator;
    inline Iterator Infill(MemoryAccessRangeMap *accesses, Iterator pos, MemoryAccessRange range) const { return pos; }

    Iterator operator()(MemoryAccessRangeMap *accesses, Iterator pos) const {
        auto &access_state = pos->second;
        access_state.ApplyMemoryAccessBarrier(src_stage_mask, src_scope, dst_stage_mask, dst_scope);
        return pos;
    }

    ApplyMemoryAccessBarrierFunctor(VkPipelineStageFlags src_stage_mask_, SyncStageAccessFlags src_scope_,
                                    VkPipelineStageFlags dst_stage_mask_, SyncStageAccessFlags dst_scope_)
        : src_stage_mask(src_stage_mask_), src_scope(src_scope_), dst_stage_mask(dst_stage_mask_), dst_scope(dst_scope_) {}

    VkPipelineStageFlags src_stage_mask;
    SyncStageAccessFlags src_scope;
    VkPipelineStageFlags dst_stage_mask;
    SyncStageAccessFlags dst_scope;
};

struct ApplyGlobalBarrierFunctor {
    using Iterator = MemoryAccessRangeMap::iterator;
    inline Iterator Infill(MemoryAccessRangeMap *accesses, Iterator pos, MemoryAccessRange range) const { return pos; }

    Iterator operator()(MemoryAccessRangeMap *accesses, Iterator pos) const {
        auto &access_state = pos->second;
        access_state.ApplyExecutionBarrier(src_stage_mask, dst_stage_mask);

        for (const auto &functor : barrier_functor) {
            functor(accesses, pos);
        }
        return pos;
    }

    ApplyGlobalBarrierFunctor(VkPipelineStageFlags srcStageMask, VkPipelineStageFlags dstStageMask,
                              SyncStageAccessFlags src_stage_scope, SyncStageAccessFlags dst_stage_scope,
                              uint32_t memoryBarrierCount, const VkMemoryBarrier *pMemoryBarriers)
        : src_stage_mask(srcStageMask), dst_stage_mask(dstStageMask) {
        // Don't want to create this per tracked item, but don't want to loop through all tracked items per barrier...
        barrier_functor.reserve(memoryBarrierCount);
        for (uint32_t barrier_index = 0; barrier_index < memoryBarrierCount; barrier_index++) {
            const auto &barrier = pMemoryBarriers[barrier_index];
            barrier_functor.emplace_back(srcStageMask, SyncStageAccess::AccessScope(src_stage_scope, barrier.srcAccessMask),
                                         dstStageMask, SyncStageAccess::AccessScope(dst_stage_scope, barrier.dstAccessMask));
        }
    }

    const VkPipelineStageFlags src_stage_mask;
    const VkPipelineStageFlags dst_stage_mask;
    std::vector<ApplyMemoryAccessBarrierFunctor> barrier_functor;
};

HazardResult ResourceAccessState::DetectHazard(SyncStageAccessIndex usage_index) const {
    HazardResult hazard;
    auto usage = FlagBit(usage_index);
    if (IsRead(usage)) {
        if (IsWriteHazard(usage)) {
            hazard.Set(READ_AFTER_WRITE, write_tag);
        }
    } else {
        // Assume write
        // TODO determine what to do with READ-WRITE usage states if any
        // Write-After-Write check -- if we have a previous write to test against
        if (last_write && IsWriteHazard(usage)) {
            hazard.Set(WRITE_AFTER_WRITE, write_tag);
        } else {
            // Only look for casus belli for WAR
            const auto usage_stage = PipelineStageBit(usage_index);
            for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
                if (IsReadHazard(usage_stage, last_reads[read_index])) {
                    hazard.Set(WRITE_AFTER_READ, last_reads[read_index].tag);
                    break;
                }
            }
        }
    }
    return hazard;
}

void ResourceAccessState::Update(SyncStageAccessIndex usage_index, const ResourceUsageTag &tag) {
    // Move this logic in the ResourceStateTracker as methods, thereof (or we'll repeat it for every flavor of resource...
    const auto usage_bit = FlagBit(usage_index);
    if (IsRead(usage_index)) {
        // Mulitple outstanding reads may be of interest and do dependency chains independently
        // However, for purposes of barrier tracking, only one read per pipeline stage matters
        const auto usage_stage = PipelineStageBit(usage_index);
        if (usage_stage & last_read_stages) {
            for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
                ReadState &access = last_reads[read_index];
                if (access.stage == usage_stage) {
                    access.barriers = 0;
                    access.tag = tag;
                    break;
                }
            }
        } else {
            // We don't have this stage in the list yet...
            assert(last_read_count < last_reads.size());
            ReadState &access = last_reads[last_read_count++];
            access.stage = usage_stage;
            access.barriers = 0;
            access.tag = tag;
            last_read_stages |= usage_stage;
        }
    } else {
        // Assume write
        // TODO determine what to do with READ-WRITE operations if any
        // Clobber last read and both sets of barriers... because all we have is DANGER, DANGER, WILL ROBINSON!!!
        // if the last_reads/last_write were unsafe, we've reported them,
        // in either case the prior access is irrelevant, we can overwrite them as *this* write is now after them
        last_read_count = 0;
        last_read_stages = 0;

        write_barriers = 0;
        write_dependency_chain = 0;
        write_tag = tag;
        last_write = usage_bit;
    }
}
void ResourceAccessState::ApplyExecutionBarrier(VkPipelineStageFlags srcStageMask, VkPipelineStageFlags dstStageMask) {
    // Execution Barriers only protect read operations
    for (uint32_t read_index = 0; read_index < last_read_count; read_index++) {
        ReadState &access = last_reads[read_index];
        // The | implements the "dependency chain" logic for this access, as the barriers field stores the second sync scope
        if (srcStageMask & (access.stage | access.barriers)) {
            access.barriers |= dstStageMask;
        }
    }
    if (write_dependency_chain & srcStageMask) write_dependency_chain |= dstStageMask;
}

void ResourceAccessState::ApplyMemoryAccessBarrier(VkPipelineStageFlags src_stage_mask, SyncStageAccessFlags src_scope,
                                                   VkPipelineStageFlags dst_stage_mask, SyncStageAccessFlags dst_scope) {
    // Assuming we've applied the execution side of this barrier, we update just the write
    // The || implements the "dependency chain" logic for this barrier
    if ((src_scope & last_write) || (write_dependency_chain & src_stage_mask)) {
        write_barriers |= dst_scope;
        write_dependency_chain |= dst_stage_mask;
    }
}

void SyncValidator::ResetCommandBuffer(VkCommandBuffer command_buffer) {
    auto *tracker = GetAccessTrackerNoInsert(command_buffer);
    if (tracker) {
        tracker->Reset();
    }
}

void SyncValidator::ApplyGlobalBarriers(MemoryAccessTracker *tracker, VkPipelineStageFlags srcStageMask,
                                        VkPipelineStageFlags dstStageMask, SyncStageAccessFlags src_stage_scope,
                                        SyncStageAccessFlags dst_stage_scope, uint32_t memoryBarrierCount,
                                        const VkMemoryBarrier *pMemoryBarriers) {
    // TODO: Implement this better (maybe some delayed/on-demand integration).
    ApplyGlobalBarrierFunctor barriers_functor(srcStageMask, dstStageMask, src_stage_scope, dst_stage_scope, memoryBarrierCount,
                                               pMemoryBarriers);
    for (auto &mem_access_pair : tracker->map) {  // TODO hide the tracker details
        UpdateMemoryAccessState(&mem_access_pair.second, full_range, barriers_functor);
    }
}

void SyncValidator::ApplyBufferBarriers(MemoryAccessTracker *tracker, VkPipelineStageFlags src_stage_mask,
                                        SyncStageAccessFlags src_stage_scope, VkPipelineStageFlags dst_stage_mask,
                                        SyncStageAccessFlags dst_stage_scope, uint32_t barrier_count,
                                        const VkBufferMemoryBarrier *barriers) {
    // TODO Implement this at subresource/memory_range accuracy
    for (uint32_t index = 0; index < barrier_count; index++) {
        const auto &barrier = barriers[index];
        const auto *buffer = Get<BUFFER_STATE>(barrier.buffer);
        if (!buffer) continue;
        auto *accesses = tracker->GetNoInsert(buffer->binding.mem_state->mem);
        if (!accesses) continue;
        MemoryAccessRange range = MakeMemoryAccessRange(*buffer, barrier.offset, barrier.size);
        UpdateMemoryAccessState(
            accesses, range,
            ApplyMemoryAccessBarrierFunctor(src_stage_mask, AccessScope(src_stage_scope, barrier.srcAccessMask), dst_stage_mask,
                                            AccessScope(dst_stage_scope, barrier.dstAccessMask)));
    }
}

void SyncValidator::ApplyImageBarriers(MemoryAccessTracker *tracker, SyncStageAccessFlags src_stage_scope,
                                       SyncStageAccessFlags dst_stage_scope, uint32_t imageMemoryBarrierCount,
                                       const VkImageMemoryBarrier *pImageMemoryBarriers) {
    // TODO: Implement this. First pass a sub-resource (not-memory) accuracy
}

HazardResult SyncValidator::DetectHazard(const MemoryAccessRangeMap &accesses, SyncStageAccessIndex current_usage,
                                         const MemoryAccessRange &range) const {
    const auto from = accesses.lower_bound(range);
    const auto to = accesses.upper_bound(range);
    for (auto pos = from; pos != to; ++pos) {
        const auto &access_state = pos->second;
        HazardResult hazard = access_state.DetectHazard(current_usage);
        if (hazard.hazard) return hazard;
    }
    return HazardResult();
}

void SyncValidator::UpdateAccessState(MemoryAccessRangeMap *accesses, SyncStageAccessIndex current_usage,
                                      const MemoryAccessRange &range) {
    UpdateMemoryAccessStateFunctor action(current_usage, tag);
    UpdateMemoryAccessState(accesses, range, action);
}

bool SyncValidator::PreCallValidateCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer,
                                                 uint32_t regionCount, const VkBufferCopy *pRegions) const {
    bool skip = false;
    const auto *const const_this = this;
    const auto *tracker = const_this->GetAccessTracker(commandBuffer);
    if (tracker) {
        // If we have no previous accesses, we have no hazards
        // TODO: make this sub-resource capable
        // TODO: make this general, and stuff it into templates/utility functions
        const auto *src_buffer = Get<BUFFER_STATE>(srcBuffer);
        const auto src_access = (src_buffer && !src_buffer->sparse) ? tracker->Get(src_buffer->binding.mem_state->mem) : nullptr;
        const auto *dst_buffer = Get<BUFFER_STATE>(dstBuffer);
        const auto dst_access = (dst_buffer && !dst_buffer->sparse) ? tracker->Get(dst_buffer->binding.mem_state->mem) : nullptr;

        for (uint32_t region = 0; region < regionCount; region++) {
            const auto &copy_region = pRegions[region];
            if (src_access) {
                MemoryAccessRange src_range = MakeMemoryAccessRange(*src_buffer, copy_region.srcOffset, copy_region.size);
                auto hazard = DetectHazard(*src_access, SYNC_TRANSFER_TRANSFER_READ, src_range);
                if (hazard.hazard) {
                    // TODO -- add tag information to log msg when useful.
                    skip |= LogError(srcBuffer, string_SyncHazardVUID(hazard.hazard), "Hazard %s for srcBuffer %s, region %" PRIu32,
                                     string_SyncHazard(hazard.hazard), report_data->FormatHandle(srcBuffer).c_str(), region);
                }
            }
            if (dst_access && !skip) {
                MemoryAccessRange dst_range = MakeMemoryAccessRange(*dst_buffer, copy_region.dstOffset, copy_region.size);
                auto hazard = DetectHazard(*dst_access, SYNC_TRANSFER_TRANSFER_WRITE, dst_range);
                if (hazard.hazard) {
                    skip |= LogError(dstBuffer, string_SyncHazardVUID(hazard.hazard), "Hazard %s for dstBuffer %s, region %" PRIu32,
                                     string_SyncHazard(hazard.hazard), report_data->FormatHandle(dstBuffer).c_str(), region);
                }
            }
            if (skip) break;
        }
    }
    return skip;
}

void SyncValidator::PreCallRecordCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer,
                                               uint32_t regionCount, const VkBufferCopy *pRegions) {
    auto *tracker = GetAccessTracker(commandBuffer);
    assert(tracker);
    const auto *src_buffer = Get<BUFFER_STATE>(srcBuffer);
    const auto src_access = (src_buffer && !src_buffer->sparse) ? tracker->Get(src_buffer->binding.mem_state->mem) : nullptr;
    const auto *dst_buffer = Get<BUFFER_STATE>(dstBuffer);
    const auto dst_access = (dst_buffer && !dst_buffer->sparse) ? tracker->Get(dst_buffer->binding.mem_state->mem) : nullptr;

    for (uint32_t region = 0; region < regionCount; region++) {
        const auto &copy_region = pRegions[region];
        if (src_access) {
            MemoryAccessRange src_range = MakeMemoryAccessRange(*src_buffer, copy_region.srcOffset, copy_region.size);
            UpdateAccessState(src_access, SYNC_TRANSFER_TRANSFER_READ, src_range);
        }
        if (dst_access) {
            MemoryAccessRange dst_range = MakeMemoryAccessRange(*dst_buffer, copy_region.dstOffset, copy_region.size);
            UpdateAccessState(dst_access, SYNC_TRANSFER_TRANSFER_WRITE, dst_range);
        }
    }
}

bool SyncValidator::PreCallValidateCmdPipelineBarrier(VkCommandBuffer commandBuffer, VkPipelineStageFlags srcStageMask,
                                                      VkPipelineStageFlags dstStageMask, VkDependencyFlags dependencyFlags,
                                                      uint32_t memoryBarrierCount, const VkMemoryBarrier *pMemoryBarriers,
                                                      uint32_t bufferMemoryBarrierCount,
                                                      const VkBufferMemoryBarrier *pBufferMemoryBarriers,
                                                      uint32_t imageMemoryBarrierCount,
                                                      const VkImageMemoryBarrier *pImageMemoryBarriers) const {
    bool skip = false;

    return skip;
}

void SyncValidator::PreCallRecordCmdPipelineBarrier(VkCommandBuffer commandBuffer, VkPipelineStageFlags srcStageMask,
                                                    VkPipelineStageFlags dstStageMask, VkDependencyFlags dependencyFlags,
                                                    uint32_t memoryBarrierCount, const VkMemoryBarrier *pMemoryBarriers,
                                                    uint32_t bufferMemoryBarrierCount,
                                                    const VkBufferMemoryBarrier *pBufferMemoryBarriers,
                                                    uint32_t imageMemoryBarrierCount,
                                                    const VkImageMemoryBarrier *pImageMemoryBarriers) {
    // Just implement the buffer barrier for now
    auto *tracker = GetAccessTracker(commandBuffer);
    assert(tracker);
    auto src_stage_scope = AccessScopeByStage(srcStageMask);
    auto dst_stage_scope = AccessScopeByStage(dstStageMask);

    ApplyBufferBarriers(tracker, srcStageMask, src_stage_scope, dstStageMask, dst_stage_scope, bufferMemoryBarrierCount,
                        pBufferMemoryBarriers);
    ApplyImageBarriers(tracker, src_stage_scope, dst_stage_scope, imageMemoryBarrierCount, pImageMemoryBarriers);

    // Apply these last in-case there operation is a superset of the other two and would clean them up...
    ApplyGlobalBarriers(tracker, srcStageMask, dstStageMask, src_stage_scope, dst_stage_scope, memoryBarrierCount, pMemoryBarriers);
}

void SyncValidator::PostCallRecordCreateDevice(VkPhysicalDevice gpu, const VkDeviceCreateInfo *pCreateInfo,
                                               const VkAllocationCallbacks *pAllocator, VkDevice *pDevice, VkResult result) {
    // The state tracker sets up the device state
    StateTracker::PostCallRecordCreateDevice(gpu, pCreateInfo, pAllocator, pDevice, result);

    // Add the callback hooks for the functions that are either broadly or deeply used and that the ValidationStateTracker refactor
    // would be messier without.
    // TODO: Find a good way to do this hooklessly.
    ValidationObject *device_object = GetLayerDataPtr(get_dispatch_key(*pDevice), layer_data_map);
    ValidationObject *validation_data = GetValidationObject(device_object->object_dispatch, LayerObjectTypeSyncValidation);
    SyncValidator *sync_device_state = static_cast<SyncValidator *>(validation_data);

    sync_device_state->SetCommandBufferResetCallback(
        [sync_device_state](VkCommandBuffer command_buffer) -> void { sync_device_state->ResetCommandBuffer(command_buffer); });
}