text: // Copyright 2019 The Fuchsia Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  
  #include "block_pool.h"
  
  #include "common/vk/assert.h"
  #include "common/vk/barrier.h"
  #include "device.h"
  #include "dispatch.h"
  #include "handle_pool.h"
  #include "queue_pool.h"
  #include "spinel_assert.h"
  #include "vk.h"
  #include "vk_target.h"
  
  //
  //
  //
  
  #ifdef SPN_BP_DEBUG
  
  #include <stdio.h>
  
  #include "common/vk/barrier.h"
  
  #define SPN_BP_DEBUG_SIZE ((size_t)1 << 24)
  
  #endif
  
  //
  //
  //
  
  struct spn_block_pool
  {
    struct spn_vk_ds_block_pool_t ds_block_pool;
  
  #ifdef SPN_BP_DEBUG
    struct
    {
      struct
      {
        VkDescriptorBufferInfo * dbi;
        VkDeviceMemory           dm;
      } d;
      struct
      {
        VkDescriptorBufferInfo dbi;
        VkDeviceMemory         dm;
  
        SPN_VK_BUFFER_NAME(block_pool, bp_debug) * mapped;
      } h;
    } bp_debug;
  #endif
  
    struct
    {
      VkDescriptorBufferInfo * dbi;
      VkDeviceMemory           dm;
    } bp_ids;
  
    struct
    {
      VkDescriptorBufferInfo * dbi;
      VkDeviceMemory           dm;
    } bp_blocks;
  
    struct
    {
      VkDescriptorBufferInfo * dbi;
      VkDeviceMemory           dm;
    } bp_host_map;
  
    uint32_t bp_size;
    uint32_t bp_mask;
  };
  
  //
  //
  //
  
  static uint32_t
  spn_pow2_ru_u32(uint32_t n)
  {
    n--;
    n |= n >> 1;
    n |= n >> 2;
    n |= n >> 4;
    n |= n >> 8;
    n |= n >> 16;
    n++;
  
    return n;
  }
  
  //
  //
  //
  
  #ifdef SPN_BP_DEBUG
  
  void
  spn_device_block_pool_debug_snap(struct spn_device * const device, VkCommandBuffer cb)
  {
    VkBufferCopy const bc = {
  
      .srcOffset = 0,
      .dstOffset = 0,
      .size      = SPN_VK_BUFFER_OFFSETOF(block_pool, bp_debug, bp_debug) + SPN_BP_DEBUG_SIZE
    };
  
    vk_barrier_debug(cb);
  
    vkCmdCopyBuffer(cb,
                    device->block_pool->bp_debug.d.dbi->buffer,
                    device->block_pool->bp_debug.h.dbi.buffer,
                    1,
                    &bc);
  
    vk_barrier_debug(cb);
  }
  
  void
  spn_device_block_pool_debug_print(struct spn_device * const device)
  {
    struct spn_vk_target_config const * const config = spn_vk_get_config(device->instance);
  
    if ((config->allocator.device.hr_dw.properties & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0)
      {
        VkMappedMemoryRange const mmr = { .sType  = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
                                          .pNext  = NULL,
                                          .memory = device->block_pool->bp_debug.h.dm,
                                          .offset = 0,
                                          .size   = VK_WHOLE_SIZE };
  
        vk(InvalidateMappedMemoryRanges(device->environment.d, 1, &mmr));
      }
  
    struct spn_vk_buf_block_pool_bp_debug const * mapped = device->block_pool->bp_debug.h.mapped;
    uint32_t const                                count  = mapped->bp_debug_count[0];
  
    //
    // HEX
    //
  #if 1
    {
      uint32_t const subgroup_size =
        MIN_MACRO(uint32_t, 32, 1 << config->p.group_sizes.named.paths_copy.subgroup_log2);
  
      fprintf(stderr, "[ %u ] = {", count);
  
      for (uint32_t ii = 0; ii < count; ii++)
        {
          if ((ii % subgroup_size) == 0)
            fprintf(stderr, "\n");
  
          fprintf(stderr, "%08X, ", mapped->bp_debug[ii]);
        }
  
      fprintf(stderr, "\n}\n");
    }
  #endif
  
    //
    // INT
    //
  #if 0
    {
      uint32_t const subgroup_size =
        MIN_MACRO(uint32_t, 32, 1 << config->p.group_sizes.named.paths_copy.subgroup_log2);
  
      fprintf(stderr, "[ %u ] = {", count);
  
      for (uint32_t ii = 0; ii < count; ii++)
        {
          if ((ii % subgroup_size) == 0)
            fprintf(stderr, "\n");
  
          fprintf(stderr, "%11d, ", mapped->bp_debug[ii]);
        }
  
      fprintf(stderr, "\n}\n");
    }
  #endif
  
    //
    // FLOAT
    //
  #if 0
    {
      uint32_t const subgroup_size =
        MIN_MACRO(uint32_t, 32, 1 << config->p.group_sizes.named.paths_copy.subgroup_log2);
  
      fprintf(stderr, "[ %u ] = {", count);
  
      float const * bp_debug_float = (float *)mapped->bp_debug;
  
      for (uint32_t ii = 0; ii < count; ii++)
        {
          if ((ii % subgroup_size) == 0)
            fprintf(stderr, "\n");
  
          fprintf(stderr, "%10.2f, ", bp_debug_float[ii]);
        }
  
      fprintf(stderr, "\n}\n");
    }
  #endif
  
    //
    // COORDS
    //
  #if 0
    {
      // FILE * file = fopen("debug.segs", "w");
  
      float const * bp_debug_float = (float *)mapped->bp_debug;
  
      for (uint32_t ii = 0; ii < count; ii += 4)
        {
          fprintf(stderr,
                  "{ { %10.2f, %10.2f }, { %10.2f, %10.2f } }\n",
                  bp_debug_float[ii + 0],
                  bp_debug_float[ii + 1],
                  bp_debug_float[ii + 2],
                  bp_debug_float[ii + 3]);
        }
  
      // fclose(file);
    }
  #endif
  
    //
    // TTS
    //
  #if 0
    fprintf(stderr,"[ %u ] = {", count);
  
    for (uint32_t ii = 2; ii < count; ii += 2)
      {
        if ((ii % 2) == 0)
          fprintf(stderr,"\n");
  
        union spn_tts const tts = { .u32 = mapped->bp_debug[ii + 1] };
  
        fprintf(stderr,"%07X : %08X : < %4u | %3d | %4u | %3d > ",
               mapped->bp_debug[ii + 0],
               tts.u32,
               tts.tx,
               tts.dx,
               tts.ty,
               tts.dy);
      }
  
    fprintf(stderr,"\n}\n");
  #endif
  
    //
    // TTRK
    //
  #if 0
    fprintf(stderr,"[ %u ] = {", count);
  
    for (uint32_t ii = 0; ii < count; ii += 2)
      {
        if ((ii % 2) == 0)
          fprintf(stderr,"\n");
  
        union spn_ttrk const ttrk = { .u32v2 = { .x = mapped->bp_debug[ii + 0],
                                                 .y = mapped->bp_debug[ii + 1] } };
  
        fprintf(stderr,"%08X%08X : < %08X : %4u : %4u : %4u >\n",
               ttrk.u32v2.y,
               ttrk.u32v2.x,
               ttrk.ttsb_id,
               (uint32_t)ttrk.y,
               ttrk.x,
               ttrk.cohort);
      }
  
    fprintf(stderr,"\n}\n");
  #endif
  }
  
  #endif
  
  //
  //
  //
  
  void
  spn_device_block_pool_create(struct spn_device * const device,
                               uint64_t const            block_pool_size,  // in bytes
                               uint32_t const            handle_count)
  {
    struct spn_block_pool * const block_pool =
      spn_allocator_host_perm_alloc(&device->allocator.host.perm,
                                    SPN_MEM_FLAGS_READ_WRITE,
                                    sizeof(*block_pool));
  
    device->block_pool = block_pool;
  
    struct spn_vk * const                     instance = device->instance;
    struct spn_vk_target_config const * const config   = spn_vk_get_config(instance);
  
    // how large is this target's block?
    uint32_t const block_dwords_log2 = config->block_pool.block_dwords_log2;
    uint32_t const block_dwords      = 1 << block_dwords_log2;
  
    // block pool sizing
    uint64_t const block_pool_size_pad = block_pool_size + sizeof(uint32_t) - 1;
    uint32_t const block_pool_dwords   = (uint32_t)(block_pool_size_pad / sizeof(uint32_t));
    uint32_t const block_count         = (block_pool_dwords + block_dwords - 1) >> block_dwords_log2;
    uint32_t const id_count            = spn_pow2_ru_u32(block_count);
  
    block_pool->bp_size = block_count;
    block_pool->bp_mask = id_count - 1;  // ids ring is power-of-two
  
    // get a descriptor set -- there is only one per Spinel device!
    spn_vk_ds_acquire_block_pool(instance, device, &block_pool->ds_block_pool);
  
    // get descriptor set DBIs
    block_pool->bp_ids.dbi = spn_vk_ds_get_block_pool_bp_ids(instance, block_pool->ds_block_pool);
  
    block_pool->bp_blocks.dbi =
      spn_vk_ds_get_block_pool_bp_blocks(instance, block_pool->ds_block_pool);
  
    block_pool->bp_host_map.dbi =
      spn_vk_ds_get_block_pool_bp_host_map(instance, block_pool->ds_block_pool);
  
  #ifdef SPN_BP_DEBUG
    block_pool->bp_debug.d.dbi =
      spn_vk_ds_get_block_pool_bp_debug(instance, block_pool->ds_block_pool);
  
    size_t const bp_debug_size =
      SPN_VK_BUFFER_OFFSETOF(block_pool, bp_debug, bp_debug) + SPN_BP_DEBUG_SIZE;
  
    spn_allocator_device_perm_alloc(&device->allocator.device.perm.drw,
                                    &device->environment,
                                    bp_debug_size,
                                    NULL,
                                    block_pool->bp_debug.d.dbi,
                                    &block_pool->bp_debug.d.dm);
  
    spn_allocator_device_perm_alloc(&device->allocator.device.perm.hr_dw,
                                    &device->environment,
                                    bp_debug_size,
                                    NULL,
                                    &block_pool->bp_debug.h.dbi,
                                    &block_pool->bp_debug.h.dm);
  
    vk(MapMemory(device->environment.d,
                 block_pool->bp_debug.h.dm,
                 0,
                 VK_WHOLE_SIZE,
                 0,
                 (void **)&block_pool->bp_debug.h.mapped));
  #endif
  
    // allocate buffers
    size_t const bp_ids_size =
      SPN_VK_BUFFER_OFFSETOF(block_pool, bp_ids, bp_ids) + id_count * sizeof(spn_block_id_t);
  
    spn_allocator_device_perm_alloc(&device->allocator.device.perm.drw,
                                    &device->environment,
                                    bp_ids_size,
                                    NULL,
                                    block_pool->bp_ids.dbi,
                                    &block_pool->bp_ids.dm);
  
    uint32_t const bp_dwords = block_count * block_dwords;
    size_t const   bp_blocks_size =
      SPN_VK_BUFFER_OFFSETOF(block_pool, bp_blocks, bp_blocks) + bp_dwords * sizeof(uint32_t);
  
    spn_allocator_device_perm_alloc(&device->allocator.device.perm.drw,
                                    &device->environment,
                                    bp_blocks_size,
                                    NULL,
                                    block_pool->bp_blocks.dbi,
                                    &block_pool->bp_blocks.dm);
  
    size_t const bp_host_map_size = SPN_VK_BUFFER_OFFSETOF(block_pool, bp_host_map, bp_host_map) +
                                    handle_count * sizeof(spn_handle_t);
  
    spn_allocator_device_perm_alloc(&device->allocator.device.perm.drw,
                                    &device->environment,
                                    bp_host_map_size,
                                    NULL,
                                    block_pool->bp_host_map.dbi,
                                    &block_pool->bp_host_map.dm);
  
    // update the block pool ds
    spn_vk_ds_update_block_pool(instance, &device->environment, block_pool->ds_block_pool);
  
    //
    // initialize the block pool
    //
    spn_dispatch_id_t id;
  
    spn(device_dispatch_acquire(device, SPN_DISPATCH_STAGE_BLOCK_POOL, &id));
  
    VkCommandBuffer cb = spn_device_dispatch_get_cb(device, id);
  
  #ifdef SPN_BP_DEBUG
    vkCmdFillBuffer(cb, block_pool->bp_debug.d.dbi->buffer, 0, sizeof(uint32_t), 0);
  
    vk_barrier_transfer_w_to_compute_r(cb);
  #endif
  
    // bind the global block pool
    spn_vk_ds_bind_block_pool_init_block_pool(instance, cb, block_pool->ds_block_pool);
  
    // append push constants
    struct spn_vk_push_block_pool_init const push = { .bp_size = block_pool->bp_size };
  
    spn_vk_p_push_block_pool_init(instance, cb, &push);
  
    // bind pipeline
    spn_vk_p_bind_block_pool_init(instance, cb);
  
    // size the grid
    uint32_t const wg_ids =
      config->p.group_sizes.named.block_pool_init.workgroup * config->block_pool.ids_per_invocation;
  
    uint32_t const wgs = (block_pool->bp_size + wg_ids - 1) / wg_ids;
  
    // dispatch the pipeline
    vkCmdDispatch(cb, wgs, 1, 1);
  
    spn_device_dispatch_submit(device, id);
  
    //
    // FIXME(allanmac): we could continue intializing and drain the device
    // as late as possible.
    //
    spn(device_wait_all(device, true, __func__));
  }
  
  //
  //
  //
  
  void
  spn_device_block_pool_dispose(struct spn_device * const device)
  {
    struct spn_vk * const         instance   = device->instance;
    struct spn_block_pool * const block_pool = device->block_pool;
  
    spn_vk_ds_release_block_pool(instance, block_pool->ds_block_pool);
  
  #ifdef SPN_BP_DEBUG
    spn_allocator_device_perm_free(&device->allocator.device.perm.hr_dw,
                                   &device->environment,
                                   &block_pool->bp_debug.h.dbi,
                                   block_pool->bp_debug.h.dm);
  
    spn_allocator_device_perm_free(&device->allocator.device.perm.drw,
                                   &device->environment,
                                   block_pool->bp_debug.d.dbi,
                                   block_pool->bp_debug.d.dm);
  #endif
  
    spn_allocator_device_perm_free(&device->allocator.device.perm.drw,
                                   &device->environment,
                                   block_pool->bp_host_map.dbi,
                                   block_pool->bp_host_map.dm);
  
    spn_allocator_device_perm_free(&device->allocator.device.perm.drw,
                                   &device->environment,
                                   block_pool->bp_blocks.dbi,
                                   block_pool->bp_blocks.dm);
  
    spn_allocator_device_perm_free(&device->allocator.device.perm.drw,
                                   &device->environment,
                                   block_pool->bp_ids.dbi,
                                   block_pool->bp_ids.dm);
  
    spn_allocator_host_perm_free(&device->allocator.host.perm, device->block_pool);
  }
  
  //
  //
  //
  
  uint32_t
  spn_device_block_pool_get_mask(struct spn_device * const device)
  {
    return device->block_pool->bp_mask;
  }
  
  //
  //
  //
  
  struct spn_vk_ds_block_pool_t
  spn_device_block_pool_get_ds(struct spn_device * const device)
  {
    return device->block_pool->ds_block_pool;
  }
  
  //
  //
  //
  
  uint32_t
  spn_device_block_pool_get_size(struct spn_device * const device)
  {
    return device->block_pool->bp_size;
  }
  
  //
  //
  //
  