text: // Copyright 2015 The Gemmlowp Authors. All Rights Reserved.
  //
  // Licensed under the Apache License, Version 2.0 (the "License");
  // you may not use this file except in compliance with the License.
  // You may obtain a copy of the License at
  //
  //     http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing, software
  // distributed under the License is distributed on an "AS IS" BASIS,
  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  // See the License for the specific language governing permissions and
  // limitations under the License.
  
  // unpack.h: unpacking the result blocks computed by compute.h,
  // storing them into the destination matrix.
  
  #ifndef GEMMLOWP_INTERNAL_UNPACK_H_
  #define GEMMLOWP_INTERNAL_UNPACK_H_
  
  #include "allocator.h"
  #include "block_params.h"
  #include "output.h"
  #include "pack.h"
  
  #include <cmath>
  
  namespace gemmlowp {
  
  class PackedResult {
   public:
    PackedResult(Allocator* _allocator, const BlockParams& _block_params)
        : allocator_(_allocator), block_params_(_block_params) {
      matrix_handle_ = allocator_->Reserve<std::int32_t>(block_params_.l2_rows *
                                                         block_params_.l2_cols);
    }
  
    ~PackedResult() {}
  
    MatrixMap<std::int32_t, MapOrder::ColMajor> Map() {
      return MatrixMap<std::int32_t, MapOrder::ColMajor>(
          allocator_->GetPointer<std::int32_t>(matrix_handle_),
          block_params_.l2_rows, block_params_.l2_cols, block_params_.l2_rows);
    }
  
    MatrixMap<const std::int32_t, MapOrder::ColMajor> Map() const {
      return MatrixMap<const std::int32_t, MapOrder::ColMajor>(
          allocator_->GetPointer<const std::int32_t>(matrix_handle_),
          block_params_.l2_rows, block_params_.l2_cols, block_params_.l2_rows);
    }
  
   private:
    Allocator* allocator_;
    Allocator::Handle matrix_handle_;
    const BlockParams& block_params_;
  };
  
  struct MatrixBlockBounds {
    int start_row;
    int start_col;
    int rows;
    int cols;
  
    MatrixBlockBounds(int start_row_, int start_col_, int rows_, int cols_)
        : start_row(start_row_),
          start_col(start_col_),
          rows(rows_),
          cols(cols_) {}
  };
  
  template <int Rows, int Cols, typename SrcMapType>
  void PrefetchResultBlock(const SrcMapType& src,
                           const VectorMap<const std::int32_t, VectorShape::Col>&
                               lhs_sums_of_each_slice,
                           int src_row, int src_col) {
    const std::int32_t* src_data = src.data(src_row, src_col);
    const int src_stride = src.stride();
    const std::int32_t* lhs_sums_data = lhs_sums_of_each_slice.data(src_row);
    for (int r = 0; r < Rows; r += 4) {
      Prefetch(lhs_sums_data + r);
    }
    for (int c = 0; c < Cols; c++) {
      for (int r = 0; r < Rows; r += 4) {
        Prefetch(src_data + r + c * src_stride);
      }
    }
  }
  
  template <typename KernelFormat, typename RegisterBlockType,
            typename SrcMapType, typename LhsOffset, typename RhsOffset,
            typename OutputPipelineExecutorType, typename DstType>
  void UnpackResultBlock(const SrcMapType& src,
                         const OutputPipelineExecutorType& executor, DstType* dst,
                         const VectorMap<const std::int32_t, VectorShape::Col>&
                             lhs_sums_of_each_slice,
                         const VectorMap<const std::int32_t, VectorShape::Row>&
                             rhs_sums_of_each_slice,
                         const LhsOffset& lhs_offset, const RhsOffset& rhs_offset,
                         int depth, int src_row, int src_col, int src_global_row,
                         int src_global_col, int dst_row, int dst_col) {
    using KernelLhsScalar = typename KernelFormat::Lhs::Scalar;
    using KernelRhsScalar = typename KernelFormat::Rhs::Scalar;
    static constexpr int KernelLhsZeroPointInput =
        ZeroPointInputValue<KernelLhsScalar>::kValue;
    static constexpr int KernelRhsZeroPointInput =
        ZeroPointInputValue<KernelRhsScalar>::kValue;
    auto acc = Load<RegisterBlockType>(src, src_row, src_col);
    const auto& lhs_sums_of_each_slice_block =
        LoadForBroadcasting<RegisterBlockType>(lhs_sums_of_each_slice, src_row);
    const auto& rhs_sums_of_each_slice_block =
        LoadForBroadcasting<RegisterBlockType>(rhs_sums_of_each_slice, src_col);
    auto lhs_offset_block =
        LoadForBroadcasting<RegisterBlockType>(lhs_offset, src_row);
    auto rhs_offset_block =
        LoadForBroadcasting<RegisterBlockType>(rhs_offset, src_col);
    AddConstant<KernelLhsZeroPointInput>(&lhs_offset_block);
    AddConstant<KernelRhsZeroPointInput>(&rhs_offset_block);
    BroadcastMulAdd(lhs_sums_of_each_slice_block, rhs_offset_block, &acc);
    for (int i = 0; i < decltype(rhs_offset_block)::kRegisterCount; i++) {
      rhs_offset_block.buf.reg[i] = Mul(rhs_offset_block.buf.reg[i], depth);
    }
    BroadcastMulAdd(BroadcastAdd(rhs_sums_of_each_slice_block, rhs_offset_block),
                    lhs_offset_block, &acc);
    executor.Execute(acc, dst, src_global_row, src_global_col, dst_row, dst_col);
  }
  
  template <typename KernelFormat, typename ResultBlockType,
            typename PackedResultType, typename LhsOffset, typename RhsOffset,
            typename OutputPipelineType>
  void UnpackResult(ResultBlockType* dst, const MatrixBlockBounds& dst_block,
                    const PackedResultType& src, int depth,
                    const std::int32_t* lhs_sums_of_each_slice_ptr,
                    const std::int32_t* rhs_sums_of_each_slice_ptr,
                    const LhsOffset& lhs_offset, const RhsOffset& rhs_offset,
                    const OutputPipelineType& output_pipeline) {
    ScopedProfilingLabel label(ResultBlockType::kOrder == MapOrder::ColMajor
                                   ? "unpack to column-major"
                                   : "unpack to row-major");
    assert(dst_block.start_row >= 0);
    assert(dst_block.start_row + dst_block.rows <= dst->rows());
    assert(dst_block.start_col >= 0);
    assert(dst_block.start_col + dst_block.cols <= dst->cols());
    const auto src_map = src.Map();
    const VectorMap<const std::int32_t, VectorShape::Col> lhs_sums_of_each_slice(
        lhs_sums_of_each_slice_ptr, dst_block.rows);
    const VectorMap<const std::int32_t, VectorShape::Row> rhs_sums_of_each_slice(
        rhs_sums_of_each_slice_ptr, dst_block.cols);
    using Int32x1x1 = RegisterBlock<std::int32_t, 1, 1>;
    using Int32x4x1 = RegisterBlock<std::int32_t, 4, 1>;
    using Int32x8x1 = RegisterBlock<std::int32_t, 8, 1>;
    using Int32x1x4 = RegisterBlock<std::int32_t, 1, 4>;
    using Int32x4x4 = RegisterBlock<std::int32_t, 4, 4>;
    using Int32x8x4 = RegisterBlock<std::int32_t, 8, 4>;
  
    using DstScalarType = typename ResultBlockType::Scalar;
    using DstScalarx8x8 = RegisterBlock<DstScalarType, 8, 8>;
  
    OutputPipelineExecutor<OutputPipelineType, Int32x1x1>
        output_pipeline_executor_1x1(output_pipeline);
    OutputPipelineExecutor<OutputPipelineType, Int32x4x1>
        output_pipeline_executor_4x1(output_pipeline);
    OutputPipelineExecutor<OutputPipelineType, Int32x8x1>
        output_pipeline_executor_8x1(output_pipeline);
    OutputPipelineExecutor<OutputPipelineType, Int32x1x4>
        output_pipeline_executor_1x4(output_pipeline);
    OutputPipelineExecutor<OutputPipelineType, Int32x4x4>
        output_pipeline_executor_4x4(output_pipeline);
    OutputPipelineExecutor<OutputPipelineType, Int32x8x4>
        output_pipeline_executor_8x4(output_pipeline);
  
    int c8 = 0;
    if (ResultBlockType::kOrder == MapOrder::RowMajor) {
      for (; c8 <= dst_block.cols - 8; c8 += 8) {
        PrefetchResultBlock<8, 8>(src_map, lhs_sums_of_each_slice, 0, c8);
        int r = 0;
        for (; r <= dst_block.rows - 8; r += 8) {
          const int global_row = r + dst_block.start_row;
          PrefetchResultBlock<8, 8>(src_map, lhs_sums_of_each_slice, r + 8, c8);
          DstScalarType dst_colmajor_buf[64];
          MatrixMap<DstScalarType, MapOrder::ColMajor> dst_colmajor_map(
              dst_colmajor_buf, 8, 8);
          for (int cx = 0; cx < 8; cx += 4) {
            const int c = c8 + cx;
            const int global_col = c + dst_block.start_col;
            UnpackResultBlock<KernelFormat, Int32x8x4>(
                src_map, output_pipeline_executor_8x4, &dst_colmajor_map,
                lhs_sums_of_each_slice, rhs_sums_of_each_slice, lhs_offset,
                rhs_offset, depth, r, c, global_row, global_col, 0, cx);
          }
          StoreFinalOutput(LoadContiguous<DstScalarx8x8>(dst_colmajor_buf), dst,
                           r + dst_block.start_row, c8 + dst_block.start_col);
        }
        for (; r <= dst_block.rows - 4; r += 4) {
          const int global_row = r + dst_block.start_row;
          for (int cx = 0; cx < 8; cx += 4) {
            const int c = c8 + cx;
            const int global_col = c + dst_block.start_col;
            UnpackResultBlock<KernelFormat, Int32x4x4>(
                src_map, output_pipeline_executor_4x4, dst,
                lhs_sums_of_each_slice, rhs_sums_of_each_slice, lhs_offset,
                rhs_offset, depth, r, c, global_row, global_col, global_row,
                global_col);
          }
        }
        for (; r < dst_block.rows; r++) {
          const int global_row = r + dst_block.start_row;
          for (int cx = 0; cx < 8; cx += 4) {
            const int c = c8 + cx;
            const int global_col = c + dst_block.start_col;
            UnpackResultBlock<KernelFormat, Int32x1x4>(
                src_map, output_pipeline_executor_1x4, dst,
                lhs_sums_of_each_slice, rhs_sums_of_each_slice, lhs_offset,
                rhs_offset, depth, r, c, global_row, global_col, global_row,
                global_col);
          }
        }
      }
    }
    int c = c8;
    for (; c <= dst_block.cols - 4; c += 4) {
      const int global_col = c + dst_block.start_col;
      PrefetchResultBlock<8, 4>(src_map, lhs_sums_of_each_slice, 0, c);
      int r = 0;
      for (; r <= dst_block.rows - 8; r += 8) {
        const int global_row = r + dst_block.start_row;
        PrefetchResultBlock<8, 4>(src_map, lhs_sums_of_each_slice, r + 8, c);
        UnpackResultBlock<KernelFormat, Int32x8x4>(
            src_map, output_pipeline_executor_8x4, dst, lhs_sums_of_each_slice,
            rhs_sums_of_each_slice, lhs_offset, rhs_offset, depth, r, c,
            global_row, global_col, global_row, global_col);
      }
      for (; r <= dst_block.rows - 4; r += 4) {
        const int global_row = r + dst_block.start_row;
        UnpackResultBlock<KernelFormat, Int32x4x4>(
            src_map, output_pipeline_executor_4x4, dst, lhs_sums_of_each_slice,
            rhs_sums_of_each_slice, lhs_offset, rhs_offset, depth, r, c,
            global_row, global_col, global_row, global_col);
      }
      for (; r < dst_block.rows; r++) {
        const int global_row = r + dst_block.start_row;
        UnpackResultBlock<KernelFormat, Int32x1x4>(
            src_map, output_pipeline_executor_1x4, dst, lhs_sums_of_each_slice,
            rhs_sums_of_each_slice, lhs_offset, rhs_offset, depth, r, c,
            global_row, global_col, global_row, global_col);
      }
    }
    for (; c < dst_block.cols; c++) {
      const int global_col = c + dst_block.start_col;
      PrefetchResultBlock<8, 1>(src_map, lhs_sums_of_each_slice, 0, c);
      int r = 0;
      for (; r <= dst_block.rows - 8; r += 8) {
        const int global_row = r + dst_block.start_row;
        PrefetchResultBlock<8, 1>(src_map, lhs_sums_of_each_slice, r + 8, c);
        UnpackResultBlock<KernelFormat, Int32x8x1>(
            src_map, output_pipeline_executor_8x1, dst, lhs_sums_of_each_slice,
            rhs_sums_of_each_slice, lhs_offset, rhs_offset, depth, r, c,
            global_row, global_col, global_row, global_col);
      }
      for (; r <= dst_block.rows - 4; r += 4) {
        const int global_row = r + dst_block.start_row;
        UnpackResultBlock<KernelFormat, Int32x4x1>(
            src_map, output_pipeline_executor_4x1, dst, lhs_sums_of_each_slice,
            rhs_sums_of_each_slice, lhs_offset, rhs_offset, depth, r, c,
            global_row, global_col, global_row, global_col);
      }
      for (; r < dst_block.rows; r++) {
        const int global_row = r + dst_block.start_row;
        UnpackResultBlock<KernelFormat, Int32x1x1>(
            src_map, output_pipeline_executor_1x1, dst, lhs_sums_of_each_slice,
            rhs_sums_of_each_slice, lhs_offset, rhs_offset, depth, r, c,
            global_row, global_col, global_row, global_col);
      }
    }
  }
  
  }  // end namespace gemmlowp
  
  #endif  // GEMMLOWP_INTERNAL_UNPACK_H_
  