hunk.code_window: #include "scalars.h"
   using namespace arrow;
   extern PyObject* numbuf_serialize_callback;
   extern PyObject* numbuf_deserialize_callback;
   namespace numbuf {
  
hunk.labels: keep keep keep keep add keep keep keep keep
hunk.after_edit: int32_t MAX_RECURSION_DEPTH = 100;
  
hunk.file_path: python/src/pynumbuf/adapters/python.cc
hunk.type: add
file: #include "sequence.h"
  
  using namespace arrow;
  
  namespace numbuf {
  
  SequenceBuilder::SequenceBuilder(MemoryPool* pool)
      : pool_(pool), types_(pool), offsets_(pool),
        nones_(pool, std::make_shared<NullType>()),
        bools_(pool, std::make_shared<BooleanType>()),
        ints_(pool),
        bytes_(pool, std::make_shared<BinaryType>()),
        strings_(pool, std::make_shared<StringType>()),
        floats_(pool), doubles_(pool),
        uint8_tensors_(std::make_shared<UInt8Type>(), pool),
        int8_tensors_(std::make_shared<Int8Type>(), pool),
        uint16_tensors_(std::make_shared<UInt16Type>(), pool),
        int16_tensors_(std::make_shared<Int16Type>(), pool),
        uint32_tensors_(std::make_shared<UInt32Type>(), pool),
        int32_tensors_(std::make_shared<Int32Type>(), pool),
        uint64_tensors_(std::make_shared<UInt64Type>(), pool),
        int64_tensors_(std::make_shared<Int64Type>(), pool),
        float_tensors_(std::make_shared<FloatType>(), pool),
        double_tensors_(std::make_shared<DoubleType>(), pool),
        list_offsets_({0}), tuple_offsets_({0}), dict_offsets_({0}) {}
  
  #define UPDATE(OFFSET, TAG)                    \
    if (TAG == -1) {                             \
      TAG = num_tags;                            \
      num_tags += 1;                             \
    }                                            \
    RETURN_NOT_OK(offsets_.Append(OFFSET));      \
    RETURN_NOT_OK(types_.Append(TAG));           \
    RETURN_NOT_OK(nones_.AppendToBitmap(true));
  
  Status SequenceBuilder::AppendNone() {
    RETURN_NOT_OK(offsets_.Append(0));
    RETURN_NOT_OK(types_.Append(0));
    return nones_.AppendToBitmap(false);
  }
  
  Status SequenceBuilder::AppendBool(bool data) {
    UPDATE(bools_.length(), bool_tag);
    return bools_.Append(data);
  }
  
  Status SequenceBuilder::AppendInt64(int64_t data) {
    UPDATE(ints_.length(), int_tag);
    return ints_.Append(data);
  }
  
  Status SequenceBuilder::AppendUInt64(uint64_t data) {
    UPDATE(ints_.length(), int_tag);
    return ints_.Append(data);
  }
  
  Status SequenceBuilder::AppendBytes(const uint8_t* data, int32_t length) {
    UPDATE(bytes_.length(), bytes_tag);
    return bytes_.Append(data, length);
  }
  
  Status SequenceBuilder::AppendString(const char* data, int32_t length) {
    UPDATE(strings_.length(), string_tag);
    return strings_.Append(data, length);
  }
  
  Status SequenceBuilder::AppendFloat(float data) {
    UPDATE(floats_.length(), float_tag);
    return floats_.Append(data);
  }
  
  Status SequenceBuilder::AppendDouble(double data) {
    UPDATE(doubles_.length(), double_tag);
    return doubles_.Append(data);
  }
  
  #define DEF_TENSOR_APPEND(NAME, TYPE, TAG)                                             \
    Status SequenceBuilder::AppendTensor(const std::vector<int64_t>& dims, TYPE* data) { \
      UPDATE(NAME.length(), TAG);                                                        \
      return NAME.Append(dims, data);                                                    \
    }
  
  DEF_TENSOR_APPEND(uint8_tensors_, uint8_t, uint8_tensor_tag);
  DEF_TENSOR_APPEND(int8_tensors_, int8_t, int8_tensor_tag);
  DEF_TENSOR_APPEND(uint16_tensors_, uint16_t, uint16_tensor_tag);
  DEF_TENSOR_APPEND(int16_tensors_, int16_t, int16_tensor_tag);
  DEF_TENSOR_APPEND(uint32_tensors_, uint32_t, uint32_tensor_tag);
  DEF_TENSOR_APPEND(int32_tensors_, int32_t, int32_tensor_tag);
  DEF_TENSOR_APPEND(uint64_tensors_, uint64_t, uint64_tensor_tag);
  DEF_TENSOR_APPEND(int64_tensors_, int64_t, int64_tensor_tag);
  DEF_TENSOR_APPEND(float_tensors_, float, float_tensor_tag);
  DEF_TENSOR_APPEND(double_tensors_, double, double_tensor_tag);
  
  Status SequenceBuilder::AppendList(int32_t size) {
    UPDATE(list_offsets_.size() - 1, list_tag);
    list_offsets_.push_back(list_offsets_.back() + size);
    return Status::OK();
  }
  
  Status SequenceBuilder::AppendTuple(int32_t size) {
    UPDATE(tuple_offsets_.size() - 1, tuple_tag);
    tuple_offsets_.push_back(tuple_offsets_.back() + size);
    return Status::OK();
  }
  
  Status SequenceBuilder::AppendDict(int32_t size) {
    UPDATE(dict_offsets_.size() - 1, dict_tag);
    dict_offsets_.push_back(dict_offsets_.back() + size);
    return Status::OK();
  }
  
  #define ADD_ELEMENT(VARNAME, TAG)                 \
    if (TAG != -1) {                                \
      types[TAG] = VARNAME.type();                  \
      children[TAG] = VARNAME.Finish();             \
      ARROW_CHECK_OK(nones_.AppendToBitmap(true));  \
    }
  
  #define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME)                    \
    if (DATA) {                                                                 \
      DCHECK(DATA->length() == OFFSETS.back());                                 \
      auto list_builder = std::make_shared<ListBuilder>(pool_, DATA);           \
      auto field = std::make_shared<Field>(NAME, list_builder->type());         \
      auto type = std::make_shared<StructType>(std::vector<FieldPtr>({field})); \
      auto lists = std::vector<std::shared_ptr<ArrayBuilder>>({list_builder});  \
      StructBuilder builder(pool_, type, lists);                                \
      OFFSETS.pop_back();                                                       \
      ARROW_CHECK_OK(list_builder->Append(OFFSETS.data(), OFFSETS.size()));     \
      builder.Append();                                                         \
      ADD_ELEMENT(builder, TAG);                                                \
    } else {                                                                    \
      DCHECK(OFFSETS.size() == 1);                                              \
    }
  
  std::shared_ptr<DenseUnionArray> SequenceBuilder::Finish(
    std::shared_ptr<Array> list_data,
    std::shared_ptr<Array> tuple_data,
    std::shared_ptr<Array> dict_data) {
  
    std::vector<TypePtr> types(num_tags);
    std::vector<ArrayPtr> children(num_tags);
  
    ADD_ELEMENT(bools_, bool_tag);
    ADD_ELEMENT(ints_, int_tag);
    ADD_ELEMENT(strings_, string_tag);
    ADD_ELEMENT(bytes_, bytes_tag);
    ADD_ELEMENT(floats_, float_tag);
    ADD_ELEMENT(doubles_, double_tag);
  
    ADD_ELEMENT(uint8_tensors_, uint8_tensor_tag);
  
    ADD_ELEMENT(int8_tensors_, int8_tensor_tag);
    ADD_ELEMENT(uint16_tensors_, uint16_tensor_tag);
    ADD_ELEMENT(int16_tensors_, int16_tensor_tag);
    ADD_ELEMENT(uint32_tensors_, uint32_tensor_tag);
  
    ADD_ELEMENT(int32_tensors_, int32_tensor_tag);
    ADD_ELEMENT(uint64_tensors_, uint64_tensor_tag);
    ADD_ELEMENT(int64_tensors_, int64_tensor_tag);
  
    ADD_ELEMENT(float_tensors_, float_tensor_tag);
    ADD_ELEMENT(double_tensors_, double_tensor_tag);
  
    ADD_SUBSEQUENCE(list_data, list_offsets_, list_builder, list_tag, "list");
    ADD_SUBSEQUENCE(tuple_data, tuple_offsets_, tuple_builder, tuple_tag, "tuple");
    ADD_SUBSEQUENCE(dict_data, dict_offsets_, dict_builder, dict_tag, "dict");
  
    TypePtr type = TypePtr(new DenseUnionType(types));
  
    return std::make_shared<DenseUnionArray>(type, types_.length(),
             children, types_.data(), offsets_.data(),
             nones_.null_count(), nones_.null_bitmap());
  }
  
  }
  