CarbonDictionaryWriterImpl implements CarbonDictionaryWriter { private void writeDictionaryFile() throws IOException { ColumnDictionaryChunk columnDictionaryChunk = new ColumnDictionaryChunk(); columnDictionaryChunk.setValues(oneDictionaryChunkList); writeThriftObject(columnDictionaryChunk); } CarbonDictionaryWriterImpl(String hdfsStorePath,
      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier); @Override void write(String value); @Override void write(byte[] value); @Override void write(List<byte[]> valueList); @Override void close(); @Override void commit();  }
@Test public void testWriteMultipleChunksForOneSegment() throws IOException { deleteStorePath(); CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE, "1"); CarbonDictionaryWriterImpl writer = prepareWriter(); writeDictionaryFile(writer, dataSet1); long end_offset = CarbonUtil.getFileSize(this.dictionaryFilePath); List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile(); assertTrue(1 == carbonDictionaryColumnMetaChunks.size()); long start_offset = 0L; CarbonDictionaryColumnMetaChunk expected = new CarbonDictionaryColumnMetaChunk(1, dataSet1.size(), start_offset, end_offset, dataSet1.size()); for (CarbonDictionaryColumnMetaChunk chunk : carbonDictionaryColumnMetaChunks) { validateDictionaryMetadata(chunk, expected); } List<byte[]> dictionaryValues = readDictionaryFile(0L, 0L); List<String> actual = convertByteArrayListToStringValueList(dictionaryValues); assertTrue(dataSet1.size() == actual.size()); compareDictionaryData(actual, dataSet1); CarbonProperties.getInstance().addProperty(CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE, CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE_DEFAULT); }
@Test public void testTruncateOperation() throws IOException { deleteStorePath(); CarbonDictionaryWriterImpl writer = prepareWriter(); writeDictionaryFile(writer, dataSet1); long endOffsetAfterFirstDictionaryChunk = CarbonUtil.getFileSize(dictionaryFilePath); writer = prepareWriter(); writeDictionaryFile(writer, dataSet2); ColumnDictionaryChunkMeta firstDictionaryChunkMeta = new ColumnDictionaryChunkMeta(1, 2, 0, endOffsetAfterFirstDictionaryChunk, 1); overwriteDictionaryMetaFile(firstDictionaryChunkMeta, dictionaryMetaFilePath); writer = prepareWriter(); writeDictionaryFile(writer, dataSet3); List<byte[]> dictionaryValues = readDictionaryFile(0L, 0L); List<String> actual = convertByteArrayListToStringValueList(dictionaryValues); List<String> expected = new ArrayList<>(4); expected.addAll(dataSet1); expected.addAll(dataSet3); compareDictionaryData(actual, expected); }
@Test public void testReadingOfDictionaryChunkFromAnOffset() throws Exception { deleteStorePath(); CarbonDictionaryWriterImpl writer = prepareWriter(); writeDictionaryFile(writer, dataSet1); writer = prepareWriter(); writeDictionaryFile(writer, dataSet2); long dictionaryFileOffsetToRead = CarbonUtil.getFileSize(this.dictionaryFilePath); writer = prepareWriter(); writeDictionaryFile(writer, dataSet3); List<byte[]> dictionaryData = readDictionaryFile(dictionaryFileOffsetToRead, 0L); List<String> actual = convertByteArrayListToStringValueList(dictionaryData); compareDictionaryData(actual, dataSet3); List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile(); assertTrue(3 == carbonDictionaryColumnMetaChunks.size()); }
@Test public void testReadingOfDictionaryChunkBetweenStartAndEndOffset() throws Exception { deleteStorePath(); CarbonDictionaryWriterImpl writer = prepareWriter(); writeDictionaryFile(writer, dataSet1); long dictionaryStartOffset = CarbonUtil.getFileSize(this.dictionaryFilePath); writer = prepareWriter(); writeDictionaryFile(writer, dataSet2); long dictionaryFileEndOffset = CarbonUtil.getFileSize(this.dictionaryFilePath); writer = prepareWriter(); writeDictionaryFile(writer, dataSet3); List<byte[]> dictionaryData = readDictionaryFile(dictionaryStartOffset, dictionaryFileEndOffset); List<String> actual = convertByteArrayListToStringValueList(dictionaryData); compareDictionaryData(actual, dataSet2); List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile(); assertTrue(3 == carbonDictionaryColumnMetaChunks.size()); CarbonDictionaryColumnMetaChunk expected = new CarbonDictionaryColumnMetaChunk(3, 4, dictionaryStartOffset, dictionaryFileEndOffset, 1); validateDictionaryMetadata(carbonDictionaryColumnMetaChunks.get(1), expected); }