@Test public void testColumnMinMaxValue() throws Exception { constructV1Segment(); PropertiesConfiguration configuration = SegmentMetadataImpl.getPropertiesConfiguration(_indexDir); Iterator<String> keys = configuration.getKeys(); while (keys.hasNext()) { String key = keys.next(); if (key.endsWith(V1Constants.MetadataKeys.Column.MIN_VALUE) || key .endsWith(V1Constants.MetadataKeys.Column.MAX_VALUE)) { configuration.clearProperty(key); } } configuration.save(); IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(); indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NONE); try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) { processor.process(); } SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_indexDir); ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch"); ColumnMetadata dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column1"); ColumnMetadata metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count"); Assert.assertNull(timeColumnMetadata.getMinValue()); Assert.assertNull(timeColumnMetadata.getMaxValue()); Assert.assertNull(dimensionColumnMetadata.getMinValue()); Assert.assertNull(dimensionColumnMetadata.getMaxValue()); Assert.assertNull(metricColumnMetadata.getMinValue()); Assert.assertNull(metricColumnMetadata.getMaxValue()); indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.TIME); try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) { processor.process(); } segmentMetadata = new SegmentMetadataImpl(_indexDir); timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch"); dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5"); metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count"); Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683); Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683); Assert.assertNull(dimensionColumnMetadata.getMinValue()); Assert.assertNull(dimensionColumnMetadata.getMaxValue()); Assert.assertNull(metricColumnMetadata.getMinValue()); Assert.assertNull(metricColumnMetadata.getMaxValue()); indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NON_METRIC); try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) { processor.process(); } segmentMetadata = new SegmentMetadataImpl(_indexDir); timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch"); dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5"); metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count"); Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683); Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683); Assert.assertEquals(dimensionColumnMetadata.getMinValue(), "AKXcXcIqsqOJFsdwxZ"); Assert.assertEquals(dimensionColumnMetadata.getMaxValue(), "yQkJTLOQoOqqhkAClgC"); Assert.assertNull(metricColumnMetadata.getMinValue()); Assert.assertNull(metricColumnMetadata.getMaxValue()); indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.ALL); try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) { processor.process(); } segmentMetadata = new SegmentMetadataImpl(_indexDir); timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch"); dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5"); metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count"); Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683); Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683); Assert.assertEquals(dimensionColumnMetadata.getMinValue(), "AKXcXcIqsqOJFsdwxZ"); Assert.assertEquals(dimensionColumnMetadata.getMaxValue(), "yQkJTLOQoOqqhkAClgC"); Assert.assertEquals(metricColumnMetadata.getMinValue(), 890662862); Assert.assertEquals(metricColumnMetadata.getMaxValue(), 890662862); }
public void process() throws Exception { if (_segmentMetadata.getTotalDocs() == 0) { return; } File[] directoryListing = _indexDir.listFiles(); String tempFileExtension = V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION + ".tmp"; if (directoryListing != null) { for (File child : directoryListing) { if (child.getName().endsWith(tempFileExtension)) { FileUtils.deleteQuietly(child); } } } try (SegmentDirectory.Writer segmentWriter = _segmentDirectory.createWriter()) { if (_schema != null) { DefaultColumnHandler defaultColumnHandler = DefaultColumnHandlerFactory.getDefaultColumnHandler(_indexDir, _schema, _segmentMetadata, segmentWriter); defaultColumnHandler.updateDefaultColumns(_indexLoadingConfig); _segmentMetadata = new SegmentMetadataImpl(_indexDir); _segmentDirectory.reloadMetadata(); } InvertedIndexHandler invertedIndexHandler = new InvertedIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); invertedIndexHandler.createInvertedIndices(); RangeIndexHandler rangeIndexHandler = new RangeIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); rangeIndexHandler.createRangeIndices(); Set<String> textIndexColumns = _indexLoadingConfig.getTextIndexColumns(); if (textIndexColumns.size() > 0) { TextIndexHandler textIndexHandler = new TextIndexHandler(_indexDir, _segmentMetadata, textIndexColumns, segmentWriter); textIndexHandler.createTextIndexesOnSegmentLoad(); } BloomFilterHandler bloomFilterHandler = new BloomFilterHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); bloomFilterHandler.createBloomFilters(); if (_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { List<StarTreeV2BuilderConfig> starTreeBuilderConfigs = StarTreeBuilderUtils .generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), _segmentMetadata); boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty(); List<StarTreeV2Metadata> starTreeMetadataList = _segmentMetadata.getStarTreeV2MetadataList(); if (starTreeMetadataList != null) { if (StarTreeUtils.shouldRemoveExistingStarTrees(starTreeBuilderConfigs, starTreeMetadataList)) { LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName()); StarTreeUtils.removeStarTrees(_indexDir); _segmentMetadata = new SegmentMetadataImpl(_indexDir); } else { shouldGenerateStarTree = false; } } if (shouldGenerateStarTree) { try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, _indexDir, MultipleTreesBuilder.BuildMode.OFF_HEAP)) { builder.build(); } _segmentMetadata = new SegmentMetadataImpl(_indexDir); } } ColumnMinMaxValueGeneratorMode columnMinMaxValueGeneratorMode = _indexLoadingConfig.getColumnMinMaxValueGeneratorMode(); if (columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE) { ColumnMinMaxValueGenerator columnMinMaxValueGenerator = new ColumnMinMaxValueGenerator(_segmentMetadata, segmentWriter, columnMinMaxValueGeneratorMode); columnMinMaxValueGenerator.addColumnMinMaxValue(); } segmentWriter.save(); } }
SegmentPreProcessor implements AutoCloseable { public void process() throws Exception { if (_segmentMetadata.getTotalDocs() == 0) { return; } File[] directoryListing = _indexDir.listFiles(); String tempFileExtension = V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION + ".tmp"; if (directoryListing != null) { for (File child : directoryListing) { if (child.getName().endsWith(tempFileExtension)) { FileUtils.deleteQuietly(child); } } } try (SegmentDirectory.Writer segmentWriter = _segmentDirectory.createWriter()) { if (_schema != null) { DefaultColumnHandler defaultColumnHandler = DefaultColumnHandlerFactory.getDefaultColumnHandler(_indexDir, _schema, _segmentMetadata, segmentWriter); defaultColumnHandler.updateDefaultColumns(_indexLoadingConfig); _segmentMetadata = new SegmentMetadataImpl(_indexDir); _segmentDirectory.reloadMetadata(); } InvertedIndexHandler invertedIndexHandler = new InvertedIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); invertedIndexHandler.createInvertedIndices(); RangeIndexHandler rangeIndexHandler = new RangeIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); rangeIndexHandler.createRangeIndices(); Set<String> textIndexColumns = _indexLoadingConfig.getTextIndexColumns(); if (textIndexColumns.size() > 0) { TextIndexHandler textIndexHandler = new TextIndexHandler(_indexDir, _segmentMetadata, textIndexColumns, segmentWriter); textIndexHandler.createTextIndexesOnSegmentLoad(); } BloomFilterHandler bloomFilterHandler = new BloomFilterHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); bloomFilterHandler.createBloomFilters(); if (_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { List<StarTreeV2BuilderConfig> starTreeBuilderConfigs = StarTreeBuilderUtils .generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), _segmentMetadata); boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty(); List<StarTreeV2Metadata> starTreeMetadataList = _segmentMetadata.getStarTreeV2MetadataList(); if (starTreeMetadataList != null) { if (StarTreeUtils.shouldRemoveExistingStarTrees(starTreeBuilderConfigs, starTreeMetadataList)) { LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName()); StarTreeUtils.removeStarTrees(_indexDir); _segmentMetadata = new SegmentMetadataImpl(_indexDir); } else { shouldGenerateStarTree = false; } } if (shouldGenerateStarTree) { try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, _indexDir, MultipleTreesBuilder.BuildMode.OFF_HEAP)) { builder.build(); } _segmentMetadata = new SegmentMetadataImpl(_indexDir); } } ColumnMinMaxValueGeneratorMode columnMinMaxValueGeneratorMode = _indexLoadingConfig.getColumnMinMaxValueGeneratorMode(); if (columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE) { ColumnMinMaxValueGenerator columnMinMaxValueGenerator = new ColumnMinMaxValueGenerator(_segmentMetadata, segmentWriter, columnMinMaxValueGeneratorMode); columnMinMaxValueGenerator.addColumnMinMaxValue(); } segmentWriter.save(); } } }
SegmentPreProcessor implements AutoCloseable { public void process() throws Exception { if (_segmentMetadata.getTotalDocs() == 0) { return; } File[] directoryListing = _indexDir.listFiles(); String tempFileExtension = V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION + ".tmp"; if (directoryListing != null) { for (File child : directoryListing) { if (child.getName().endsWith(tempFileExtension)) { FileUtils.deleteQuietly(child); } } } try (SegmentDirectory.Writer segmentWriter = _segmentDirectory.createWriter()) { if (_schema != null) { DefaultColumnHandler defaultColumnHandler = DefaultColumnHandlerFactory.getDefaultColumnHandler(_indexDir, _schema, _segmentMetadata, segmentWriter); defaultColumnHandler.updateDefaultColumns(_indexLoadingConfig); _segmentMetadata = new SegmentMetadataImpl(_indexDir); _segmentDirectory.reloadMetadata(); } InvertedIndexHandler invertedIndexHandler = new InvertedIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); invertedIndexHandler.createInvertedIndices(); RangeIndexHandler rangeIndexHandler = new RangeIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); rangeIndexHandler.createRangeIndices(); Set<String> textIndexColumns = _indexLoadingConfig.getTextIndexColumns(); if (textIndexColumns.size() > 0) { TextIndexHandler textIndexHandler = new TextIndexHandler(_indexDir, _segmentMetadata, textIndexColumns, segmentWriter); textIndexHandler.createTextIndexesOnSegmentLoad(); } BloomFilterHandler bloomFilterHandler = new BloomFilterHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); bloomFilterHandler.createBloomFilters(); if (_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { List<StarTreeV2BuilderConfig> starTreeBuilderConfigs = StarTreeBuilderUtils .generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), _segmentMetadata); boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty(); List<StarTreeV2Metadata> starTreeMetadataList = _segmentMetadata.getStarTreeV2MetadataList(); if (starTreeMetadataList != null) { if (StarTreeUtils.shouldRemoveExistingStarTrees(starTreeBuilderConfigs, starTreeMetadataList)) { LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName()); StarTreeUtils.removeStarTrees(_indexDir); _segmentMetadata = new SegmentMetadataImpl(_indexDir); } else { shouldGenerateStarTree = false; } } if (shouldGenerateStarTree) { try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, _indexDir, MultipleTreesBuilder.BuildMode.OFF_HEAP)) { builder.build(); } _segmentMetadata = new SegmentMetadataImpl(_indexDir); } } ColumnMinMaxValueGeneratorMode columnMinMaxValueGeneratorMode = _indexLoadingConfig.getColumnMinMaxValueGeneratorMode(); if (columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE) { ColumnMinMaxValueGenerator columnMinMaxValueGenerator = new ColumnMinMaxValueGenerator(_segmentMetadata, segmentWriter, columnMinMaxValueGeneratorMode); columnMinMaxValueGenerator.addColumnMinMaxValue(); } segmentWriter.save(); } } SegmentPreProcessor(File indexDir, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema); }
SegmentPreProcessor implements AutoCloseable { public void process() throws Exception { if (_segmentMetadata.getTotalDocs() == 0) { return; } File[] directoryListing = _indexDir.listFiles(); String tempFileExtension = V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION + ".tmp"; if (directoryListing != null) { for (File child : directoryListing) { if (child.getName().endsWith(tempFileExtension)) { FileUtils.deleteQuietly(child); } } } try (SegmentDirectory.Writer segmentWriter = _segmentDirectory.createWriter()) { if (_schema != null) { DefaultColumnHandler defaultColumnHandler = DefaultColumnHandlerFactory.getDefaultColumnHandler(_indexDir, _schema, _segmentMetadata, segmentWriter); defaultColumnHandler.updateDefaultColumns(_indexLoadingConfig); _segmentMetadata = new SegmentMetadataImpl(_indexDir); _segmentDirectory.reloadMetadata(); } InvertedIndexHandler invertedIndexHandler = new InvertedIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); invertedIndexHandler.createInvertedIndices(); RangeIndexHandler rangeIndexHandler = new RangeIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); rangeIndexHandler.createRangeIndices(); Set<String> textIndexColumns = _indexLoadingConfig.getTextIndexColumns(); if (textIndexColumns.size() > 0) { TextIndexHandler textIndexHandler = new TextIndexHandler(_indexDir, _segmentMetadata, textIndexColumns, segmentWriter); textIndexHandler.createTextIndexesOnSegmentLoad(); } BloomFilterHandler bloomFilterHandler = new BloomFilterHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); bloomFilterHandler.createBloomFilters(); if (_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { List<StarTreeV2BuilderConfig> starTreeBuilderConfigs = StarTreeBuilderUtils .generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), _segmentMetadata); boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty(); List<StarTreeV2Metadata> starTreeMetadataList = _segmentMetadata.getStarTreeV2MetadataList(); if (starTreeMetadataList != null) { if (StarTreeUtils.shouldRemoveExistingStarTrees(starTreeBuilderConfigs, starTreeMetadataList)) { LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName()); StarTreeUtils.removeStarTrees(_indexDir); _segmentMetadata = new SegmentMetadataImpl(_indexDir); } else { shouldGenerateStarTree = false; } } if (shouldGenerateStarTree) { try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, _indexDir, MultipleTreesBuilder.BuildMode.OFF_HEAP)) { builder.build(); } _segmentMetadata = new SegmentMetadataImpl(_indexDir); } } ColumnMinMaxValueGeneratorMode columnMinMaxValueGeneratorMode = _indexLoadingConfig.getColumnMinMaxValueGeneratorMode(); if (columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE) { ColumnMinMaxValueGenerator columnMinMaxValueGenerator = new ColumnMinMaxValueGenerator(_segmentMetadata, segmentWriter, columnMinMaxValueGeneratorMode); columnMinMaxValueGenerator.addColumnMinMaxValue(); } segmentWriter.save(); } } SegmentPreProcessor(File indexDir, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema); void process(); @Override void close(); }
SegmentPreProcessor implements AutoCloseable { public void process() throws Exception { if (_segmentMetadata.getTotalDocs() == 0) { return; } File[] directoryListing = _indexDir.listFiles(); String tempFileExtension = V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION + ".tmp"; if (directoryListing != null) { for (File child : directoryListing) { if (child.getName().endsWith(tempFileExtension)) { FileUtils.deleteQuietly(child); } } } try (SegmentDirectory.Writer segmentWriter = _segmentDirectory.createWriter()) { if (_schema != null) { DefaultColumnHandler defaultColumnHandler = DefaultColumnHandlerFactory.getDefaultColumnHandler(_indexDir, _schema, _segmentMetadata, segmentWriter); defaultColumnHandler.updateDefaultColumns(_indexLoadingConfig); _segmentMetadata = new SegmentMetadataImpl(_indexDir); _segmentDirectory.reloadMetadata(); } InvertedIndexHandler invertedIndexHandler = new InvertedIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); invertedIndexHandler.createInvertedIndices(); RangeIndexHandler rangeIndexHandler = new RangeIndexHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); rangeIndexHandler.createRangeIndices(); Set<String> textIndexColumns = _indexLoadingConfig.getTextIndexColumns(); if (textIndexColumns.size() > 0) { TextIndexHandler textIndexHandler = new TextIndexHandler(_indexDir, _segmentMetadata, textIndexColumns, segmentWriter); textIndexHandler.createTextIndexesOnSegmentLoad(); } BloomFilterHandler bloomFilterHandler = new BloomFilterHandler(_indexDir, _segmentMetadata, _indexLoadingConfig, segmentWriter); bloomFilterHandler.createBloomFilters(); if (_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { List<StarTreeV2BuilderConfig> starTreeBuilderConfigs = StarTreeBuilderUtils .generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), _segmentMetadata); boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty(); List<StarTreeV2Metadata> starTreeMetadataList = _segmentMetadata.getStarTreeV2MetadataList(); if (starTreeMetadataList != null) { if (StarTreeUtils.shouldRemoveExistingStarTrees(starTreeBuilderConfigs, starTreeMetadataList)) { LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName()); StarTreeUtils.removeStarTrees(_indexDir); _segmentMetadata = new SegmentMetadataImpl(_indexDir); } else { shouldGenerateStarTree = false; } } if (shouldGenerateStarTree) { try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, _indexDir, MultipleTreesBuilder.BuildMode.OFF_HEAP)) { builder.build(); } _segmentMetadata = new SegmentMetadataImpl(_indexDir); } } ColumnMinMaxValueGeneratorMode columnMinMaxValueGeneratorMode = _indexLoadingConfig.getColumnMinMaxValueGeneratorMode(); if (columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE) { ColumnMinMaxValueGenerator columnMinMaxValueGenerator = new ColumnMinMaxValueGenerator(_segmentMetadata, segmentWriter, columnMinMaxValueGeneratorMode); columnMinMaxValueGenerator.addColumnMinMaxValue(); } segmentWriter.save(); } } SegmentPreProcessor(File indexDir, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema); void process(); @Override void close();  }