@Test public void testCreateTermFrequencyVectors() throws Exception { runTest(false, false); }
public static void createTermFrequencyVectors(Path input, Path output, String tfVectorsFolderName, Configuration baseConf, int minSupport, int maxNGramSize, float minLLRValue, float normPower, boolean logNormalize, int numReducers, int chunkSizeInMegabytes, boolean sequentialAccess, boolean namedVectors) throws IOException, InterruptedException, ClassNotFoundException { Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || normPower >= 0, "If specified normPower must be nonnegative", normPower); Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || (normPower > 1 && !Double.isInfinite(normPower)) || !logNormalize, "normPower must be > 1 and not infinite if log normalization is chosen", normPower); if (chunkSizeInMegabytes < MIN_CHUNKSIZE) { chunkSizeInMegabytes = MIN_CHUNKSIZE; } else if (chunkSizeInMegabytes > MAX_CHUNKSIZE) { chunkSizeInMegabytes = MAX_CHUNKSIZE; } if (minSupport < 0) { minSupport = DEFAULT_MIN_SUPPORT; } Path dictionaryJobPath = new Path(output, DICTIONARY_JOB_FOLDER); log.info("Creating dictionary from {} and saving at {}", input, dictionaryJobPath); int[] maxTermDimension = new int[1]; List<Path> dictionaryChunks; if (maxNGramSize == 1) { startWordCounting(input, dictionaryJobPath, baseConf, minSupport); dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension); } else { CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize, minSupport, minLLRValue, numReducers); dictionaryChunks = createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output, baseConf, chunkSizeInMegabytes, maxTermDimension); } int partialVectorIndex = 0; Collection<Path> partialVectorPaths = Lists.newArrayList(); for (Path dictionaryChunk : dictionaryChunks) { Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++); partialVectorPaths.add(partialVectorOutputPath); makePartialVectors(input, baseConf, maxNGramSize, dictionaryChunk, partialVectorOutputPath, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); } Configuration conf = new Configuration(baseConf); Path outputDir = new Path(output, tfVectorsFolderName); PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); HadoopUtil.delete(conf, partialVectorPaths); }
DictionaryVectorizer extends AbstractJob implements Vectorizer { public static void createTermFrequencyVectors(Path input, Path output, String tfVectorsFolderName, Configuration baseConf, int minSupport, int maxNGramSize, float minLLRValue, float normPower, boolean logNormalize, int numReducers, int chunkSizeInMegabytes, boolean sequentialAccess, boolean namedVectors) throws IOException, InterruptedException, ClassNotFoundException { Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || normPower >= 0, "If specified normPower must be nonnegative", normPower); Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || (normPower > 1 && !Double.isInfinite(normPower)) || !logNormalize, "normPower must be > 1 and not infinite if log normalization is chosen", normPower); if (chunkSizeInMegabytes < MIN_CHUNKSIZE) { chunkSizeInMegabytes = MIN_CHUNKSIZE; } else if (chunkSizeInMegabytes > MAX_CHUNKSIZE) { chunkSizeInMegabytes = MAX_CHUNKSIZE; } if (minSupport < 0) { minSupport = DEFAULT_MIN_SUPPORT; } Path dictionaryJobPath = new Path(output, DICTIONARY_JOB_FOLDER); log.info("Creating dictionary from {} and saving at {}", input, dictionaryJobPath); int[] maxTermDimension = new int[1]; List<Path> dictionaryChunks; if (maxNGramSize == 1) { startWordCounting(input, dictionaryJobPath, baseConf, minSupport); dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension); } else { CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize, minSupport, minLLRValue, numReducers); dictionaryChunks = createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output, baseConf, chunkSizeInMegabytes, maxTermDimension); } int partialVectorIndex = 0; Collection<Path> partialVectorPaths = Lists.newArrayList(); for (Path dictionaryChunk : dictionaryChunks) { Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++); partialVectorPaths.add(partialVectorOutputPath); makePartialVectors(input, baseConf, maxNGramSize, dictionaryChunk, partialVectorOutputPath, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); } Configuration conf = new Configuration(baseConf); Path outputDir = new Path(output, tfVectorsFolderName); PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); HadoopUtil.delete(conf, partialVectorPaths); } }
DictionaryVectorizer extends AbstractJob implements Vectorizer { public static void createTermFrequencyVectors(Path input, Path output, String tfVectorsFolderName, Configuration baseConf, int minSupport, int maxNGramSize, float minLLRValue, float normPower, boolean logNormalize, int numReducers, int chunkSizeInMegabytes, boolean sequentialAccess, boolean namedVectors) throws IOException, InterruptedException, ClassNotFoundException { Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || normPower >= 0, "If specified normPower must be nonnegative", normPower); Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || (normPower > 1 && !Double.isInfinite(normPower)) || !logNormalize, "normPower must be > 1 and not infinite if log normalization is chosen", normPower); if (chunkSizeInMegabytes < MIN_CHUNKSIZE) { chunkSizeInMegabytes = MIN_CHUNKSIZE; } else if (chunkSizeInMegabytes > MAX_CHUNKSIZE) { chunkSizeInMegabytes = MAX_CHUNKSIZE; } if (minSupport < 0) { minSupport = DEFAULT_MIN_SUPPORT; } Path dictionaryJobPath = new Path(output, DICTIONARY_JOB_FOLDER); log.info("Creating dictionary from {} and saving at {}", input, dictionaryJobPath); int[] maxTermDimension = new int[1]; List<Path> dictionaryChunks; if (maxNGramSize == 1) { startWordCounting(input, dictionaryJobPath, baseConf, minSupport); dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension); } else { CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize, minSupport, minLLRValue, numReducers); dictionaryChunks = createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output, baseConf, chunkSizeInMegabytes, maxTermDimension); } int partialVectorIndex = 0; Collection<Path> partialVectorPaths = Lists.newArrayList(); for (Path dictionaryChunk : dictionaryChunks) { Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++); partialVectorPaths.add(partialVectorOutputPath); makePartialVectors(input, baseConf, maxNGramSize, dictionaryChunk, partialVectorOutputPath, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); } Configuration conf = new Configuration(baseConf); Path outputDir = new Path(output, tfVectorsFolderName); PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); HadoopUtil.delete(conf, partialVectorPaths); } private  DictionaryVectorizer(); }
DictionaryVectorizer extends AbstractJob implements Vectorizer { public static void createTermFrequencyVectors(Path input, Path output, String tfVectorsFolderName, Configuration baseConf, int minSupport, int maxNGramSize, float minLLRValue, float normPower, boolean logNormalize, int numReducers, int chunkSizeInMegabytes, boolean sequentialAccess, boolean namedVectors) throws IOException, InterruptedException, ClassNotFoundException { Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || normPower >= 0, "If specified normPower must be nonnegative", normPower); Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || (normPower > 1 && !Double.isInfinite(normPower)) || !logNormalize, "normPower must be > 1 and not infinite if log normalization is chosen", normPower); if (chunkSizeInMegabytes < MIN_CHUNKSIZE) { chunkSizeInMegabytes = MIN_CHUNKSIZE; } else if (chunkSizeInMegabytes > MAX_CHUNKSIZE) { chunkSizeInMegabytes = MAX_CHUNKSIZE; } if (minSupport < 0) { minSupport = DEFAULT_MIN_SUPPORT; } Path dictionaryJobPath = new Path(output, DICTIONARY_JOB_FOLDER); log.info("Creating dictionary from {} and saving at {}", input, dictionaryJobPath); int[] maxTermDimension = new int[1]; List<Path> dictionaryChunks; if (maxNGramSize == 1) { startWordCounting(input, dictionaryJobPath, baseConf, minSupport); dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension); } else { CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize, minSupport, minLLRValue, numReducers); dictionaryChunks = createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output, baseConf, chunkSizeInMegabytes, maxTermDimension); } int partialVectorIndex = 0; Collection<Path> partialVectorPaths = Lists.newArrayList(); for (Path dictionaryChunk : dictionaryChunks) { Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++); partialVectorPaths.add(partialVectorOutputPath); makePartialVectors(input, baseConf, maxNGramSize, dictionaryChunk, partialVectorOutputPath, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); } Configuration conf = new Configuration(baseConf); Path outputDir = new Path(output, tfVectorsFolderName); PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); HadoopUtil.delete(conf, partialVectorPaths); } private  DictionaryVectorizer(); @Override void createVectors(Path input, Path output, VectorizerConfig config); static void createTermFrequencyVectors(Path input,
                                                Path output,
                                                String tfVectorsFolderName,
                                                Configuration baseConf,
                                                int minSupport,
                                                int maxNGramSize,
                                                float minLLRValue,
                                                float normPower,
                                                boolean logNormalize,
                                                int numReducers,
                                                int chunkSizeInMegabytes,
                                                boolean sequentialAccess,
                                                boolean namedVectors); @Override int run(String[] args); static void main(String[] args); }
DictionaryVectorizer extends AbstractJob implements Vectorizer { public static void createTermFrequencyVectors(Path input, Path output, String tfVectorsFolderName, Configuration baseConf, int minSupport, int maxNGramSize, float minLLRValue, float normPower, boolean logNormalize, int numReducers, int chunkSizeInMegabytes, boolean sequentialAccess, boolean namedVectors) throws IOException, InterruptedException, ClassNotFoundException { Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || normPower >= 0, "If specified normPower must be nonnegative", normPower); Preconditions.checkArgument(normPower == PartialVectorMerger.NO_NORMALIZING || (normPower > 1 && !Double.isInfinite(normPower)) || !logNormalize, "normPower must be > 1 and not infinite if log normalization is chosen", normPower); if (chunkSizeInMegabytes < MIN_CHUNKSIZE) { chunkSizeInMegabytes = MIN_CHUNKSIZE; } else if (chunkSizeInMegabytes > MAX_CHUNKSIZE) { chunkSizeInMegabytes = MAX_CHUNKSIZE; } if (minSupport < 0) { minSupport = DEFAULT_MIN_SUPPORT; } Path dictionaryJobPath = new Path(output, DICTIONARY_JOB_FOLDER); log.info("Creating dictionary from {} and saving at {}", input, dictionaryJobPath); int[] maxTermDimension = new int[1]; List<Path> dictionaryChunks; if (maxNGramSize == 1) { startWordCounting(input, dictionaryJobPath, baseConf, minSupport); dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension); } else { CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize, minSupport, minLLRValue, numReducers); dictionaryChunks = createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output, baseConf, chunkSizeInMegabytes, maxTermDimension); } int partialVectorIndex = 0; Collection<Path> partialVectorPaths = Lists.newArrayList(); for (Path dictionaryChunk : dictionaryChunks) { Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++); partialVectorPaths.add(partialVectorOutputPath); makePartialVectors(input, baseConf, maxNGramSize, dictionaryChunk, partialVectorOutputPath, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); } Configuration conf = new Configuration(baseConf); Path outputDir = new Path(output, tfVectorsFolderName); PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize, maxTermDimension[0], sequentialAccess, namedVectors, numReducers); HadoopUtil.delete(conf, partialVectorPaths); } private  DictionaryVectorizer(); @Override void createVectors(Path input, Path output, VectorizerConfig config); static void createTermFrequencyVectors(Path input,
                                                Path output,
                                                String tfVectorsFolderName,
                                                Configuration baseConf,
                                                int minSupport,
                                                int maxNGramSize,
                                                float minLLRValue,
                                                float normPower,
                                                boolean logNormalize,
                                                int numReducers,
                                                int chunkSizeInMegabytes,
                                                boolean sequentialAccess,
                                                boolean namedVectors); @Override int run(String[] args); static void main(String[] args); static final String DOCUMENT_VECTOR_OUTPUT_FOLDER; static final String MIN_SUPPORT; static final String MAX_NGRAMS; static final int DEFAULT_MIN_SUPPORT; }