@Test public void testIsHeader_withValidHeader () { AbstractCache<VocabWord> cache = new AbstractCache<>(); String line = "48 100"; boolean isHeader = WordVectorSerializer.isHeader(line, cache); assertTrue(isHeader); }
static boolean isHeader(String line, AbstractCache cache) { if (!line.contains(" ")) { return true; } else { String[] headers = line.split(" "); try { long[] header = new long[headers.length]; for (int x = 0; x < headers.length; x++) { header[x] = Long.parseLong(headers[x]); } if (headers.length == 3) { long numberOfDocuments = header[2]; cache.incrementTotalDocCount(numberOfDocuments); } long numWords = header[0]; int vectorLength = (int) header[1]; printOutProjectedMemoryUse(numWords, vectorLength, 1); return true; } catch (Exception notHeaderException) { return false; } } }
WordVectorSerializer { static boolean isHeader(String line, AbstractCache cache) { if (!line.contains(" ")) { return true; } else { String[] headers = line.split(" "); try { long[] header = new long[headers.length]; for (int x = 0; x < headers.length; x++) { header[x] = Long.parseLong(headers[x]); } if (headers.length == 3) { long numberOfDocuments = header[2]; cache.incrementTotalDocCount(numberOfDocuments); } long numWords = header[0]; int vectorLength = (int) header[1]; printOutProjectedMemoryUse(numWords, vectorLength, 1); return true; } catch (Exception notHeaderException) { return false; } } } }
WordVectorSerializer { static boolean isHeader(String line, AbstractCache cache) { if (!line.contains(" ")) { return true; } else { String[] headers = line.split(" "); try { long[] header = new long[headers.length]; for (int x = 0; x < headers.length; x++) { header[x] = Long.parseLong(headers[x]); } if (headers.length == 3) { long numberOfDocuments = header[2]; cache.incrementTotalDocCount(numberOfDocuments); } long numWords = header[0]; int vectorLength = (int) header[1]; printOutProjectedMemoryUse(numWords, vectorLength, 1); return true; } catch (Exception notHeaderException) { return false; } } } private  WordVectorSerializer(); }
WordVectorSerializer { static boolean isHeader(String line, AbstractCache cache) { if (!line.contains(" ")) { return true; } else { String[] headers = line.split(" "); try { long[] header = new long[headers.length]; for (int x = 0; x < headers.length; x++) { header[x] = Long.parseLong(headers[x]); } if (headers.length == 3) { long numberOfDocuments = header[2]; cache.incrementTotalDocCount(numberOfDocuments); } long numWords = header[0]; int vectorLength = (int) header[1]; printOutProjectedMemoryUse(numWords, vectorLength, 1); return true; } catch (Exception notHeaderException) { return false; } } } private  WordVectorSerializer(); static Word2Vec readBinaryModel(
            InputStream inputStream,
            boolean linebreaks,
            boolean normalize); static void writeWordVectors(WeightLookupTable<T> lookupTable, String path); static void writeWordVectors(WeightLookupTable<T> lookupTable, File file); static void writeWordVectors(WeightLookupTable<T> lookupTable,
                                                                    OutputStream stream); @Deprecated static void writeWordVectors(@NonNull ParagraphVectors vectors, @NonNull File path); @Deprecated static void writeWordVectors(@NonNull ParagraphVectors vectors, @NonNull String path); static void writeParagraphVectors(ParagraphVectors vectors, File file); static void writeParagraphVectors(ParagraphVectors vectors, String path); static void writeWord2VecModel(Word2Vec vectors, File file); static void writeWord2VecModel(Word2Vec vectors, String path); static void writeWord2VecModel(Word2Vec vectors, OutputStream stream); static void writeParagraphVectors(ParagraphVectors vectors, OutputStream stream); static ParagraphVectors readParagraphVectors(String path); static ParagraphVectors readParagraphVectors(File file); @Deprecated static Word2Vec readWord2Vec(File file); static ParagraphVectors readParagraphVectors(InputStream stream); static Word2Vec readWord2VecFromText(@NonNull File vectors, @NonNull File hs, @NonNull File h_codes,
                                                @NonNull File h_points, @NonNull VectorsConfiguration configuration); @Deprecated static ParagraphVectors readParagraphVectorsFromText(@NonNull String path); @Deprecated static ParagraphVectors readParagraphVectorsFromText(@NonNull File file); @Deprecated static ParagraphVectors readParagraphVectorsFromText(@NonNull InputStream stream); @Deprecated static void writeWordVectors(ParagraphVectors vectors, OutputStream stream); @Deprecated static void writeWordVectors(InMemoryLookupTable lookupTable, InMemoryLookupCache cache, String path); @Deprecated static void writeFullModel(@NonNull Word2Vec vec, @NonNull String path); @Deprecated static Word2Vec loadFullModel(@NonNull String path); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull String path); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull File file); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull OutputStream outputStream); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull BufferedWriter writer); static WordVectors fromTableAndVocab(WeightLookupTable table, VocabCache vocab); static Word2Vec fromPair(Pair<InMemoryLookupTable, VocabCache> pair); @Deprecated static WordVectors loadTxtVectors(File vectorsFile); static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull File file); static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull InputStream inputStream); @Deprecated static WordVectors loadTxtVectors(@NonNull InputStream stream, boolean skipFirstLine); static void writeTsneFormat(Word2Vec vec, INDArray tsne, File csv); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull SequenceElementFactory<T> factory, @NonNull String path); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull SequenceElementFactory<T> factory, @NonNull File file); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull SequenceElementFactory<T> factory, @NonNull OutputStream stream); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull OutputStream stream); static SequenceVectors<T> readSequenceVectors(@NonNull String path,
                                                                                     boolean readExtendedTables); static SequenceVectors<T> readSequenceVectors(@NonNull File file,
                                                                                     boolean readExtendedTables); static SequenceVectors<T> readSequenceVectors(@NonNull InputStream stream,
                                                                                     boolean readExtendedTables); static SequenceVectors<T> readSequenceVectors(
            @NonNull SequenceElementFactory<T> factory, @NonNull File file); static SequenceVectors<T> readSequenceVectors(
            @NonNull SequenceElementFactory<T> factory, @NonNull InputStream stream); static void writeVocabCache(@NonNull VocabCache<VocabWord> vocabCache, @NonNull File file); static void writeVocabCache(@NonNull VocabCache<VocabWord> vocabCache, @NonNull OutputStream stream); static VocabCache<VocabWord> readVocabCache(@NonNull File file); static VocabCache<VocabWord> readVocabCache(@NonNull InputStream stream); static Word2Vec readWord2VecModel(String path); static Word2Vec readWord2VecModel(String path, boolean extendedModel); static Word2Vec readWord2VecModel(File file); static Word2Vec readWord2VecModel(File file, boolean extendedModel); static Word2Vec readAsBinaryNoLineBreaks(@NonNull File file); static Word2Vec readAsBinaryNoLineBreaks(@NonNull InputStream inputStream); static Word2Vec readAsBinary(@NonNull File file); static Word2Vec readAsBinary(@NonNull InputStream inputStream); static Word2Vec readAsCsv(@NonNull File file); static Word2Vec readAsCsv(@NonNull InputStream inputStream); static WordVectors loadStaticModel(InputStream inputStream); static WordVectors loadStaticModel(@NonNull File file); static void writeWord2Vec(@NonNull Word2Vec word2Vec, @NonNull OutputStream stream); static Word2Vec readWord2Vec(@NonNull String path, boolean readExtendedTables); static void writeLookupTable(WeightLookupTable<T> weightLookupTable,
                                                                     @NonNull File file); static WeightLookupTable<T> readLookupTable(File file); static WeightLookupTable<T> readLookupTable(InputStream stream); static Word2Vec readWord2Vec(@NonNull File file, boolean readExtendedTables); static Word2Vec readWord2Vec(
                @NonNull InputStream stream,
                boolean readExtendedTable); static void writeWordVectors(@NonNull FastText vectors, @NonNull File path); static FastText readWordVectors(File path); static void printOutProjectedMemoryUse(long numWords, int vectorLength, int numTables); }
WordVectorSerializer { static boolean isHeader(String line, AbstractCache cache) { if (!line.contains(" ")) { return true; } else { String[] headers = line.split(" "); try { long[] header = new long[headers.length]; for (int x = 0; x < headers.length; x++) { header[x] = Long.parseLong(headers[x]); } if (headers.length == 3) { long numberOfDocuments = header[2]; cache.incrementTotalDocCount(numberOfDocuments); } long numWords = header[0]; int vectorLength = (int) header[1]; printOutProjectedMemoryUse(numWords, vectorLength, 1); return true; } catch (Exception notHeaderException) { return false; } } } private  WordVectorSerializer(); static Word2Vec readBinaryModel(
            InputStream inputStream,
            boolean linebreaks,
            boolean normalize); static void writeWordVectors(WeightLookupTable<T> lookupTable, String path); static void writeWordVectors(WeightLookupTable<T> lookupTable, File file); static void writeWordVectors(WeightLookupTable<T> lookupTable,
                                                                    OutputStream stream); @Deprecated static void writeWordVectors(@NonNull ParagraphVectors vectors, @NonNull File path); @Deprecated static void writeWordVectors(@NonNull ParagraphVectors vectors, @NonNull String path); static void writeParagraphVectors(ParagraphVectors vectors, File file); static void writeParagraphVectors(ParagraphVectors vectors, String path); static void writeWord2VecModel(Word2Vec vectors, File file); static void writeWord2VecModel(Word2Vec vectors, String path); static void writeWord2VecModel(Word2Vec vectors, OutputStream stream); static void writeParagraphVectors(ParagraphVectors vectors, OutputStream stream); static ParagraphVectors readParagraphVectors(String path); static ParagraphVectors readParagraphVectors(File file); @Deprecated static Word2Vec readWord2Vec(File file); static ParagraphVectors readParagraphVectors(InputStream stream); static Word2Vec readWord2VecFromText(@NonNull File vectors, @NonNull File hs, @NonNull File h_codes,
                                                @NonNull File h_points, @NonNull VectorsConfiguration configuration); @Deprecated static ParagraphVectors readParagraphVectorsFromText(@NonNull String path); @Deprecated static ParagraphVectors readParagraphVectorsFromText(@NonNull File file); @Deprecated static ParagraphVectors readParagraphVectorsFromText(@NonNull InputStream stream); @Deprecated static void writeWordVectors(ParagraphVectors vectors, OutputStream stream); @Deprecated static void writeWordVectors(InMemoryLookupTable lookupTable, InMemoryLookupCache cache, String path); @Deprecated static void writeFullModel(@NonNull Word2Vec vec, @NonNull String path); @Deprecated static Word2Vec loadFullModel(@NonNull String path); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull String path); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull File file); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull OutputStream outputStream); @Deprecated static void writeWordVectors(@NonNull Word2Vec vec, @NonNull BufferedWriter writer); static WordVectors fromTableAndVocab(WeightLookupTable table, VocabCache vocab); static Word2Vec fromPair(Pair<InMemoryLookupTable, VocabCache> pair); @Deprecated static WordVectors loadTxtVectors(File vectorsFile); static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull File file); static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull InputStream inputStream); @Deprecated static WordVectors loadTxtVectors(@NonNull InputStream stream, boolean skipFirstLine); static void writeTsneFormat(Word2Vec vec, INDArray tsne, File csv); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull SequenceElementFactory<T> factory, @NonNull String path); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull SequenceElementFactory<T> factory, @NonNull File file); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull SequenceElementFactory<T> factory, @NonNull OutputStream stream); static void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                        @NonNull OutputStream stream); static SequenceVectors<T> readSequenceVectors(@NonNull String path,
                                                                                     boolean readExtendedTables); static SequenceVectors<T> readSequenceVectors(@NonNull File file,
                                                                                     boolean readExtendedTables); static SequenceVectors<T> readSequenceVectors(@NonNull InputStream stream,
                                                                                     boolean readExtendedTables); static SequenceVectors<T> readSequenceVectors(
            @NonNull SequenceElementFactory<T> factory, @NonNull File file); static SequenceVectors<T> readSequenceVectors(
            @NonNull SequenceElementFactory<T> factory, @NonNull InputStream stream); static void writeVocabCache(@NonNull VocabCache<VocabWord> vocabCache, @NonNull File file); static void writeVocabCache(@NonNull VocabCache<VocabWord> vocabCache, @NonNull OutputStream stream); static VocabCache<VocabWord> readVocabCache(@NonNull File file); static VocabCache<VocabWord> readVocabCache(@NonNull InputStream stream); static Word2Vec readWord2VecModel(String path); static Word2Vec readWord2VecModel(String path, boolean extendedModel); static Word2Vec readWord2VecModel(File file); static Word2Vec readWord2VecModel(File file, boolean extendedModel); static Word2Vec readAsBinaryNoLineBreaks(@NonNull File file); static Word2Vec readAsBinaryNoLineBreaks(@NonNull InputStream inputStream); static Word2Vec readAsBinary(@NonNull File file); static Word2Vec readAsBinary(@NonNull InputStream inputStream); static Word2Vec readAsCsv(@NonNull File file); static Word2Vec readAsCsv(@NonNull InputStream inputStream); static WordVectors loadStaticModel(InputStream inputStream); static WordVectors loadStaticModel(@NonNull File file); static void writeWord2Vec(@NonNull Word2Vec word2Vec, @NonNull OutputStream stream); static Word2Vec readWord2Vec(@NonNull String path, boolean readExtendedTables); static void writeLookupTable(WeightLookupTable<T> weightLookupTable,
                                                                     @NonNull File file); static WeightLookupTable<T> readLookupTable(File file); static WeightLookupTable<T> readLookupTable(InputStream stream); static Word2Vec readWord2Vec(@NonNull File file, boolean readExtendedTables); static Word2Vec readWord2Vec(
                @NonNull InputStream stream,
                boolean readExtendedTable); static void writeWordVectors(@NonNull FastText vectors, @NonNull File path); static FastText readWordVectors(File path); static void printOutProjectedMemoryUse(long numWords, int vectorLength, int numTables);  }