@Test public void testPreanalyzeSimpleTextEn() throws Exception { PlanImpl plan = new PlanImpl() { @Override public String getDocIdField() { return "id"; } }; String schemaFile = "solr/configsets/preanalyze/conf/schema.xml"; PreAnalyzeFields paf = new PreAnalyzeFields.Builder() .named("foo") .forTypeNamed("text_en") .preAnalyzingField("preanalyzed") .fromFile(schemaFile) .loadingResourcesVia(() -> plan.getClass().getClassLoader()) .build(); Scanner s = new ScannerImpl() { @Override public Runnable getScanOperation() { return null; } @Override public boolean isReady() { return false; } @Override public String getName() { return "foo"; } }; Document document = new DocumentImpl(new byte[0], "1", plan, Document.Operation.NEW, s); document.put("preanalyzed", "Quick red fox or 2 + 2 = 4"); paf.processDocument(document); assertEquals(ANALYZED, document.get("preanalyzed").get(0)); String testCol = "testCol"; CollectionAdminRequest.createCollection(testCol, CONFIG, 2, 2) .setMaxShardsPerNode(2) .process(solrClient); waitCol(2, testCol); SolrInputDocument sdoc = new SolrInputDocument(); for (String field : document.keySet()) { List<String> values = document.get(field); if (values.size() > 1) { sdoc.addField(field, values); } else { sdoc.addField(field, document.getFirstValue(field)); } } assertUpdateResponse(solrClient.add(testCol, Collections.singletonList(sdoc), 100)); assertUpdateResponse(solrClient.commit(testCol)); QueryResponse resp = solrClient.query(testCol, params( "q", "*:*", "rows", "10")); SolrDocumentList results = resp.getResults(); assertEquals(1, results.getNumFound()); resp = solrClient.query(testCol, params( "q", "id:1", "rows", "10")); results = resp.getResults(); assertEquals(1, results.getNumFound()); resp = solrClient.query(testCol, params( "q", "preanalyzed:*", "rows", "10")); results = resp.getResults(); assertEquals(1, results.getNumFound()); resp = solrClient.query(testCol, params( "q", "preanalyzed:quick", "rows", "10")); results = resp.getResults(); assertEquals(1, results.getNumFound()); resp = solrClient.query(testCol, params( "q", "preanalyzed:or", "rows", "10")); results = resp.getResults(); assertEquals(0, results.getNumFound()); }
@Override public Document[] processDocument(Document document) { for (String docFieldName : fieldsToAnalyze) { List<String> values = document.get(docFieldName); List<String> jsonValues = new ArrayList<>(values.size()); for (String value : values) { Map<String, Object> tokOutput = new HashMap<>(); tokOutput.put("v", "1"); tokOutput.put("str", value); ArrayList<Map<String, Object>> tokens = new ArrayList<>(); tokOutput.put("tokens", tokens); try (TokenStream ts = analyzer.get().tokenStream(docFieldName, value)) { ts.reset(); OffsetAttribute offsetA = ts.getAttribute(OffsetAttribute.class); CharTermAttribute charTermA = ts.getAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncA = ts.getAttribute(PositionIncrementAttribute.class); PayloadAttribute payloadA = ts.getAttribute(PayloadAttribute.class); TypeAttribute typeA = ts.getAttribute(TypeAttribute.class); FlagsAttribute flagsA = ts.getAttribute(FlagsAttribute.class); while (ts.incrementToken()) { Map<String, Object> tokAttrs = new HashMap<>(); char[] termChars = new char[charTermA.length()]; System.arraycopy(charTermA.buffer(), 0, termChars, 0, charTermA.length()); tokAttrs.put("t", new String(termChars)); tokAttrs.put("s", offsetA.startOffset()); tokAttrs.put("e", offsetA.endOffset()); tokAttrs.put("i", posIncA.getPositionIncrement()); Base64.Encoder encoder = Base64.getEncoder(); if (payloadA != null) { BytesRef payload = payloadA.getPayload(); if (payload != null) { tokAttrs.put("p", encoder.encode(payload.bytes)); } } tokAttrs.put("y", typeA.type()); if (flagsA != null) { tokAttrs.put("f", Integer.toString(flagsA.getFlags(), 16)); } tokens.add(tokAttrs); } ts.end(); jsonValues.add(mapper.writeValueAsString(tokOutput)); } catch (IOException e) { document.setStatus(Status.ERROR); throw new RuntimeException(e); } } document.removeAll(docFieldName); document.putAll(docFieldName, jsonValues); } return new Document[]{document}; }
PreAnalyzeFields implements DocumentProcessor { @Override public Document[] processDocument(Document document) { for (String docFieldName : fieldsToAnalyze) { List<String> values = document.get(docFieldName); List<String> jsonValues = new ArrayList<>(values.size()); for (String value : values) { Map<String, Object> tokOutput = new HashMap<>(); tokOutput.put("v", "1"); tokOutput.put("str", value); ArrayList<Map<String, Object>> tokens = new ArrayList<>(); tokOutput.put("tokens", tokens); try (TokenStream ts = analyzer.get().tokenStream(docFieldName, value)) { ts.reset(); OffsetAttribute offsetA = ts.getAttribute(OffsetAttribute.class); CharTermAttribute charTermA = ts.getAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncA = ts.getAttribute(PositionIncrementAttribute.class); PayloadAttribute payloadA = ts.getAttribute(PayloadAttribute.class); TypeAttribute typeA = ts.getAttribute(TypeAttribute.class); FlagsAttribute flagsA = ts.getAttribute(FlagsAttribute.class); while (ts.incrementToken()) { Map<String, Object> tokAttrs = new HashMap<>(); char[] termChars = new char[charTermA.length()]; System.arraycopy(charTermA.buffer(), 0, termChars, 0, charTermA.length()); tokAttrs.put("t", new String(termChars)); tokAttrs.put("s", offsetA.startOffset()); tokAttrs.put("e", offsetA.endOffset()); tokAttrs.put("i", posIncA.getPositionIncrement()); Base64.Encoder encoder = Base64.getEncoder(); if (payloadA != null) { BytesRef payload = payloadA.getPayload(); if (payload != null) { tokAttrs.put("p", encoder.encode(payload.bytes)); } } tokAttrs.put("y", typeA.type()); if (flagsA != null) { tokAttrs.put("f", Integer.toString(flagsA.getFlags(), 16)); } tokens.add(tokAttrs); } ts.end(); jsonValues.add(mapper.writeValueAsString(tokOutput)); } catch (IOException e) { document.setStatus(Status.ERROR); throw new RuntimeException(e); } } document.removeAll(docFieldName); document.putAll(docFieldName, jsonValues); } return new Document[]{document}; } }
PreAnalyzeFields implements DocumentProcessor { @Override public Document[] processDocument(Document document) { for (String docFieldName : fieldsToAnalyze) { List<String> values = document.get(docFieldName); List<String> jsonValues = new ArrayList<>(values.size()); for (String value : values) { Map<String, Object> tokOutput = new HashMap<>(); tokOutput.put("v", "1"); tokOutput.put("str", value); ArrayList<Map<String, Object>> tokens = new ArrayList<>(); tokOutput.put("tokens", tokens); try (TokenStream ts = analyzer.get().tokenStream(docFieldName, value)) { ts.reset(); OffsetAttribute offsetA = ts.getAttribute(OffsetAttribute.class); CharTermAttribute charTermA = ts.getAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncA = ts.getAttribute(PositionIncrementAttribute.class); PayloadAttribute payloadA = ts.getAttribute(PayloadAttribute.class); TypeAttribute typeA = ts.getAttribute(TypeAttribute.class); FlagsAttribute flagsA = ts.getAttribute(FlagsAttribute.class); while (ts.incrementToken()) { Map<String, Object> tokAttrs = new HashMap<>(); char[] termChars = new char[charTermA.length()]; System.arraycopy(charTermA.buffer(), 0, termChars, 0, charTermA.length()); tokAttrs.put("t", new String(termChars)); tokAttrs.put("s", offsetA.startOffset()); tokAttrs.put("e", offsetA.endOffset()); tokAttrs.put("i", posIncA.getPositionIncrement()); Base64.Encoder encoder = Base64.getEncoder(); if (payloadA != null) { BytesRef payload = payloadA.getPayload(); if (payload != null) { tokAttrs.put("p", encoder.encode(payload.bytes)); } } tokAttrs.put("y", typeA.type()); if (flagsA != null) { tokAttrs.put("f", Integer.toString(flagsA.getFlags(), 16)); } tokens.add(tokAttrs); } ts.end(); jsonValues.add(mapper.writeValueAsString(tokOutput)); } catch (IOException e) { document.setStatus(Status.ERROR); throw new RuntimeException(e); } } document.removeAll(docFieldName); document.putAll(docFieldName, jsonValues); } return new Document[]{document}; }  }
PreAnalyzeFields implements DocumentProcessor { @Override public Document[] processDocument(Document document) { for (String docFieldName : fieldsToAnalyze) { List<String> values = document.get(docFieldName); List<String> jsonValues = new ArrayList<>(values.size()); for (String value : values) { Map<String, Object> tokOutput = new HashMap<>(); tokOutput.put("v", "1"); tokOutput.put("str", value); ArrayList<Map<String, Object>> tokens = new ArrayList<>(); tokOutput.put("tokens", tokens); try (TokenStream ts = analyzer.get().tokenStream(docFieldName, value)) { ts.reset(); OffsetAttribute offsetA = ts.getAttribute(OffsetAttribute.class); CharTermAttribute charTermA = ts.getAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncA = ts.getAttribute(PositionIncrementAttribute.class); PayloadAttribute payloadA = ts.getAttribute(PayloadAttribute.class); TypeAttribute typeA = ts.getAttribute(TypeAttribute.class); FlagsAttribute flagsA = ts.getAttribute(FlagsAttribute.class); while (ts.incrementToken()) { Map<String, Object> tokAttrs = new HashMap<>(); char[] termChars = new char[charTermA.length()]; System.arraycopy(charTermA.buffer(), 0, termChars, 0, charTermA.length()); tokAttrs.put("t", new String(termChars)); tokAttrs.put("s", offsetA.startOffset()); tokAttrs.put("e", offsetA.endOffset()); tokAttrs.put("i", posIncA.getPositionIncrement()); Base64.Encoder encoder = Base64.getEncoder(); if (payloadA != null) { BytesRef payload = payloadA.getPayload(); if (payload != null) { tokAttrs.put("p", encoder.encode(payload.bytes)); } } tokAttrs.put("y", typeA.type()); if (flagsA != null) { tokAttrs.put("f", Integer.toString(flagsA.getFlags(), 16)); } tokens.add(tokAttrs); } ts.end(); jsonValues.add(mapper.writeValueAsString(tokOutput)); } catch (IOException e) { document.setStatus(Status.ERROR); throw new RuntimeException(e); } } document.removeAll(docFieldName); document.putAll(docFieldName, jsonValues); } return new Document[]{document}; }  @Override Document[] processDocument(Document document); @Override String getName(); }
PreAnalyzeFields implements DocumentProcessor { @Override public Document[] processDocument(Document document) { for (String docFieldName : fieldsToAnalyze) { List<String> values = document.get(docFieldName); List<String> jsonValues = new ArrayList<>(values.size()); for (String value : values) { Map<String, Object> tokOutput = new HashMap<>(); tokOutput.put("v", "1"); tokOutput.put("str", value); ArrayList<Map<String, Object>> tokens = new ArrayList<>(); tokOutput.put("tokens", tokens); try (TokenStream ts = analyzer.get().tokenStream(docFieldName, value)) { ts.reset(); OffsetAttribute offsetA = ts.getAttribute(OffsetAttribute.class); CharTermAttribute charTermA = ts.getAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncA = ts.getAttribute(PositionIncrementAttribute.class); PayloadAttribute payloadA = ts.getAttribute(PayloadAttribute.class); TypeAttribute typeA = ts.getAttribute(TypeAttribute.class); FlagsAttribute flagsA = ts.getAttribute(FlagsAttribute.class); while (ts.incrementToken()) { Map<String, Object> tokAttrs = new HashMap<>(); char[] termChars = new char[charTermA.length()]; System.arraycopy(charTermA.buffer(), 0, termChars, 0, charTermA.length()); tokAttrs.put("t", new String(termChars)); tokAttrs.put("s", offsetA.startOffset()); tokAttrs.put("e", offsetA.endOffset()); tokAttrs.put("i", posIncA.getPositionIncrement()); Base64.Encoder encoder = Base64.getEncoder(); if (payloadA != null) { BytesRef payload = payloadA.getPayload(); if (payload != null) { tokAttrs.put("p", encoder.encode(payload.bytes)); } } tokAttrs.put("y", typeA.type()); if (flagsA != null) { tokAttrs.put("f", Integer.toString(flagsA.getFlags(), 16)); } tokens.add(tokAttrs); } ts.end(); jsonValues.add(mapper.writeValueAsString(tokOutput)); } catch (IOException e) { document.setStatus(Status.ERROR); throw new RuntimeException(e); } } document.removeAll(docFieldName); document.putAll(docFieldName, jsonValues); } return new Document[]{document}; }  @Override Document[] processDocument(Document document); @Override String getName();  }