TSwooshGrouping { public void swooshMatchWithMultipass(CombinedRecordMatcher combinedRecordMatcher, SurvivorShipAlgorithmParams survivorShipAlgorithmParams, int indexGID2) { groupRows = new HashMap<>(); List<RecordGenerator> notMasterRecords = new ArrayList<>(); for (RecordGenerator record : rcdsGenerators) { List<DQAttribute<?>> originalRow = record.getOriginalRow(); if (!StringUtils.equalsIgnoreCase("true", StringUtils.normalizeSpace(originalRow.get(indexGID2 + 2).getValue()))) { List<List<DQAttribute<?>>> list = groupRows.get(originalRow.get(indexGID2).getValue()); if (list == null) { list = new ArrayList<>(); list.add(originalRow); groupRows.put(originalRow.get(indexGID2).getValue(), list); } else { list.add(originalRow); } notMasterRecords.add(record); } else { resetMasterData(indexGID2, originalRow); } } rcdsGenerators.removeAll(notMasterRecords); totalCount = totalCount - notMasterRecords.size(); org.talend.dataquality.record.linkage.grouping.callback.MultiPassGroupingCallBack multiPassGroupingCallBack = new org.talend.dataquality.record.linkage.grouping.callback.MultiPassGroupingCallBack<>(this.oldGID2New, this.recordGrouping, this.groupRows); multiPassGroupingCallBack.setGIDindex(indexGID2); swooshMatch(combinedRecordMatcher, survivorShipAlgorithmParams, multiPassGroupingCallBack); List<Record> result = algorithm.getResult(); List<List<DQAttribute<?>>> remainNoMasterLs = new ArrayList<>(); if (!result.isEmpty()) { for (Record master : result) { String groupId = StringUtils.isBlank(master.getGroupId()) ? ((RichRecord) master).getGID().getValue() : master.getGroupId(); List<List<DQAttribute<?>>> list = groupRows.get(groupId); restoreMasterData((RichRecord) master); addMembersIntoNewMaster(master, list, groupId); groupRows.remove(groupId); String tempGid = oldGID2New.get(master.getGroupId()); if (!StringUtils.equals(groupId, tempGid)) { list = groupRows.get(tempGid); addMembersIntoNewMaster(master, list, groupId); groupRows.remove(tempGid); } } } else { for (RecordGenerator record : notMasterRecords) { remainNoMasterLs.add(record.getOriginalRow()); } } if (!groupRows.isEmpty()) { Iterator<String> keyIterator = groupRows.keySet().iterator(); while (keyIterator.hasNext()) { String keyGid = keyIterator.next(); remainNoMasterLs.addAll(groupRows.get(keyGid)); } } if (!remainNoMasterLs.isEmpty()) { for (List<DQAttribute<?>> originalRow : remainNoMasterLs) { String gid = oldGID2New.get(originalRow.get(indexGID2).getValue()); RichRecord createRecord = createRecord(originalRow, gid != null ? gid : originalRow.get(indexGID2).getValue()); output(createRecord); } } } TSwooshGrouping(AbstractRecordGrouping<TYPE> recordGrouping); Map<String, String> getOldGID2New(); void addToList(final TYPE[] inputRow, List<List<Map<java.lang.String, java.lang.String>>> multiMatchRules); void addToList(final TYPE[] inputRow, List<List<Map<java.lang.String, java.lang.String>>> multiMatchRules,
            SurvivorShipAlgorithmParams parameter); void swooshMatch(IRecordMatcher combinedRecordMatcher, SurvivorShipAlgorithmParams survParams); void initialMFBForOneRecord(IRecordMatcher combinedRecordMatcher, SurvivorShipAlgorithmParams survParams); void oneRecordMatch(RichRecord printRcd); void afterAllRecordFinished(); void swooshMatchWithMultipass(CombinedRecordMatcher combinedRecordMatcher,
            SurvivorShipAlgorithmParams survivorShipAlgorithmParams, int indexGID2); boolean isHasPassedOriginal();  }
@Test public void testSwooshMatchWithMultipass() { List<String[]> inputDataList = new ArrayList<>(); inputDataList.add( new String[] { "10", "nihaosnihao", "hello", "0ca5ee30-6377-4995-8f93-7f167bc5e16e", "2", "true", "1.0", "0.8333333333333334", "", "true" }); inputDataList.add(new String[] { "9", "nihao", "gogogo", "0ca5ee30-6377-4995-8f93-7f167bc5e16e", "0", "false", "0.8333333333333334", "0.0", "id:0.8333333333333334", "false" }); inputDataList.add(new String[] { "10", "nihaos", "hello", "0ca5ee30-6377-4995-8f93-7f167bc5e16e", "0", "false", "0.8333333333333334", "0.0", "id:0.8333333333333334", "false" }); inputDataList.add(new String[] { "12", "12345", "singlerecord", "3f1839cf-bf43-449e-9003-84747cc92a84", "1", "true", "1.0", "1.0", "", "true" }); inputDataList.add(new String[] { "11", "16", "hello", "4149c23f-046d-4a2b-a04d-354e5fbf6afc", "1", "true", "1.0", "1.0", "", "true" }); inputDataList.add( new String[] { "6", "33", null, "66cf5b44-f290-46e9-ab88-3c0779d9fc7b", "2", "true", "1.0", "1.0", "", "true" }); inputDataList .add(new String[] { "5", "3", null, "66cf5b44-f290-46e9-ab88-3c0779d9fc7b", "0", "false", "1.0", "0.0", "id:1.0", "false" }); inputDataList .add(new String[] { "6", "3", null, "66cf5b44-f290-46e9-ab88-3c0779d9fc7b", "0", "false", "1.0", "0.0", "id:1.0", "false" }); inputDataList.add( new String[] { "4", "2", "lis", "70ae237d-c24d-4ffc-b82e-bf111b9c441f", "1", "true", "1.0", "1.0", "", "true" }); inputDataList.add( new String[] { "2", "111", "li", "7da41ca8-a4b0-4bf4-8d4d-bb53edfd97ef", "3", "true", "1.0", "1.0", "", "true" }); inputDataList .add(new String[] { "1", "1", "li", "7da41ca8-a4b0-4bf4-8d4d-bb53edfd97ef", "0", "false", "1.0", "0.0", "id:1.0", "false" }); inputDataList.add( new String[] { "2", "1", "wang", "7da41ca8-a4b0-4bf4-8d4d-bb53edfd97ef", "0", "false", "1.0", "0.0", "id:1.0", "false" }); inputDataList.add( new String[] { "3", "1", "zhang", "7da41ca8-a4b0-4bf4-8d4d-bb53edfd97ef", "0", "false", "1.0", "0.0", "id:1.0", "false" }); inputDataList.add( new String[] { "7", "12", "zhao", "aa210319-0ed9-4075-903e-9cd6b25735e1", "1", "true", "1.0", "1.0", "", "true" }); inputDataList.add(new String[] { "8", "13", "zhaos", "e5e55f2f-2dfd-4773-8c02-9acbaa13c278", "1", "true", "1.0", "1.0", "", "true" }); Map<String, String> matchRuleMap = new HashMap<>(); matchRuleMap.put("MATCHING_TYPE", "Levenshtein"); matchRuleMap.put("RECORD_MATCH_THRESHOLD", "0.25"); matchRuleMap.put("ATTRIBUTE_NAME", "name"); matchRuleMap.put("SURVIVORSHIP_FUNCTION", "Concatenate"); matchRuleMap.put("CONFIDENCE_WEIGHT", "1"); matchRuleMap.put("HANDLE_NULL", "nullMatchNull"); matchRuleMap.put("ATTRIBUTE_THRESHOLD", "0.25"); matchRuleMap.put("COLUMN_IDX", "2"); matchRuleMap.put("MATCHING_ALGORITHM", "TSWOOSH_MATCHER"); matchRuleMap.put("PARAMETER", ""); List<Map<String, String>> ruleMapList = new ArrayList<>(); List<List<Map<String, String>>> ruleMapListList = new ArrayList<>(); ruleMapList.add(matchRuleMap); ruleMapListList.add(ruleMapList); TempRecordGrouping tempRecordGrouping = new TempRecordGrouping(); tempRecordGrouping.setOrginalInputColumnSize(3); tempRecordGrouping.setIsLinkToPrevious(true); TSwooshGrouping<String> tSwooshGrouping = new TSwooshGrouping<>(tempRecordGrouping); for (String[] stringRow : inputDataList) { tSwooshGrouping.addToList(stringRow, ruleMapListList); } int originalInputColumnSize = 3; CombinedRecordMatcher combinMatcher = new CombinedRecordMatcher(); combinMatcher.setblockingThreshold(1.0); combinMatcher.setRecordSize(1); MFBRecordMatcher mfbRecordMatcher = new MFBRecordMatcher(0.25); mfbRecordMatcher.setAttributeWeights(new double[] { 1.0 }); mfbRecordMatcher.setRecordMatchThreshold(0.25); mfbRecordMatcher.setRecordSize(1); LevenshteinMatcher levenshteinMatcher = new LevenshteinMatcher(); levenshteinMatcher.setAttributeName("name"); MFBAttributeMatcher mfbattMatcher = MFBAttributeMatcher.wrap(levenshteinMatcher, 1.0, 0.25, new SubString(-1, 0)); mfbRecordMatcher.setAttributeMatchers(new IAttributeMatcher[] { mfbattMatcher }); combinMatcher.add(mfbRecordMatcher); SurvivorShipAlgorithmParams survivorParams = createSurvivoshipAlgorithm(combinMatcher, mfbRecordMatcher); tSwooshGrouping.swooshMatchWithMultipass(combinMatcher, survivorParams, originalInputColumnSize); tSwooshGrouping.afterAllRecordFinished(); boolean singlerecordMasterIsExist = false; boolean nullMasterIsExist = false; boolean lilisMasterIsExist = false; boolean zhaoszhaoMasterIsExist = false; boolean hellohelloMasterIsExist = false; Assert.assertEquals("Expect the size of result items is 18 but it is " + result.size(), 18, result.size()); for (RichRecord rr : result) { String s = rr.getGID() == null ? rr.getGroupId() : rr.getGID().getValue(); String t = rr.getGRP_SIZE() == null ? rr.getGrpSize() + "" : rr.getGRP_SIZE().getValue(); log.trace("--" + rr.getAttributes().get(0).getValue() + "--" + s + "==" + rr.isMaster() + "==" + t + "==" + rr.getGroupQuality()); if (rr.isMaster()) { Attribute attribute = rr.getAttributes().get(0); if ("singlerecord".equals(attribute.getValue()) && rr.getGRP_SIZE().getValue().equals("1") && rr.getGRP_QUALITY().getValue().equals("1.0")) { singlerecordMasterIsExist = true; } else if (null == rr.getAttributes().get(0).getValue() && rr.getGRP_SIZE().getValue().equals("2") && rr.getGRP_QUALITY().getValue().equals("1.0")) { nullMasterIsExist = true; } else if ("lilis".equals(rr.getAttributes().get(0).getValue()) && StringUtils.equals(t, "4") && rr.getGroupQuality() == 0.6666666666666667) { lilisMasterIsExist = true; } else if ("zhaoszhao".equals(rr.getAttributes().get(0).getValue()) && rr.getGrpSize() == 2 && rr.getGroupQuality() == 0.8) { zhaoszhaoMasterIsExist = true; } else if ("hellohello".equals(rr.getAttributes().get(0).getValue()) && rr.getGrpSize() == 2 && rr.getGroupQuality() == 0.8333333333333334) { hellohelloMasterIsExist = true; } } } Assert.assertTrue( "There is a master data which name is singlerecord should be show at here and which group size should be 0 which group quality should be 1.0", singlerecordMasterIsExist); Assert.assertTrue( "There is a master data which name is null should be show at here and which group size should be 2 which group quality should be 1.0", nullMasterIsExist); Assert.assertTrue( "There is a master data which name is lilis should be show at here and which group size should be 5 which group quality should be 0.6666666666666667", lilisMasterIsExist); Assert.assertTrue( "There is a master data which name is zhaoszhao should be show at here and which group size should be 2 which group quality should be 0.8", zhaoszhaoMasterIsExist); Assert.assertTrue( "There is a master data which name is hellohello should be show at here and which group size should be 4 which group quality should be 0.8333333333333334", hellohelloMasterIsExist); }