@Test public void testPartitionJobs() throws Exception { UrlTrie trie = UrlTriePostOrderIteratorTest.getUrlTrie2(_property); UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, 4); String startDate = "2016-11-29"; String endDate = "2016-11-30"; Triple<String, GoogleWebmasterFilter.FilterOperator, UrlTrieNode> node0 = grouper.next(); TrieBasedProducerJob job0 = new TrieBasedProducerJob(startDate, endDate, node0, 100); Assert.assertEquals(job0.getPage(), _property + "0"); Assert.assertEquals(job0.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(job0.getStartDate(), startDate); Assert.assertEquals(job0.getEndDate(), endDate); Assert.assertEquals(job0.getPagesSize(), 3); List<? extends ProducerJob> granularJobs = job0.partitionJobs(); Assert.assertEquals(granularJobs.size(), 3); ProducerJob job03 = granularJobs.get(0); Assert.assertEquals(job03.getPage(), _property + "03"); Assert.assertEquals(job03.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(((TrieBasedProducerJob) job03).getGroupSize(), 2); List<? extends ProducerJob> job03Dates = job03.partitionJobs(); Assert.assertEquals(job03Dates.size(), 2); Assert.assertEquals(job03Dates.get(0), new SimpleProducerJob(_property + "03", startDate, startDate)); Assert.assertEquals(job03Dates.get(1), new SimpleProducerJob(_property + "03", endDate, endDate)); ProducerJob job04 = granularJobs.get(1); Assert.assertEquals(job04.getPage(), _property + "04"); Assert.assertEquals(job04.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(((TrieBasedProducerJob) job04).getGroupSize(), 2); List<? extends ProducerJob> job04Dates = job04.partitionJobs(); Assert.assertEquals(job04Dates.size(), 2); Assert.assertEquals(job04Dates.get(0), new SimpleProducerJob(_property + "04", startDate, startDate)); Assert.assertEquals(job04Dates.get(1), new SimpleProducerJob(_property + "04", endDate, endDate)); ProducerJob job0Only = granularJobs.get(2); Assert.assertEquals(job0Only.getPage(), _property + "0"); Assert.assertEquals(job0Only.getOperator(), GoogleWebmasterFilter.FilterOperator.EQUALS); Assert.assertEquals(((TrieBasedProducerJob) job0Only).getGroupSize(), 2); List<? extends ProducerJob> job0OnlyDates = job0Only.partitionJobs(); Assert.assertEquals(job0OnlyDates.size(), 2); Assert.assertEquals(job0OnlyDates.get(0), new SimpleProducerJob(_property + "0", startDate, startDate)); Assert.assertEquals(job0OnlyDates.get(1), new SimpleProducerJob(_property + "0", endDate, endDate)); Triple<String, GoogleWebmasterFilter.FilterOperator, UrlTrieNode> node1 = grouper.next(); TrieBasedProducerJob job1 = new TrieBasedProducerJob(startDate, endDate, node1, grouper.getGroupSize()); Assert.assertEquals(job1.getPage(), _property + "1"); Assert.assertEquals(job1.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(job1.getStartDate(), startDate); Assert.assertEquals(job1.getEndDate(), endDate); Assert.assertEquals(job1.getPagesSize(), 1); Assert.assertEquals(job1.partitionJobs().size(), 2); Triple<String, GoogleWebmasterFilter.FilterOperator, UrlTrieNode> node2 = grouper.next(); TrieBasedProducerJob job2 = new TrieBasedProducerJob(startDate, endDate, node2, grouper.getGroupSize()); Assert.assertEquals(job2.getPage(), _property + "2"); Assert.assertEquals(job2.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(job2.getStartDate(), startDate); Assert.assertEquals(job2.getEndDate(), endDate); Assert.assertEquals(job2.getPagesSize(), 4); List<? extends ProducerJob> job2Partitions = job2.partitionJobs(); Assert.assertEquals(job2Partitions.size(), 3); ProducerJob job5 = job2Partitions.get(0); Assert.assertEquals(job5.getPage(), _property + "25"); Assert.assertEquals(job5.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(job5.getStartDate(), startDate); Assert.assertEquals(job5.getEndDate(), endDate); Assert.assertEquals(job5.getPagesSize(), 2); Assert.assertEquals(job5.partitionJobs().size(), 2); ProducerJob job6 = job2Partitions.get(1); Assert.assertEquals(job6.getPage(), _property + "26"); Assert.assertEquals(job6.getOperator(), GoogleWebmasterFilter.FilterOperator.CONTAINS); Assert.assertEquals(job6.getStartDate(), startDate); Assert.assertEquals(job6.getEndDate(), endDate); Assert.assertEquals(job6.getPagesSize(), 1); ProducerJob job2Only = job2Partitions.get(2); Assert.assertEquals(job2Only.getPage(), _property + "2"); Assert.assertEquals(job2Only.getOperator(), GoogleWebmasterFilter.FilterOperator.EQUALS); Assert.assertEquals(job2Only.getStartDate(), startDate); Assert.assertEquals(job2Only.getEndDate(), endDate); Assert.assertEquals(job2Only.getPagesSize(), 1); Assert.assertFalse(grouper.hasNext()); }
@Override public List<? extends ProducerJob> partitionJobs() { UrlTrieNode root = _jobNode.getRight(); if (isOperatorEquals() || root.getSize() == 1) { return super.partitionJobs(); } else { if (_groupSize <= 1) { throw new RuntimeException("This is impossible. When group size is 1, the operator must be equals"); } UrlTrie trie = new UrlTrie(getPage(), root); int gs = Math.min(root.getSize(), _groupSize); UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, (int) Math.ceil(gs / 2.0)); List<TrieBasedProducerJob> jobs = new ArrayList<>(); while (grouper.hasNext()) { jobs.add(new TrieBasedProducerJob(_startDate, _endDate, grouper.next(), grouper.getGroupSize())); } return jobs; } }
TrieBasedProducerJob extends ProducerJob { @Override public List<? extends ProducerJob> partitionJobs() { UrlTrieNode root = _jobNode.getRight(); if (isOperatorEquals() || root.getSize() == 1) { return super.partitionJobs(); } else { if (_groupSize <= 1) { throw new RuntimeException("This is impossible. When group size is 1, the operator must be equals"); } UrlTrie trie = new UrlTrie(getPage(), root); int gs = Math.min(root.getSize(), _groupSize); UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, (int) Math.ceil(gs / 2.0)); List<TrieBasedProducerJob> jobs = new ArrayList<>(); while (grouper.hasNext()) { jobs.add(new TrieBasedProducerJob(_startDate, _endDate, grouper.next(), grouper.getGroupSize())); } return jobs; } } }
TrieBasedProducerJob extends ProducerJob { @Override public List<? extends ProducerJob> partitionJobs() { UrlTrieNode root = _jobNode.getRight(); if (isOperatorEquals() || root.getSize() == 1) { return super.partitionJobs(); } else { if (_groupSize <= 1) { throw new RuntimeException("This is impossible. When group size is 1, the operator must be equals"); } UrlTrie trie = new UrlTrie(getPage(), root); int gs = Math.min(root.getSize(), _groupSize); UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, (int) Math.ceil(gs / 2.0)); List<TrieBasedProducerJob> jobs = new ArrayList<>(); while (grouper.hasNext()) { jobs.add(new TrieBasedProducerJob(_startDate, _endDate, grouper.next(), grouper.getGroupSize())); } return jobs; } } TrieBasedProducerJob(String startDate, String endDate,
      Triple<String, GoogleWebmasterFilter.FilterOperator, UrlTrieNode> jobNode, int groupSize); }
TrieBasedProducerJob extends ProducerJob { @Override public List<? extends ProducerJob> partitionJobs() { UrlTrieNode root = _jobNode.getRight(); if (isOperatorEquals() || root.getSize() == 1) { return super.partitionJobs(); } else { if (_groupSize <= 1) { throw new RuntimeException("This is impossible. When group size is 1, the operator must be equals"); } UrlTrie trie = new UrlTrie(getPage(), root); int gs = Math.min(root.getSize(), _groupSize); UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, (int) Math.ceil(gs / 2.0)); List<TrieBasedProducerJob> jobs = new ArrayList<>(); while (grouper.hasNext()) { jobs.add(new TrieBasedProducerJob(_startDate, _endDate, grouper.next(), grouper.getGroupSize())); } return jobs; } } TrieBasedProducerJob(String startDate, String endDate,
      Triple<String, GoogleWebmasterFilter.FilterOperator, UrlTrieNode> jobNode, int groupSize); @Override String getPage(); @Override String getStartDate(); @Override String getEndDate(); @Override GoogleWebmasterFilter.FilterOperator getOperator(); @Override int getPagesSize(); @Override List<? extends ProducerJob> partitionJobs(); @Override String toString(); int getGroupSize(); }
TrieBasedProducerJob extends ProducerJob { @Override public List<? extends ProducerJob> partitionJobs() { UrlTrieNode root = _jobNode.getRight(); if (isOperatorEquals() || root.getSize() == 1) { return super.partitionJobs(); } else { if (_groupSize <= 1) { throw new RuntimeException("This is impossible. When group size is 1, the operator must be equals"); } UrlTrie trie = new UrlTrie(getPage(), root); int gs = Math.min(root.getSize(), _groupSize); UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, (int) Math.ceil(gs / 2.0)); List<TrieBasedProducerJob> jobs = new ArrayList<>(); while (grouper.hasNext()) { jobs.add(new TrieBasedProducerJob(_startDate, _endDate, grouper.next(), grouper.getGroupSize())); } return jobs; } } TrieBasedProducerJob(String startDate, String endDate,
      Triple<String, GoogleWebmasterFilter.FilterOperator, UrlTrieNode> jobNode, int groupSize); @Override String getPage(); @Override String getStartDate(); @Override String getEndDate(); @Override GoogleWebmasterFilter.FilterOperator getOperator(); @Override int getPagesSize(); @Override List<? extends ProducerJob> partitionJobs(); @Override String toString(); int getGroupSize();  }