From commitspaul24@example.com Wed Mar 09 11:07:07 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 2028 invoked from network); 9 Mar 2011 11:07:06 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 11:07:06 -0000 Received: (qmail 46360 invoked by uid 500); 9 Mar 2011 11:07:06 -0000 Delivered-To: paul24@example.com Received: (qmail 45604 invoked by uid 500); 9 Mar 2011 11:07:05 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 45267 invoked by uid 99); 9 Mar 2011 11:07:05 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 11:07:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 11:07:04 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 3F5192388A3C; Wed, 9 Mar 2011 11:06:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079746 - in /nutch/branches/branch-1.3: CHANGES.txt conf/nutch-default.xml src/java/org/apache/nutch/fetcher/Fetcher.java Date: Wed, 09 Mar 2011 11:06:44 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> Author: ab Date: Wed Mar 9 11:06:43 2011 New Revision: 1079746 URL: http://svn.apache.org/viewvc?rev=1079746&view=rev Log: NUTCH-872 Change the default fetcher.parse to FALSE. Modified: nutch/branches/branch-1.3/CHANGES.txt nutch/branches/branch-1.3/conf/nutch-default.xml nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java Modified: nutch/branches/branch-1.3/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1079746&r1=1079745&r2=1079746&view=diff ============================================================================== --- nutch/branches/branch-1.3/CHANGES.txt (original) +++ nutch/branches/branch-1.3/CHANGES.txt Wed Mar 9 11:06:43 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 1.3 - Current Development +* NUTCH-872 Change the default fetcher.parse to FALSE (ab) + * NUTCH-564 External parser supports encoding attribute (Antony Bowesman, mattmann) * NUTCH-964 Upgraded Xerces to 2.91, ERROR conf.Configuration - Failed to set setXIncludeAware (markus) Modified: nutch/branches/branch-1.3/conf/nutch-default.xml URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/conf/nutch-default.xml?rev=1079746&r1=1079745&r2=1079746&view=diff ============================================================================== --- nutch/branches/branch-1.3/conf/nutch-default.xml (original) +++ nutch/branches/branch-1.3/conf/nutch-default.xml Wed Mar 9 11:06:43 2011 @@ -653,8 +653,9 @@ <property> <name>fetcher.parse</name> - <value>true</value> - <description>If true, fetcher will parse content.</description> + <value>false</value> + <description>If true, fetcher will parse content. Default is false, which means + that a separate parsing step is required after fetching is finished.</description> </property> <property> Modified: nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1079746&r1=1079745&r2=1079746&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java (original) +++ nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java Wed Mar 9 11:06:43 2011 @@ -1129,7 +1129,7 @@ public class Fetcher extends Configured Path segment = new Path(args[0]); int threads = getConf().getInt("fetcher.threads.fetch", 10); - boolean parsing = true; + boolean parsing = false; for (int i = 1; i < args.length; i++) { // parse command line if (args[i].equals("-threads")) { // found -threads option From commitspaul24@example.com Wed Mar 09 11:26:47 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 47333 invoked from network); 9 Mar 2011 11:26:46 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 11:26:46 -0000 Received: (qmail 68740 invoked by uid 500); 9 Mar 2011 11:26:46 -0000 Delivered-To: paul24@example.com Received: (qmail 68710 invoked by uid 500); 9 Mar 2011 11:26:46 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 68703 invoked by uid 99); 9 Mar 2011 11:26:46 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 11:26:46 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 11:26:42 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 33DE623889FA; Wed, 9 Mar 2011 11:26:21 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079753 - in /nutch/branches/branch-1.3: CHANGES.txt src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java Date: Wed, 09 Mar 2011 11:26:21 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: ab Date: Wed Mar 9 11:26:20 2011 New Revision: 1079753 URL: http://svn.apache.org/viewvc?rev=1079753&view=rev Log: NUTCH-876 Remove remaining robots/IP blocking code in lib-http. Modified: nutch/branches/branch-1.3/CHANGES.txt nutch/branches/branch-1.3/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java Modified: nutch/branches/branch-1.3/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1079753&r1=1079752&r2=1079753&view=diff ============================================================================== --- nutch/branches/branch-1.3/CHANGES.txt (original) +++ nutch/branches/branch-1.3/CHANGES.txt Wed Mar 9 11:26:20 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 1.3 - Current Development +* NUTCH-876 Remove remaining robots/IP blocking code in lib-http (ab) + * NUTCH-872 Change the default fetcher.parse to FALSE (ab) * NUTCH-564 External parser supports encoding attribute (Antony Bowesman, mattmann) Modified: nutch/branches/branch-1.3/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=1079753&r1=1079752&r2=1079753&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (original) +++ nutch/branches/branch-1.3/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java Wed Mar 9 11:26:20 2011 @@ -18,11 +18,7 @@ package org.apache.nutch.protocol.http.a // JDK imports import java.io.IOException; -import java.net.InetAddress; import java.net.URL; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.LinkedList; // Commons Logging imports import org.apache.commons.logging.Log; @@ -72,21 +68,6 @@ public abstract class HttpBase implement /** The length limit for downloaded content, in bytes. */ protected int maxContent = 64 * 1024; - /** The number of times a thread will delay when trying to fetch a page. */ - protected int maxDelays = 3; - - /** - * The maximum number of threads that should be allowed - * to access a host at one time. - */ - protected int maxThreadsPerHost = 1; - - /** - * The number of seconds the fetcher will delay between - * successive requests to the same server. - */ - protected long serverDelay = 1000; - /** The Nutch 'User-Agent' request header */ protected String userAgent = getAgentString( "NutchCVS", null, "Nutch", @@ -96,25 +77,6 @@ public abstract class HttpBase implement /** The "Accept-Language" request header value. */ protected String acceptLanguage = "en-us,en-gb,en;q=0.7,*;q=0.3"; - /** - * Maps from host to a Long naming the time it should be unblocked. - * The Long is zero while the host is in use, then set to now+wait when - * a request finishes. This way only one thread at a time accesses a - * host. - */ - private static HashMap BLOCKED_ADDR_TO_TIME = new HashMap(); - - /** - * Maps a host to the number of threads accessing that host. - */ - private static HashMap THREADS_PER_HOST_COUNT = new HashMap(); - - /** - * Queue of blocked hosts. This contains all of the non-zero entries - * from BLOCKED_ADDR_TO_TIME, ordered by increasing time. - */ - private static LinkedList BLOCKED_ADDR_QUEUE = new LinkedList(); - /** The default logger */ private final static Log LOGGER = LogFactory.getLog(HttpBase.class); @@ -124,21 +86,12 @@ public abstract class HttpBase implement /** The nutch configuration */ private Configuration conf = null; - /** Do we block by IP addresses or by hostnames? */ - private boolean byIP = true; - /** Do we use HTTP/1.1? */ protected boolean useHttp11 = false; /** Skip page if Crawl-Delay longer than this value. */ protected long maxCrawlDelay = -1L; - /** Plugin should handle host blocking internally. */ - protected boolean checkBlocking = true; - - /** Plugin should handle robot rules checking internally. */ - protected boolean checkRobots = true; - /** Creates a new instance of HttpBase */ public HttpBase() { this(null); @@ -160,19 +113,12 @@ public abstract class HttpBase implement this.useProxy = (proxyHost != null && proxyHost.length() > 0); this.timeout = conf.getInt("http.timeout", 10000); this.maxContent = conf.getInt("http.content.limit", 64 * 1024); - this.maxDelays = conf.getInt("http.max.delays", 3); - this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1); this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"), conf .get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email")); this.acceptLanguage = conf.get("http.accept.language", acceptLanguage); - this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000); - this.maxCrawlDelay = (long)(conf.getInt("fetcher.max.crawl.delay", -1) * 1000); // backward-compatible default setting - this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true); this.useHttp11 = conf.getBoolean("http.useHttp11", false); this.robots.setConf(conf); - this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true); - this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true); logConf(); } @@ -188,43 +134,8 @@ public abstract class HttpBase implement String urlString = url.toString(); try { URL u = new URL(urlString); - long delay = serverDelay; - - if (checkRobots) { - try { - if (!robots.isAllowed(this, u)) { - return new ProtocolOutput(null, new ProtocolStatus(ProtocolStatus.ROBOTS_DENIED, url)); - } - } catch (Throwable e) { - // XXX Maybe bogus: assume this is allowed. - if (logger.isTraceEnabled()) { - logger.trace("Exception checking robot rules for " + url + ": " + e); - } - } - - long crawlDelay = robots.getCrawlDelay(this, u); - delay = crawlDelay > 0 ? crawlDelay : serverDelay; - } - if (checkBlocking && maxCrawlDelay >= 0 && delay > maxCrawlDelay) { - // skip this page, otherwise the thread would block for too long. - LOGGER.info("Skipping: " + u + " exceeds fetcher.max.crawl.delay, max=" - + (maxCrawlDelay / 1000) + ", Crawl-Delay=" + (delay / 1000)); - return new ProtocolOutput(null, ProtocolStatus.STATUS_WOULDBLOCK); - } String host = null; - if (checkBlocking) { - try { - host = blockAddr(u, delay); - } catch (BlockedException be) { - return new ProtocolOutput(null, ProtocolStatus.STATUS_BLOCKED); - } - } - Response response; - try { - response = getResponse(u, datum, false); // make a request - } finally { - if (checkBlocking) unblockAddr(host, delay); - } + Response response = getResponse(u, datum, false); // make a request int code = response.getCode(); byte[] content = response.getContent(); @@ -313,18 +224,6 @@ public abstract class HttpBase implement return maxContent; } - public int getMaxDelays() { - return maxDelays; - } - - public int getMaxThreadsPerHost() { - return maxThreadsPerHost; - } - - public long getServerDelay() { - return serverDelay; - } - public String getUserAgent() { return userAgent; } @@ -340,94 +239,6 @@ public abstract class HttpBase implement return useHttp11; } - private String blockAddr(URL url, long crawlDelay) throws ProtocolException { - - String host; - if (byIP) { - try { - InetAddress addr = InetAddress.getByName(url.getHost()); - host = addr.getHostAddress(); - } catch (UnknownHostException e) { - // unable to resolve it, so don't fall back to host name - throw new HttpException(e); - } - } else { - host = url.getHost(); - if (host == null) - throw new HttpException("Unknown host for url: " + url); - host = host.toLowerCase(); - } - - int delays = 0; - while (true) { - cleanExpiredServerBlocks(); // free held addresses - - Long time; - synchronized (BLOCKED_ADDR_TO_TIME) { - time = (Long) BLOCKED_ADDR_TO_TIME.get(host); - if (time == null) { // address is free - - // get # of threads already accessing this addr - Integer counter = (Integer)THREADS_PER_HOST_COUNT.get(host); - int count = (counter == null) ? 0 : counter.intValue(); - - count++; // increment & store - THREADS_PER_HOST_COUNT.put(host, new Integer(count)); - - if (count >= maxThreadsPerHost) { - BLOCKED_ADDR_TO_TIME.put(host, new Long(0)); // block it - } - return host; - } - } - - if (delays == maxDelays) - throw new BlockedException("Exceeded http.max.delays: retry later."); - - long done = time.longValue(); - long now = System.currentTimeMillis(); - long sleep = 0; - if (done == 0) { // address is still in use - sleep = crawlDelay; // wait at least delay - - } else if (now < done) { // address is on hold - sleep = done - now; // wait until its free - } - - try { - Thread.sleep(sleep); - } catch (InterruptedException e) {} - delays++; - } - } - - private void unblockAddr(String host, long crawlDelay) { - synchronized (BLOCKED_ADDR_TO_TIME) { - int addrCount = ((Integer)THREADS_PER_HOST_COUNT.get(host)).intValue(); - if (addrCount == 1) { - THREADS_PER_HOST_COUNT.remove(host); - BLOCKED_ADDR_QUEUE.addFirst(host); - BLOCKED_ADDR_TO_TIME.put - (host, new Long(System.currentTimeMillis() + crawlDelay)); - } else { - THREADS_PER_HOST_COUNT.put(host, new Integer(addrCount - 1)); - } - } - } - - private static void cleanExpiredServerBlocks() { - synchronized (BLOCKED_ADDR_TO_TIME) { - for (int i = BLOCKED_ADDR_QUEUE.size() - 1; i >= 0; i--) { - String host = (String) BLOCKED_ADDR_QUEUE.get(i); - long time = ((Long) BLOCKED_ADDR_TO_TIME.get(host)).longValue(); - if (time <= System.currentTimeMillis()) { - BLOCKED_ADDR_TO_TIME.remove(host); - BLOCKED_ADDR_QUEUE.remove(i); - } - } - } - } - private static String getAgentString(String agentName, String agentVersion, String agentDesc, @@ -481,12 +292,6 @@ public abstract class HttpBase implement logger.info("http.content.limit = " + maxContent); logger.info("http.agent = " + userAgent); logger.info("http.accept.language = " + acceptLanguage); - logger.info(Protocol.CHECK_BLOCKING + " = " + checkBlocking); - logger.info(Protocol.CHECK_ROBOTS + " = " + checkRobots); - if (checkBlocking) { - logger.info("fetcher.server.delay = " + serverDelay); - logger.info("http.max.delays = " + maxDelays); - } } } Modified: nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java?rev=1079753&r1=1079752&r2=1079753&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java (original) +++ nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java Wed Mar 9 11:26:20 2011 @@ -170,11 +170,6 @@ public class Http extends HttpBase { params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); - if (maxThreadsTotal > maxThreadsPerHost) { - params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost); - } else { - params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); - } // executeMethod(HttpMethod) seems to ignore the connection timeout on the connection manager. // set it explicitly on the HttpClient. From commitspaul24@example.com Wed Mar 09 11:48:51 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 28652 invoked from network); 9 Mar 2011 11:48:50 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 11:48:50 -0000 Received: (qmail 95313 invoked by uid 500); 9 Mar 2011 11:48:50 -0000 Delivered-To: paul24@example.com Received: (qmail 95289 invoked by uid 500); 9 Mar 2011 11:48:50 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 95282 invoked by uid 99); 9 Mar 2011 11:48:50 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 11:48:50 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 11:48:45 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8265F2388A2C; Wed, 9 Mar 2011 11:48:23 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079760 - in /nutch/branches/branch-1.3: ./ src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/ src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/ src/plugin/urlfilter-domain/src/java/org/apache/... Date: Wed, 09 Mar 2011 11:48:23 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: ab Date: Wed Mar 9 11:48:22 2011 New Revision: 1079760 URL: http://svn.apache.org/viewvc?rev=1079760&view=rev Log: NUTCH-921 Reduce dependency of Nutch on config files. Modified: nutch/branches/branch-1.3/CHANGES.txt nutch/branches/branch-1.3/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java nutch/branches/branch-1.3/src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/AutomatonURLFilter.java nutch/branches/branch-1.3/src/plugin/urlfilter-domain/src/java/org/apache/nutch/urlfilter/domain/DomainURLFilter.java nutch/branches/branch-1.3/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java nutch/branches/branch-1.3/src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java nutch/branches/branch-1.3/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java Modified: nutch/branches/branch-1.3/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/CHANGES.txt (original) +++ nutch/branches/branch-1.3/CHANGES.txt Wed Mar 9 11:48:22 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 1.3 - Current Development +* NUTCH-921 Reduce dependency of Nutch on config files (ab) + * NUTCH-876 Remove remaining robots/IP blocking code in lib-http (ab) * NUTCH-872 Change the default fetcher.parse to FALSE (ab) Modified: nutch/branches/branch-1.3/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java (original) +++ nutch/branches/branch-1.3/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java Wed Mar 9 11:48:22 2011 @@ -17,11 +17,13 @@ package org.apache.nutch.urlfilter.api; // JDK imports +import java.io.File; import java.io.Reader; import java.io.FileReader; import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.IOException; +import java.io.StringReader; import java.util.List; import java.util.ArrayList; @@ -74,10 +76,21 @@ public abstract class RegexURLFilterBase * Constructs a new RegexURLFilter and init it with a file of rules. * @param filename is the name of rules file. */ - public RegexURLFilterBase(String filename) + public RegexURLFilterBase(File filename) throws IOException, IllegalArgumentException { this(new FileReader(filename)); } + + /** + * Constructs a new RegexURLFilter and inits it with a list of rules. + * @param rules string with a list of rules, one rule per line + * @throws IOException + * @throws IllegalArgumentException + */ + public RegexURLFilterBase(String rules) throws IOException, + IllegalArgumentException { + this(new StringReader(rules)); + } /** * Constructs a new RegexURLFilter and init it with a Reader of rules. @@ -85,7 +98,7 @@ public abstract class RegexURLFilterBase */ protected RegexURLFilterBase(Reader reader) throws IOException, IllegalArgumentException { - rules = readRulesFile(reader); + rules = readRules(reader); } /** @@ -102,9 +115,9 @@ public abstract class RegexURLFilterBase * Returns the name of the file of rules to use for * a particular implementation. * @param conf is the current configuration. - * @return the name of the file of rules to use. + * @return the name of the resource containing the rules to use. */ - protected abstract String getRulesFile(Configuration conf); + protected abstract Reader getRulesReader(Configuration conf) throws IOException; /* -------------------------- * @@ -132,18 +145,18 @@ public abstract class RegexURLFilterBase public void setConf(Configuration conf) { this.conf = conf; - String file = getRulesFile(conf); - Reader reader = conf.getConfResourceAsReader(file); - if (reader == null) { - if (LOG.isFatalEnabled()) { LOG.fatal("Can't find resource: " + file); } - } else { - try { - rules = readRulesFile(reader); - } catch (IOException e) { - if (LOG.isFatalEnabled()) { LOG.fatal(e.getMessage()); } - //TODO paul24@example.com: throw Exception? Because broken api. - throw new RuntimeException(e.getMessage(), e); - } + Reader reader = null; + try { + reader = getRulesReader(conf); + } catch (Exception e) { + if (LOG.isErrorEnabled()) { LOG.error(e.getMessage()); } + throw new RuntimeException(e.getMessage(), e); + } + try { + rules = readRules(reader); + } catch (IOException e) { + if (LOG.isErrorEnabled()) { LOG.error(e.getMessage()); } + throw new RuntimeException(e.getMessage(), e); } } @@ -161,7 +174,7 @@ public abstract class RegexURLFilterBase * @param reader is a reader of regular expressions rules. * @return the corresponding {@RegexRule rules}. */ - private RegexRule[] readRulesFile(Reader reader) + private RegexRule[] readRules(Reader reader) throws IOException, IllegalArgumentException { BufferedReader in = new BufferedReader(reader); Modified: nutch/branches/branch-1.3/src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/AutomatonURLFilter.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/AutomatonURLFilter.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/AutomatonURLFilter.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/AutomatonURLFilter.java Wed Mar 9 11:48:22 2011 @@ -19,6 +19,7 @@ package org.apache.nutch.urlfilter.autom // JDK imports import java.io.Reader; import java.io.IOException; +import java.io.StringReader; import java.util.regex.PatternSyntaxException; // Hadoop imports @@ -41,6 +42,8 @@ import org.apache.nutch.urlfilter.api.Re * @see <a href="http://www.brics.dk/automaton/">dk.brics.automaton</a> */ public class AutomatonURLFilter extends RegexURLFilterBase { + public static final String URLFILTER_AUTOMATON_FILE = "urlfilter.automaton.file"; + public static final String URLFILTER_AUTOMATON_RULES = "urlfilter.automaton.rules"; public AutomatonURLFilter() { super(); @@ -61,9 +64,17 @@ public class AutomatonURLFilter extends * <implementation:RegexURLFilterBase> * * ----------------------------------- */ - // Inherited Javadoc - protected String getRulesFile(Configuration conf) { - return conf.get("urlfilter.automaton.file"); + /** + * Rules specified as a config property will override rules specified + * as a config file. + */ + protected Reader getRulesReader(Configuration conf) throws IOException { + String stringRules = conf.get(URLFILTER_AUTOMATON_RULES); + if (stringRules != null) { + return new StringReader(stringRules); + } + String fileRules = conf.get(URLFILTER_AUTOMATON_FILE); + return conf.getConfResourceAsReader(fileRules); } // Inherited Javadoc Modified: nutch/branches/branch-1.3/src/plugin/urlfilter-domain/src/java/org/apache/nutch/urlfilter/domain/DomainURLFilter.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlfilter-domain/src/java/org/apache/nutch/urlfilter/domain/DomainURLFilter.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlfilter-domain/src/java/org/apache/nutch/urlfilter/domain/DomainURLFilter.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlfilter-domain/src/java/org/apache/nutch/urlfilter/domain/DomainURLFilter.java Wed Mar 9 11:48:22 2011 @@ -20,6 +20,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import java.util.LinkedHashSet; import java.util.Set; @@ -70,7 +71,7 @@ public class DomainURLFilter private String domainFile = null; private Set<String> domainSet = new LinkedHashSet<String>(); - private void readConfigurationFile(Reader configReader) + private void readConfiguration(Reader configReader) throws IOException { // read the configuration file, line by line @@ -140,21 +141,24 @@ public class DomainURLFilter // domain file and attribute "file" take precedence if defined String file = conf.get("urlfilter.domain.file"); + String stringRules = conf.get("urlfilter.domain.rules"); if (domainFile != null) { file = domainFile; } else if (attributeFile != null) { file = attributeFile; } - - // get the file as a classpath resource and populate the domain set with - // the domains from the file + Reader reader = null; + if (stringRules != null) { // takes precedence over files + reader = new StringReader(stringRules); + } else { + reader = conf.getConfResourceAsReader(file); + } try { - Reader reader = conf.getConfResourceAsReader(file); if (reader == null) { reader = new FileReader(file); } - readConfigurationFile(reader); + readConfiguration(reader); } catch (IOException e) { LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); Modified: nutch/branches/branch-1.3/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java Wed Mar 9 11:48:22 2011 @@ -36,6 +36,7 @@ import java.io.FileReader; import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.IOException; +import java.io.StringReader; import java.util.List; import java.util.ArrayList; @@ -63,8 +64,8 @@ public class PrefixURLFilter implements } - public PrefixURLFilter(String filename) throws IOException { - trie = readConfigurationFile(new FileReader(filename)); + public PrefixURLFilter(String stringRules) throws IOException { + trie = readConfiguration(new StringReader(stringRules)); } public String filter(String url) { @@ -74,7 +75,7 @@ public class PrefixURLFilter implements return url; } - private TrieStringMatcher readConfigurationFile(Reader reader) + private TrieStringMatcher readConfiguration(Reader reader) throws IOException { BufferedReader in=new BufferedReader(reader); @@ -144,16 +145,22 @@ public class PrefixURLFilter implements } String file = conf.get("urlfilter.prefix.file"); + String stringRules = conf.get("urlfilter.prefix.rules"); // attribute "file" takes precedence if defined if (attributeFile != null) file = attributeFile; - Reader reader = conf.getConfResourceAsReader(file); + Reader reader = null; + if (stringRules != null) { // takes precedence over files + reader = new StringReader(stringRules); + } else { + reader = conf.getConfResourceAsReader(file); + } if (reader == null) { trie = new PrefixStringMatcher(new String[0]); } else { try { - trie = readConfigurationFile(reader); + trie = readConfiguration(reader); } catch (IOException e) { if (LOG.isFatalEnabled()) { LOG.fatal(e.getMessage()); } // TODO paul24@example.com: throw Exception? Because broken api. Modified: nutch/branches/branch-1.3/src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java Wed Mar 9 11:48:22 2011 @@ -19,6 +19,7 @@ package org.apache.nutch.urlfilter.regex // JDK imports import java.io.Reader; import java.io.IOException; +import java.io.StringReader; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -35,6 +36,9 @@ import org.apache.nutch.util.NutchConfig * {@link java.util.regex Java Regex implementation}. */ public class RegexURLFilter extends RegexURLFilterBase { + + public static final String URLFILTER_REGEX_FILE = "urlfilter.regex.file"; + public static final String URLFILTER_REGEX_RULES = "urlfilter.regex.rules"; public RegexURLFilter() { super(); @@ -55,9 +59,17 @@ public class RegexURLFilter extends Rege * <implementation:RegexURLFilterBase> * * ----------------------------------- */ - // Inherited Javadoc - protected String getRulesFile(Configuration conf) { - return conf.get("urlfilter.regex.file"); + /** + * Rules specified as a config property will override rules specified + * as a config file. + */ + protected Reader getRulesReader(Configuration conf) throws IOException { + String stringRules = conf.get(URLFILTER_REGEX_RULES); + if (stringRules != null) { + return new StringReader(stringRules); + } + String fileRules = conf.get(URLFILTER_REGEX_FILE); + return conf.getConfResourceAsReader(fileRules); } // Inherited Javadoc Modified: nutch/branches/branch-1.3/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java Wed Mar 9 11:48:22 2011 @@ -34,6 +34,7 @@ import java.io.FileReader; import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.IOException; +import java.io.StringReader; import java.util.List; import java.util.ArrayList; @@ -139,7 +140,7 @@ public class SuffixURLFilter implements } public SuffixURLFilter(Reader reader) throws IOException { - readConfigurationFile(reader); + readConfiguration(reader); } public String filter(String url) { @@ -167,7 +168,7 @@ public class SuffixURLFilter implements } } - public void readConfigurationFile(Reader reader) throws IOException { + public void readConfiguration(Reader reader) throws IOException { // handle missing config file if (reader == null) { @@ -269,12 +270,18 @@ public class SuffixURLFilter implements } String file = conf.get("urlfilter.suffix.file"); + String stringRules = conf.get("urlfilter.suffix.rules"); // attribute "file" takes precedence if defined if (attributeFile != null) file = attributeFile; - Reader reader = conf.getConfResourceAsReader(file); + Reader reader = null; + if (stringRules != null) { // takes precedence over files + reader = new StringReader(stringRules); + } else { + reader = conf.getConfResourceAsReader(file); + } try { - readConfigurationFile(reader); + readConfiguration(reader); } catch (IOException e) { if (LOG.isFatalEnabled()) { LOG.fatal(e.getMessage()); } throw new RuntimeException(e.getMessage(), e); Modified: nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java Wed Mar 9 11:48:22 2011 @@ -20,8 +20,11 @@ package org.apache.nutch.net.urlnormaliz import java.net.URL; import java.net.MalformedURLException; import java.io.FileInputStream; +import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; import java.util.Collections; import java.util.HashMap; @@ -40,6 +43,7 @@ import org.apache.nutch.util.NutchConfig import javax.xml.parsers.*; import org.w3c.dom.*; +import org.xml.sax.InputSource; import org.apache.oro.text.regex.*; /** @@ -106,17 +110,23 @@ public class RegexURLNormalizer extends // the default constructor was called if (this.scopedRules == null) { String filename = getConf().get("urlnormalizer.regex.file"); + String stringRules = getConf().get("urlnormalizer.regex.rules"); scopedRules = new HashMap(); - URL url = getConf().getResource(filename); + Reader reader = null; + if (stringRules != null) { + reader = new StringReader(stringRules); + } else { + reader = getConf().getConfResourceAsReader(filename); + } List rules = null; - if (url == null) { - LOG.warn("Can't load the default config file! " + filename); + if (reader == null) { + LOG.warn("Can't load the default rules! "); rules = EMPTY_RULES; } else { try { - rules = readConfiguration(url.openStream()); + rules = readConfiguration(reader); } catch (Exception e) { - LOG.warn("Couldn't read default config from '" + url + "': " + e); + LOG.warn("Couldn't read default config: " + e); rules = EMPTY_RULES; } } @@ -125,8 +135,8 @@ public class RegexURLNormalizer extends } // used in JUnit test. - void setConfiguration(InputStream is, String scope) { - List rules = readConfiguration(is); + void setConfiguration(Reader reader, String scope) { + List rules = readConfiguration(reader); scopedRules.put(scope, rules); LOG.debug("Set config for scope '" + scope + "': " + rules.size() + " rules."); } @@ -141,17 +151,16 @@ public class RegexURLNormalizer extends // try to populate String configFile = getConf().get("urlnormalizer.regex.file." + scope); if (configFile != null) { - URL resource = getConf().getResource(configFile); - LOG.debug("resource for scope '" + scope + "': " + resource); - if (resource == null) { + LOG.debug("resource for scope '" + scope + "': " + configFile); + if (configFile == null) { LOG.warn("Can't load resource for config file: " + configFile); } else { try { - InputStream is = resource.openStream(); - curRules = readConfiguration(resource.openStream()); + Reader reader = getConf().getConfResourceAsReader(configFile); + curRules = readConfiguration(reader); scopedRules.put(scope, curRules); } catch (Exception e) { - LOG.warn("Couldn't load resource '" + resource + "': " + e); + LOG.warn("Couldn't load resource '" + configFile + "': " + e); } } } @@ -185,22 +194,22 @@ public class RegexURLNormalizer extends LOG.info("loading " + filename); } try { - FileInputStream fis = new FileInputStream(filename); - return readConfiguration(fis); + FileReader reader = new FileReader(filename); + return readConfiguration(reader); } catch (Exception e) { - LOG.fatal("Error loading rules from '" + filename + "': " + e); + LOG.error("Error loading rules from '" + filename + "': " + e); return EMPTY_RULES; } } - private List readConfiguration(InputStream is) { + private List readConfiguration(Reader reader) { Perl5Compiler compiler = new Perl5Compiler(); List rules = new ArrayList(); try { // borrowed heavily from code in Configuration.java Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() - .parse(is); + .parse(new InputSource(reader)); Element root = doc.getDocumentElement(); if ((!"regex-normalize".equals(root.getTagName())) && (LOG.isFatalEnabled())) { Modified: nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java?rev=1079760&r1=1079759&r2=1079760&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java (original) +++ nutch/branches/branch-1.3/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java Wed Mar 9 11:48:22 2011 @@ -21,6 +21,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; +import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; @@ -64,10 +65,10 @@ public class TestRegexURLNormalizer exte }); for (int i = 0; i < configs.length; i++) { try { - FileInputStream fis = new FileInputStream(configs[i]); + FileReader reader = new FileReader(configs[i]); String cname = configs[i].getName(); cname = cname.substring(16, cname.indexOf(".xml")); - normalizer.setConfiguration(fis, cname); + normalizer.setConfiguration(reader, cname); NormalizedURL[] urls = readTestFile(cname); testData.put(cname, urls); } catch (Exception e) { From commitspaul24@example.com Wed Mar 09 12:01:25 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 49667 invoked from network); 9 Mar 2011 12:01:25 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 12:01:25 -0000 Received: (qmail 15851 invoked by uid 500); 9 Mar 2011 12:01:25 -0000 Delivered-To: paul24@example.com Received: (qmail 15816 invoked by uid 500); 9 Mar 2011 12:01:25 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 15809 invoked by uid 99); 9 Mar 2011 12:01:25 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 12:01:25 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 12:01:24 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 5416D23889FA; Wed, 9 Mar 2011 12:01:04 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079764 - in /nutch/trunk: CHANGES.txt src/java/org/apache/nutch/fetcher/FetcherReducer.java Date: Wed, 09 Mar 2011 12:01:04 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> Author: ab Date: Wed Mar 9 12:01:03 2011 New Revision: 1079764 URL: http://svn.apache.org/viewvc?rev=1079764&view=rev Log: NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects. Modified: nutch/trunk/CHANGES.txt nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java Modified: nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1079764&r1=1079763&r2=1079764&view=diff ============================================================================== --- nutch/trunk/CHANGES.txt (original) +++ nutch/trunk/CHANGES.txt Wed Mar 9 12:01:03 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 2.0 - Current Development +* NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects (Sebastian Nagel via ab) + * NUTCH-964 Upgraded Xerces to 2.91 (markus) * NUTCH-824 Crawling - File Error 404 when fetching file with an hexadecimal character in the file name (Michela Becchi via jnioche) Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=1079764&r1=1079763&r2=1079764&view=diff ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java (original) +++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java Wed Mar 9 12:01:03 2011 @@ -514,14 +514,14 @@ extends GoraReducer<IntWritable, FetchEn output(fit, null, status, CrawlStatus.STATUS_RETRY); } - if (redirecting && redirectCount >= maxRedirect) { + if (redirecting && redirectCount > maxRedirect) { fetchQueues.finishFetchItem(fit); LOG.info(" - redirect count exceeded " + fit.url); output(fit, null, ProtocolStatusUtils.STATUS_REDIR_EXCEEDED, CrawlStatus.STATUS_GONE); } - } while (redirecting && (redirectCount < maxRedirect)); + } while (redirecting && (redirectCount <= maxRedirect)); } catch (final Throwable t) { // unexpected exception // unblock From commitspaul24@example.com Wed Mar 09 12:02:19 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 57255 invoked from network); 9 Mar 2011 12:02:19 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 12:02:19 -0000 Received: (qmail 18444 invoked by uid 500); 9 Mar 2011 12:02:19 -0000 Delivered-To: paul24@example.com Received: (qmail 18409 invoked by uid 500); 9 Mar 2011 12:02:19 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 18402 invoked by uid 99); 9 Mar 2011 12:02:19 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 12:02:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 12:02:17 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 143A023889FA; Wed, 9 Mar 2011 12:01:56 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079765 - in /nutch/branches/branch-1.3: CHANGES.txt src/java/org/apache/nutch/fetcher/Fetcher.java Date: Wed, 09 Mar 2011 12:01:56 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: ab Date: Wed Mar 9 12:01:55 2011 New Revision: 1079765 URL: http://svn.apache.org/viewvc?rev=1079765&view=rev Log: NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects. Modified: nutch/branches/branch-1.3/CHANGES.txt nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java Modified: nutch/branches/branch-1.3/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1079765&r1=1079764&r2=1079765&view=diff ============================================================================== --- nutch/branches/branch-1.3/CHANGES.txt (original) +++ nutch/branches/branch-1.3/CHANGES.txt Wed Mar 9 12:01:55 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 1.3 - Current Development +* NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects (Sebastian Nagel via ab) + * NUTCH-921 Reduce dependency of Nutch on config files (ab) * NUTCH-876 Remove remaining robots/IP blocking code in lib-http (ab) Modified: nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1079765&r1=1079764&r2=1079765&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java (original) +++ nutch/branches/branch-1.3/src/java/org/apache/nutch/fetcher/Fetcher.java Wed Mar 9 12:01:55 2011 @@ -751,7 +751,7 @@ public class Fetcher extends Configured output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_FETCH_RETRY); } - if (redirecting && redirectCount >= maxRedirect) { + if (redirecting && redirectCount > maxRedirect) { fetchQueues.finishFetchItem(fit); if (LOG.isInfoEnabled()) { LOG.info(" - redirect count exceeded " + fit.url); @@ -759,7 +759,7 @@ public class Fetcher extends Configured output(fit.url, fit.datum, null, ProtocolStatus.STATUS_REDIR_EXCEEDED, CrawlDatum.STATUS_FETCH_GONE); } - } while (redirecting && (redirectCount < maxRedirect)); + } while (redirecting && (redirectCount <= maxRedirect)); } catch (Throwable t) { // unexpected exception // unblock From commitspaul24@example.com Wed Mar 09 12:22:04 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 16783 invoked from network); 9 Mar 2011 12:22:04 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 12:22:04 -0000 Received: (qmail 46917 invoked by uid 500); 9 Mar 2011 12:22:04 -0000 Delivered-To: paul24@example.com Received: (qmail 46900 invoked by uid 500); 9 Mar 2011 12:22:04 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 46893 invoked by uid 99); 9 Mar 2011 12:22:04 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 12:22:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 12:22:01 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4CB902388A3F; Wed, 9 Mar 2011 12:21:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079770 - in /nutch/trunk: CHANGES.txt ivy/ivy.xml src/plugin/protocol-sftp/ivy.xml Date: Wed, 09 Mar 2011 12:21:40 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: ab Date: Wed Mar 9 12:21:39 2011 New Revision: 1079770 URL: http://svn.apache.org/viewvc?rev=1079770&view=rev Log: NUTCH-955 Ivy configuration improvements. Upgrade to Xerces 2.9.1 and Restlet 2.0.5. Modified: nutch/trunk/CHANGES.txt nutch/trunk/ivy/ivy.xml nutch/trunk/src/plugin/protocol-sftp/ivy.xml Modified: nutch/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1079770&r1=1079769&r2=1079770&view=diff ============================================================================== --- nutch/trunk/CHANGES.txt (original) +++ nutch/trunk/CHANGES.txt Wed Mar 9 12:21:39 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 2.0 - Current Development +* NUTCH-955 Ivy configuration improvements. Upgrade to Xerces 2.9.1 and Restlet 2.0.5 (alexis via ab) + * NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects (Sebastian Nagel via ab) * NUTCH-964 Upgraded Xerces to 2.91 (markus) Modified: nutch/trunk/ivy/ivy.xml URL: http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1079770&r1=1079769&r2=1079770&view=diff ============================================================================== --- nutch/trunk/ivy/ivy.xml (original) +++ nutch/trunk/ivy/ivy.xml Wed Mar 9 12:21:39 2011 @@ -108,9 +108,32 @@ <exclude org="com.sun.jmx"/> <exclude org="javax.jms"/> </dependency> - <dependency org="org.restlet.jse" name="org.restlet" rev="2.0.1" conf="*->default"/> - <dependency org="org.restlet.jse" name="org.restlet.ext.jackson" rev="2.0.1" conf="*->default"/> - + <dependency org="org.restlet.jse" name="org.restlet" rev="2.0.5" conf="*->default"/> + <dependency org="org.restlet.jse" name="org.restlet.ext.jackson" rev="2.0.5" conf="*->default"/> + +<!-- + Uncomment this to use MySQL as database with SQL as Gora store. +--> +<!-- + <dependency org="mysql" name="mysql-connector-java" rev="5.1.13" conf="*->default"/> +--> + +<!-- + Uncomment this to use HBase as Gora backend. Then manually add hbase-0.20.6 jar to the lib directory. +--> +<!-- + <dependency org="org.apache.gora" name="gora-hbase" rev="0.1" conf="*->compile"> + <exclude org="com.sun.jdmk"/> + <exclude org="com.sun.jmx"/> + <exclude org="javax.jms"/> + </dependency> + <dependency org="org.apache.zookeeper" name="zookeeper" rev="3.3.2" conf="*->default"> + <exclude org="com.sun.jdmk"/> + <exclude org="com.sun.jmx"/> + <exclude org="javax.jms"/> + </dependency> +--> + <!--global exclusion--> <exclude module="ant" /> </dependencies> Modified: nutch/trunk/src/plugin/protocol-sftp/ivy.xml URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-sftp/ivy.xml?rev=1079770&r1=1079769&r2=1079770&view=diff ============================================================================== --- nutch/trunk/src/plugin/protocol-sftp/ivy.xml (original) +++ nutch/trunk/src/plugin/protocol-sftp/ivy.xml Wed Mar 9 12:21:39 2011 @@ -27,7 +27,7 @@ </info> <configurations> - <include file="${nutch.root}/ivy/ivy-configurations.xml"/> + <include file="${basedir}/ivy/ivy-configurations.xml"/> </configurations> <publications> From commitspaul24@example.com Wed Mar 09 19:40:59 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 90863 invoked from network); 9 Mar 2011 19:40:59 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 9 Mar 2011 19:40:59 -0000 Received: (qmail 27548 invoked by uid 500); 9 Mar 2011 19:40:59 -0000 Delivered-To: paul24@example.com Received: (qmail 27529 invoked by uid 500); 9 Mar 2011 19:40:59 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 27522 invoked by uid 99); 9 Mar 2011 19:40:59 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 19:40:59 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2011 19:40:57 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 6316923889B9; Wed, 9 Mar 2011 19:40:36 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079960 - /nutch/trunk/src/plugin/protocol-sftp/ivy.xml Date: Wed, 09 Mar 2011 19:40:36 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: ab Date: Wed Mar 9 19:40:36 2011 New Revision: 1079960 URL: http://svn.apache.org/viewvc?rev=1079960&view=rev Log: Unbreak the build. I've been warned, just didn't pay attention... :) Modified: nutch/trunk/src/plugin/protocol-sftp/ivy.xml Modified: nutch/trunk/src/plugin/protocol-sftp/ivy.xml URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-sftp/ivy.xml?rev=1079960&r1=1079959&r2=1079960&view=diff ============================================================================== --- nutch/trunk/src/plugin/protocol-sftp/ivy.xml (original) +++ nutch/trunk/src/plugin/protocol-sftp/ivy.xml Wed Mar 9 19:40:36 2011 @@ -27,7 +27,7 @@ </info> <configurations> - <include file="${basedir}/ivy/ivy-configurations.xml"/> + <include file="${nutch.root}/ivy/ivy-configurations.xml"/> </configurations> <publications> From commitspaul24@example.com Thu Mar 10 21:54:35 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 18326 invoked from network); 10 Mar 2011 21:54:35 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 10 Mar 2011 21:54:35 -0000 Received: (qmail 69863 invoked by uid 500); 10 Mar 2011 21:54:35 -0000 Delivered-To: paul24@example.com Received: (qmail 69803 invoked by uid 500); 10 Mar 2011 21:54:34 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 69795 invoked by uid 99); 10 Mar 2011 21:54:34 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Mar 2011 21:54:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Mar 2011 21:54:29 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id DA76823889E5; Thu, 10 Mar 2011 21:54:05 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r1080368 - in /nutch/branches/branch-1.3: CHANGES.txt build.xml pom.xml Date: Thu, 10 Mar 2011 21:54:05 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: jnioche Date: Thu Mar 10 21:54:05 2011 New Revision: 1080368 URL: http://svn.apache.org/viewvc?rev=1080368&view=rev Log: NUTCH-951 : backport changes from 2.0 into 1.3 - NUTCH-825 done Added: nutch/branches/branch-1.3/pom.xml Modified: nutch/branches/branch-1.3/CHANGES.txt nutch/branches/branch-1.3/build.xml Modified: nutch/branches/branch-1.3/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1080368&r1=1080367&r2=1080368&view=diff ============================================================================== --- nutch/branches/branch-1.3/CHANGES.txt (original) +++ nutch/branches/branch-1.3/CHANGES.txt Thu Mar 10 21:54:05 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 1.3 - Current Development +* NUTCH-825 Publish nutch artifacts to central maven repository (mattmann, jnioche) + * NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects (Sebastian Nagel via ab) * NUTCH-921 Reduce dependency of Nutch on config files (ab) Modified: nutch/branches/branch-1.3/build.xml URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/build.xml?rev=1080368&r1=1080367&r2=1080368&view=diff ============================================================================== --- nutch/branches/branch-1.3/build.xml (original) +++ nutch/branches/branch-1.3/build.xml Thu Mar 10 21:54:05 2011 @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> -<project name="Nutch" default="runtime" xmlns:ivy="antlib:org.apache.ivy.ant"> +<project name="Nutch" default="runtime" xmlns:ivy="antlib:org.apache.ivy.ant" xmlns:artifact="antlib:org.apache.maven.artifact.ant"> <!-- Load all the default properties, and any the user wants --> <!-- to contribute (without having to type -D or edit this file --> @@ -23,7 +23,17 @@ <property file="${basedir}/build.properties" /> <property file="${basedir}/default.properties" /> <property name="test.junit.output.format" value="plain"/> - <!-- <loadproperties srcfile="${ivy.dir}/libraries.properties"/> --> + <property name="release.dir" value="${build.dir}/release"/> + + <!-- define Maven coordinates, repository url and artifacts name etc --> + <property name="groupId" value="org.apache.nutch" /> + <property name="artifactId" value="nutch" /> + <property name="maven-repository-url" value="https://repository.apache.org/service/local/staging/deploy/maven2" /> + <property name="maven-repository-id" value="apache.releases.https" /> + <property name="maven-jar" value="${release.dir}/${artifactId}-${version}.jar" /> + <property name="maven-javadoc-jar" value="${release.dir}/${artifactId}-${version}-javadoc.jar" /> + <property name="maven-sources-jar" value="${release.dir}/${artifactId}-${version}-sources.jar" /> + <!-- the normal classpath --> <path id="classpath"> @@ -53,6 +63,7 @@ <target name="init" depends="ivy-init"> <mkdir dir="${build.dir}"/> <mkdir dir="${build.classes}"/> + <mkdir dir="${release.dir}"/> <mkdir dir="${test.build.dir}"/> <mkdir dir="${test.build.classes}"/> @@ -109,6 +120,126 @@ </target> <!-- ================================================================== --> + <!-- Make Maven Central Release --> + <!-- ================================================================== --> + <!-- --> + <!-- ================================================================== --> + <target name="release" depends="compile-core" description="generate the release distribution"> + <copy file="${conf.dir}/nutch-default.xml" + todir="${build.classes}"/> + <copy file="${conf.dir}/nutch-site.xml" + todir="${build.classes}"/> + + <!-- build the main artifact --> + <jar jarfile="${maven-jar}" basedir="${build.classes}" /> + + <!-- build the javadoc artifact --> + <javadoc + destdir="${release.dir}/javadoc" + overview="${src.dir}/overview.html" + author="true" + version="true" + use="true" + windowtitle="${Name} ${version} API" + doctitle="${Name} ${version} API" + bottom="Copyright &amp;copy; ${year} The Apache Software Foundation" + > + <arg value="${javadoc.proxy.host}"/> + <arg value="${javadoc.proxy.port}"/> + + <packageset dir="${src.dir}"/> + <packageset dir="${plugins.dir}/lib-http/src/java"/> + <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/> + <packageset dir="${plugins.dir}/microformats-reltag/src/java"/> + <packageset dir="${plugins.dir}/protocol-file/src/java"/> + <packageset dir="${plugins.dir}/protocol-ftp/src/java"/> + <packageset dir="${plugins.dir}/protocol-http/src/java"/> + <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/> + <packageset dir="${plugins.dir}/parse-tika/src/java"/> + <packageset dir="${plugins.dir}/parse-ext/src/java"/> + <packageset dir="${plugins.dir}/parse-js/src/java"/> + <packageset dir="${plugins.dir}/parse-rss/src/java"/> + <packageset dir="${plugins.dir}/parse-swf/src/java"/> + <packageset dir="${plugins.dir}/parse-zip/src/java"/> + <packageset dir="${plugins.dir}/index-basic/src/java"/> + <packageset dir="${plugins.dir}/index-more/src/java"/> + <packageset dir="${plugins.dir}/scoring-opic/src/java"/> + <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/> + <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/> + <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/> + <packageset dir="${plugins.dir}/creativecommons/src/java"/> + <packageset dir="${plugins.dir}/languageidentifier/src/java"/> + + <link href="${javadoc.link.java}"/> + <link href="${javadoc.link.lucene}"/> + <link href="${javadoc.link.hadoop}"/> + + <classpath refid="classpath"/> + <classpath> + <fileset dir="${plugins.dir}" > + <include name="**/*.jar"/> + </fileset> + </classpath> + + <group title="Core" packages="org.apache.nutch.*"/> + <group title="Plugins API" packages="${plugins.api}"/> + <group title="Protocol Plugins" packages="${plugins.protocol}"/> + <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/> + <group title="Scoring Plugins" packages="${plugins.scoring}"/> + <group title="Parse Plugins" packages="${plugins.parse}"/> + <group title="Indexing Filter Plugins" packages="${plugins.index}"/> + <group title="Misc. Plugins" packages="${plugins.misc}"/> + </javadoc> + <jar jarfile="${maven-javadoc-jar}"> + <fileset dir="${release.dir}/javadoc" /> + </jar> + + <!-- build the sources artifact --> + <jar jarfile="${maven-sources-jar}"> + <fileset dir="${src.dir}" /> + </jar> + </target> + + <!-- ================================================================== --> + <!-- Deploy to Apache Nexus --> + <!-- ================================================================== --> + <!-- --> + <!-- ================================================================== --> + <target name="deploy" depends="release" description="deploy to Apache Nexus"> + <!-- sign and deploy the main artifact --> + <artifact:mvn> + <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" /> + <arg value="-Durl=${maven-repository-url}" /> + <arg value="-DrepositoryId=${maven-repository-id}" /> + <arg value="-DpomFile=pom.xml" /> + <arg value="-Dfile=${maven-jar}" /> + <arg value="-Papache-release" /> + </artifact:mvn> + + <!-- sign and deploy the sources artifact --> + <artifact:mvn> + <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" /> + <arg value="-Durl=${maven-repository-url}" /> + <arg value="-DrepositoryId=${maven-repository-id}" /> + <arg value="-DpomFile=pom.xml" /> + <arg value="-Dfile=${maven-sources-jar}" /> + <arg value="-Dclassifier=sources" /> + <arg value="-Papache-release" /> + </artifact:mvn> + + <!-- sign and deploy the javadoc artifact --> + <artifact:mvn> + <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" /> + <arg value="-Durl=${maven-repository-url}" /> + <arg value="-DrepositoryId=${maven-repository-id}" /> + <arg value="-DpomFile=pom.xml" /> + <arg value="-Dfile=${maven-javadoc-jar}" /> + <arg value="-Dclassifier=javadoc" /> + <arg value="-Papache-release" /> + </artifact:mvn> + </target> + + <!-- ================================================================== --> <!-- Make job jar --> <!-- ================================================================== --> <!-- --> Added: nutch/branches/branch-1.3/pom.xml URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/pom.xml?rev=1080368&view=auto ============================================================================== --- nutch/branches/branch-1.3/pom.xml (added) +++ nutch/branches/branch-1.3/pom.xml Thu Mar 10 21:54:05 2011 @@ -0,0 +1,294 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.nutch</groupId> + <artifactId>nutch</artifactId> + <packaging>jar</packaging> + <name>Apache Nutch</name> + <version>1.3-dev</version> + <description>Nutch is open source web-search software. It builds on + Lucene and Solr, adding web-specifics, such as a crawler, a link-graph + database, parsers for HTML and other document formats, etc. + </description> + <url>http://maven.apache.org</url> + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt + </url> + <distribution>repo</distribution> + </license> + </licenses> + <scm> + <url>http://svn.apache.org/viewvc/nutch + </url> + <connection>http://svn.apache.org/viewvc/nutch + </connection> + </scm> + <developers> + <developer> + <id>ab</id> + <name>Andrzej Bialecki</name> + <email>paul24@example.com</email> + </developer> + <developer> + <id>mattmann</id> + <name>Chris A. Mattmann</name> + <email>paul24@example.com</email> + </developer> + <developer> + <id>kubes</id> + <name>Dennis Kubes</name> + <email>paul24@example.com</email> + </developer> + <developer> + <id>dogacan</id> + <name>Dogacan Güney</name> + <email>paul24@example.com</email> + </developer> + <developer> + <id>jnioche</id> + <name>Julien Nioche</name> + <email>paul24@example.com</email> + </developer> + <developer> + <id>otis</id> + <name>Otis Gospodnetić</name> + <email>paul24@example.com</email> + </developer> + <developer> + <id>siren</id> + <name>Sami Siren</name> + <email>paul24@example.com</email> + </developer> + </developers> + <dependencies> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-solrj</artifactId> + <version>1.4.1</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>1.5.11</version> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.4</version> + </dependency> + <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <version>3.1</version> + </dependency> + <dependency> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + <version>3.1</version> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.3</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>0.20.2</version> + <exclusions> + <exclusion> + <groupId>hsqldb</groupId> + <artifactId>hsqldb</artifactId> + </exclusion> + <exclusion> + <groupId>net.sf.kosmosfs</groupId> + <artifactId>kfs</artifactId> + </exclusion> + <exclusion> + <groupId>net.java.dev.jets3t</groupId> + <artifactId>jets3t</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jdt</groupId> + <artifactId>core</artifactId> + </exclusion> + <exclusion> + <groupId>ant</groupId> + <artifactId>ant</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + <version>4.0.1</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-misc</artifactId> + <version>3.0.2</version> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-core</artifactId> + <version>0.7</version> + </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-parsers</artifactId> + <version>0.7</version> + <exclusions> + <exclusion> + <groupId>org.sun.jdmk</groupId> + </exclusion> + <exclusion> + <groupId>com.sun.jmx</groupId> + </exclusion> + <exclusion> + <groupId>javax.jms</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.15</version> + <exclusions> + <exclusion> + <groupId>org.sun.jdmk</groupId> + </exclusion> + <exclusion> + <groupId>com.sun.jmx</groupId> + </exclusion> + <exclusion> + <groupId>javax.jms</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xmlParserAPIs</artifactId> + <version>2.6.2</version> + </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + <version>2.9.1</version> + </dependency> + <dependency> + <groupId>oro</groupId> + <artifactId>oro</artifactId> + <version>2.0.8</version> + </dependency> + <dependency> + <groupId>com.healthmarketscience.sqlbuilder</groupId> + <artifactId>sqlbuilder</artifactId> + <version>2.0.6</version> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + <version>6.1.22</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>3.8.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>0.20.2</version> + <exclusions> + <exclusion> + <groupId>hsqldb</groupId> + <artifactId>hsqldb</artifactId> + </exclusion> + <exclusion> + <groupId>net.sf.kosmosfs</groupId> + <artifactId>kfs</artifactId> + </exclusion> + <exclusion> + <groupId>net.java.dev.jets3t</groupId> + <artifactId>jets3t</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jdt</groupId> + <artifactId>core</artifactId> + </exclusion> + </exclusions> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + <version>6.1.22</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty-util</artifactId> + <version>6.1.22</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty-client</artifactId> + <version>6.1.22</version> + </dependency> + <dependency> + <groupId>org.hsqldb</groupId> + <artifactId>hsqldb</artifactId> + <version>2.0.0</version> + </dependency> + <dependency> + <groupId>org.jdom</groupId> + <artifactId>jdom</artifactId> + <version>1.1</version> + </dependency> + <dependency> + <groupId>org.gora</groupId> + <artifactId>gora-sql</artifactId> + <version>0.1</version> + <exclusions> + <exclusion> + <groupId>com.sun.jdmk</groupId> + </exclusion> + <exclusion> + <groupId>com.sun.jmx</groupId> + </exclusion> + <exclusion> + <groupId>javax.jms</groupId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project> From commitspaul24@example.com Fri Mar 18 15:06:01 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 45437 invoked from network); 18 Mar 2011 15:06:01 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 18 Mar 2011 15:06:01 -0000 Received: (qmail 48707 invoked by uid 500); 18 Mar 2011 15:06:01 -0000 Delivered-To: paul24@example.com Received: (qmail 48667 invoked by uid 500); 18 Mar 2011 15:06:00 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 48660 invoked by uid 99); 18 Mar 2011 15:06:00 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Mar 2011 15:06:00 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Mar 2011 15:05:58 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 293BA23889ED; Fri, 18 Mar 2011 15:05:35 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1082943 - in /nutch/branches/branch-1.3: CHANGES.txt conf/log4j.properties src/bin/nutch Date: Fri, 18 Mar 2011 15:05:35 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> X-Virus-Checked: Checked by ClamAV on apache.org Author: markus Date: Fri Mar 18 15:05:34 2011 New Revision: 1082943 URL: http://svn.apache.org/viewvc?rev=1082943&view=rev Log: NUTCH-963 Add support for deleting Solr documents with STATUS_DB_GONE in CrawlDB (Claudio Martella, markus) Modified: nutch/branches/branch-1.3/CHANGES.txt nutch/branches/branch-1.3/conf/log4j.properties nutch/branches/branch-1.3/src/bin/nutch Modified: nutch/branches/branch-1.3/CHANGES.txt URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1082943&r1=1082942&r2=1082943&view=diff ============================================================================== --- nutch/branches/branch-1.3/CHANGES.txt (original) +++ nutch/branches/branch-1.3/CHANGES.txt Fri Mar 18 15:05:34 2011 @@ -2,6 +2,8 @@ Nutch Change Log Release 1.3 - Current Development +* NUTCH-963 Add support for deleting Solr documents with STATUS_DB_GONE in CrawlDB (Claudio Martella, markus) + * NUTCH-825 Publish nutch artifacts to central maven repository (mattmann, jnioche) * NUTCH-962 max. redirects not handled correctly: fetcher stops at max-1 redirects (Sebastian Nagel via ab) Modified: nutch/branches/branch-1.3/conf/log4j.properties URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/conf/log4j.properties?rev=1082943&r1=1082942&r2=1082943&view=diff ============================================================================== --- nutch/branches/branch-1.3/conf/log4j.properties (original) +++ nutch/branches/branch-1.3/conf/log4j.properties Fri Mar 18 15:05:34 2011 @@ -24,6 +24,7 @@ log4j.logger.org.apache.nutch.crawl.Link log4j.logger.org.apache.nutch.crawl.LinkDbMerger=INFO,cmdstdout log4j.logger.org.apache.nutch.indexer.solr.SolrIndexer=INFO,cmdstdout log4j.logger.org.apache.nutch.indexer.solr.SolrDeleteDuplicates=INFO,cmdstdout +log4j.logger.org.apache.nutch.indexer.solr.SolrClean=INFO,cmdstdout log4j.logger.org.apache.nutch=INFO log4j.logger.org.apache.hadoop=WARN Modified: nutch/branches/branch-1.3/src/bin/nutch URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/bin/nutch?rev=1082943&r1=1082942&r2=1082943&view=diff ============================================================================== --- nutch/branches/branch-1.3/src/bin/nutch (original) +++ nutch/branches/branch-1.3/src/bin/nutch Fri Mar 18 15:05:34 2011 @@ -49,6 +49,7 @@ if [ $# = 0 ]; then echo " mergelinkdb merge linkdb-s, with optional filtering" echo " solrindex run the solr indexer on parsed segments and linkdb" echo " solrdedup remove duplicates from solr" + echo " solrclean remove HTTP 301 and 404 documents from solr" echo " plugin load a plugin and run one of its classes main()" echo " or" echo " CLASSNAME run the class named CLASSNAME" @@ -210,6 +211,8 @@ elif [ "$COMMAND" = "solrindex" ] ; then CLASS=org.apache.nutch.indexer.solr.SolrIndexer elif [ "$COMMAND" = "solrdedup" ] ; then CLASS=org.apache.nutch.indexer.solr.SolrDeleteDuplicates +elif [ "$COMMAND" = "solrclean" ] ; then + CLASS=org.apache.nutch.indexer.solr.SolrClean elif [ "$COMMAND" = "plugin" ] ; then CLASS=org.apache.nutch.plugin.PluginRepository else From commitspaul24@example.com Fri Mar 18 15:07:38 2011 Return-Path: <commitspaul24@example.com> Delivered-To: paul24@example.com Received: (qmail 48633 invoked from network); 18 Mar 2011 15:07:38 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 18 Mar 2011 15:07:38 -0000 Received: (qmail 49998 invoked by uid 500); 18 Mar 2011 15:07:38 -0000 Delivered-To: paul24@example.com Received: (qmail 49966 invoked by uid 500); 18 Mar 2011 15:07:38 -0000 Mailing-List: contact paul24@example.com; run by ezmlm Precedence: bulk List-Help: <mailto:paul24@example.com> List-Unsubscribe: <mailto:paul24@example.com> List-Post: <mailto:paul24@example.com> List-Id: <commits.nutch.apache.org> Reply-To: paul24@example.com Delivered-To: mailing list paul24@example.com Received: (qmail 49959 invoked by uid 99); 18 Mar 2011 15:07:38 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Mar 2011 15:07:38 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Mar 2011 15:07:37 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 307CC23889ED; Fri, 18 Mar 2011 15:07:15 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1082944 - /nutch/branches/branch-1.3/src/java/org/apache/nutch/indexer/solr/SolrClean.java Date: Fri, 18 Mar 2011 15:07:15 -0000 To: paul24@example.com From: paul24@example.com X-Mailer: svnmailer-1.0.8 Message-Id: <paul24@example.com> Author: markus Date: Fri Mar 18 15:07:14 2011 New Revision: 1082944 URL: http://svn.apache.org/viewvc?rev=1082944&view=rev Log: NUTCH-963 SolrClean is now added Added: nutch/branches/branch-1.3/src/java/org/apache/nutch/indexer/solr/SolrClean.java Added: nutch/branches/branch-1.3/src/java/org/apache/nutch/indexer/solr/SolrClean.java URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/java/org/apache/nutch/indexer/solr/SolrClean.java?rev=1082944&view=auto ============================================================================== --- nutch/branches/branch-1.3/src/java/org/apache/nutch/indexer/solr/SolrClean.java (added) +++ nutch/branches/branch-1.3/src/java/org/apache/nutch/indexer/solr/SolrClean.java Fri Mar 18 15:07:14 2011 @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.indexer.solr; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.text.SimpleDateFormat; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.ByteWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reducer; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.mapred.lib.NullOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.CrawlDb; +import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.NutchJob; +import org.apache.nutch.util.TimingUtil; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; +import org.apache.solr.client.solrj.request.UpdateRequest; +
{'source': 'Apache Software Foundation Public Mail Archives'}