From pig-commitscarterdavid@example.net Mon Aug 03 18:59:42 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 54718 invoked from network); 3 Aug 2009 18:59:42 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 3 Aug 2009 18:59:42 -0000 Received: (qmail 6587 invoked by uid 500); 3 Aug 2009 18:59:47 -0000 Delivered-To: carterdavid@example.net Received: (qmail 6546 invoked by uid 500); 3 Aug 2009 18:59:47 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 6537 invoked by uid 500); 3 Aug 2009 18:59:47 -0000 Delivered-To: carterdavid@example.net Received: (qmail 6533 invoked by uid 99); 3 Aug 2009 18:59:47 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 18:59:47 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 18:59:35 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 03EAF2388872; Mon, 3 Aug 2009 18:59:14 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r800494 - in /hadoop/pig/trunk: ./ conf/ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/impl/ Date: Mon, 03 Aug 2009 18:59:13 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: daijy Date: Mon Aug 3 18:59:13 2009 New Revision: 800494 URL: http://svn.apache.org/viewvc?rev=800494&view=rev Log: PIG-882: log level not propogated to loggers Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/conf/log4j.properties hadoop/pig/trunk/src/org/apache/pig/Main.java hadoop/pig/trunk/src/org/apache/pig/PigServer.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Mon Aug 3 18:59:13 2009 @@ -44,6 +44,8 @@ BUG FIXES + PIG-882: log level not propogated to loggers (daijy) + PIG-880: Order by is borken with complex fields (sms) PIG-773: Empty complex constants (empty bag, empty tuple and empty map) Modified: hadoop/pig/trunk/conf/log4j.properties URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/conf/log4j.properties?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/conf/log4j.properties (original) +++ hadoop/pig/trunk/conf/log4j.properties Mon Aug 3 18:59:13 2009 @@ -1,5 +1,5 @@ # ***** Set root logger level to DEBUG and its only appender to A. -log4j.rootLogger=info, A +log4j.logger.org.apache.pig=info, A # ***** A is set to be a ConsoleAppender. log4j.appender.A=org.apache.log4j.ConsoleAppender Modified: hadoop/pig/trunk/src/org/apache/pig/Main.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/Main.java?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/Main.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/Main.java Mon Aug 3 18:59:13 2009 @@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; import org.apache.pig.ExecType; import org.apache.pig.impl.PigContext; @@ -260,10 +261,11 @@ } } } - // configure logging - configureLog4J(properties); // create the context with the parameter PigContext pigContext = new PigContext(execType, properties); + + // configure logging + configureLog4J(properties, pigContext); if(logFileName == null && !userSpecifiedLog) { logFileName = validateLogFile(null, null); @@ -445,7 +447,7 @@ } //TODO jz: log4j.properties should be used instead -private static void configureLog4J(Properties properties) { +private static void configureLog4J(Properties properties, PigContext pigContext) { // TODO Add a file appender for the logs // TODO Need to create a property in the properties file for it. // sgroschupf, 25Feb2008: this method will be obsolete with PIG-115. @@ -460,35 +462,43 @@ logLevel = Level.toLevel(logLevelString, Level.INFO); } + Properties props = new Properties(); if (log4jconf != null) { - PropertyConfigurator.configure(log4jconf); - } else if (!brief ) { - // non-brief logging - timestamps - Properties props = new Properties(); - props.setProperty("log4j.rootLogger", "INFO, PIGCONSOLE"); - props.setProperty("log4j.appender.PIGCONSOLE", - "org.apache.log4j.ConsoleAppender"); - props.setProperty("log4j.appender.PIGCONSOLE.layout", - "org.apache.log4j.PatternLayout"); - props.setProperty("log4j.appender.PIGCONSOLE.layout.ConversionPattern", - "%d [%t] %-5p %c - %m%n"); - props.setProperty("log4j.appender.PIGCONSOLE.target", - "System.err"); - PropertyConfigurator.configure(props); - } else { - // brief logging - no timestamps - Properties props = new Properties(); - props.setProperty("log4j.rootLogger", "INFO, PIGCONSOLE"); - props.setProperty("log4j.appender.PIGCONSOLE", - "org.apache.log4j.ConsoleAppender"); - props.setProperty("log4j.appender.PIGCONSOLE.layout", - "org.apache.log4j.PatternLayout"); - props.setProperty("log4j.appender.PIGCONSOLE.layout.ConversionPattern", - "%m%n"); - props.setProperty("log4j.appender.PIGCONSOLE.target", - "System.err"); - PropertyConfigurator.configure(props); - } + try { + FileReader propertyReader = new FileReader(log4jconf); + props.load(propertyReader); + } + catch (IOException e) + { + System.err.println("Warn: Cannot open log4j properties file, use default"); + } + } + if (props.size() == 0) { + props.setProperty("log4j.logger.org.apache.pig", logLevel.toString()+", PIGCONSOLE"); + props.setProperty("log4j.appender.PIGCONSOLE", + "org.apache.log4j.ConsoleAppender"); + props.setProperty("log4j.appender.PIGCONSOLE.layout", + "org.apache.log4j.PatternLayout"); + props.setProperty("log4j.appender.PIGCONSOLE.target", "System.err"); + + if (!brief) { + // non-brief logging - timestamps + props.setProperty( + "log4j.appender.PIGCONSOLE.layout.ConversionPattern", + "%d [%t] %-5p %c - %m%n"); + } else { + // brief logging - no timestamps + props.setProperty( + "log4j.appender.PIGCONSOLE.layout.ConversionPattern", + "%m%n"); + } + } + PropertyConfigurator.configure(props); + logLevel = Logger.getLogger("org.apache.pig").getLevel(); + Properties backendProps = pigContext.getLog4jProperties(); + backendProps.setProperty("log4j.logger.org.apache.pig.level", logLevel.toString()); + pigContext.setLog4jProperties(backendProps); + pigContext.setDefaultLogLevel(logLevel); } // returns the stream of final pig script to be passed to Grunt Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Mon Aug 3 18:59:13 2009 @@ -36,6 +36,9 @@ import java.util.Properties; import java.util.Set; import java.util.Stack; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.apache.pig.impl.plan.PlanException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -178,11 +181,13 @@ } public void debugOn() { - pigContext.debug = true; + Logger.getLogger("org.apache.pig").setLevel(Level.DEBUG); + pigContext.getLog4jProperties().setProperty("log4j.logger.org.apache.pig", Level.DEBUG.toString()); } public void debugOff() { - pigContext.debug = false; + Logger.getLogger("org.apache.pig").setLevel(pigContext.getDefaultLogLevel()); + pigContext.getLog4jProperties().setProperty("log4j.logger.org.apache.pig", pigContext.getDefaultLogLevel().toString()); } public void setDefaultParallel(int p) { @@ -321,7 +326,7 @@ resourceLocation = urls.nextElement(); } - if (pigContext.debug && urls.hasMoreElements()) { + if (urls.hasMoreElements()) { String logMessage = "Found multiple resources that match " + jarName + ": " + resourceLocation; Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java Mon Aug 3 18:59:13 2009 @@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; +import org.apache.log4j.PropertyConfigurator; import org.apache.pig.PigException; import org.apache.pig.backend.executionengine.ExecException; @@ -96,6 +97,10 @@ sJobConf = jConf; try { PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(jConf.get("udf.import.list"))); + pigContext = (PigContext)ObjectSerializer.deserialize(jConf.get("pig.pigContext")); + if (pigContext.getLog4jProperties()!=null) + PropertyConfigurator.configure(pigContext.getLog4jProperties()); + cp = (PhysicalPlan) ObjectSerializer.deserialize(jConf .get("pig.combinePlan")); pack = (POPackage)ObjectSerializer.deserialize(jConf.get("pig.combine.package")); @@ -118,9 +123,6 @@ roots = cp.getRoots().toArray(new PhysicalOperator[1]); leaf = cp.getLeaves().get(0); } - - pigContext = (PigContext)ObjectSerializer.deserialize(jConf.get("pig.pigContext")); - } catch (IOException ioe) { String msg = "Problem while configuring combiner's reduce plan."; throw new RuntimeException(msg, ioe); Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java Mon Aug 3 18:59:13 2009 @@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; +import org.apache.log4j.PropertyConfigurator; import org.apache.pig.PigException; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.data.TargetedTuple; @@ -147,6 +148,10 @@ PigMapReduce.sJobConf = job; try { PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list"))); + pigContext = (PigContext)ObjectSerializer.deserialize(job.get("pig.pigContext")); + if (pigContext.getLog4jProperties()!=null) + PropertyConfigurator.configure(pigContext.getLog4jProperties()); + mp = (PhysicalPlan) ObjectSerializer.deserialize( job.get("pig.mapPlan")); stores = PlanHelper.getStores(mp); @@ -176,7 +181,7 @@ leaf = mp.getLeaves().get(0); } - pigContext = (PigContext)ObjectSerializer.deserialize(job.get("pig.pigContext")); + } catch (IOException ioe) { String msg = "Problem while configuring map plan."; Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java Mon Aug 3 18:59:13 2009 @@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; +import org.apache.log4j.PropertyConfigurator; import org.apache.pig.PigException; import org.apache.pig.backend.executionengine.ExecException; @@ -276,6 +277,10 @@ sJobConf = jConf; try { PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(jConf.get("udf.import.list"))); + pigContext = (PigContext)ObjectSerializer.deserialize(jConf.get("pig.pigContext")); + if (pigContext.getLog4jProperties()!=null) + PropertyConfigurator.configure(pigContext.getLog4jProperties()); + rp = (PhysicalPlan) ObjectSerializer.deserialize(jConf .get("pig.reducePlan")); stores = PlanHelper.getStores(rp); @@ -298,9 +303,6 @@ roots = rp.getRoots().toArray(new PhysicalOperator[1]); leaf = rp.getLeaves().get(0); } - - pigContext = (PigContext)ObjectSerializer.deserialize(jConf.get("pig.pigContext")); - } catch (IOException ioe) { String msg = "Problem while configuring reduce plan."; throw new RuntimeException(msg, ioe); Modified: hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=800494&r1=800493&r2=800494&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java Mon Aug 3 18:59:13 2009 @@ -41,6 +41,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Level; import org.apache.pig.FuncSpec; import org.apache.pig.Main; import org.apache.pig.ExecType; @@ -115,7 +116,9 @@ private static ArrayList<String> packageImportList = new ArrayList<String>(); - public boolean debug = true; + private Properties log4jProperties = new Properties(); + + private Level defaultLogLevel; public int defaultParallel = -1; @@ -639,4 +642,20 @@ { packageImportList = list; } + public void setLog4jProperties(Properties p) + { + log4jProperties = p; + } + public Properties getLog4jProperties() + { + return log4jProperties; + } + public Level getDefaultLogLevel() + { + return defaultLogLevel; + } + public void setDefaultLogLevel(Level l) + { + defaultLogLevel = l; + } } From pig-commitscarterdavid@example.net Mon Aug 03 21:08:47 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 1633 invoked from network); 3 Aug 2009 21:08:47 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 3 Aug 2009 21:08:47 -0000 Received: (qmail 78043 invoked by uid 500); 3 Aug 2009 21:08:52 -0000 Delivered-To: carterdavid@example.net Received: (qmail 78016 invoked by uid 500); 3 Aug 2009 21:08:52 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 78007 invoked by uid 500); 3 Aug 2009 21:08:52 -0000 Delivered-To: carterdavid@example.net Received: (qmail 78004 invoked by uid 99); 3 Aug 2009 21:08:52 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 21:08:52 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eos.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 21:08:50 +0000 Received: from eos.apache.org (localhost [83.161.120.155]) by eos.apache.org (Postfix) with ESMTP id 14D2911154 for <carterdavid@example.net>; Mon, 3 Aug 2009 21:08:30 +0000 (GMT) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Apache Wiki <carterdavid@example.net> To: carterdavid@example.net Date: Mon, 03 Aug 2009 21:08:29 -0000 Message-ID: <carterdavid@example.net> Subject: [Pig Wiki] Update of "PigPerformance" by yinghe X-Virus-Checked: Checked by ClamAV on apache.org Dear Wiki user, You have subscribed to a wiki page or wiki category on "Pig Wiki" for change notification. The following page has been changed by yinghe: http://wiki.apache.org/pig/PigPerformance ------------------------------------------------------------------------------ Note that last 4 test cases produce exactly the same data and so their timing can be directly compared. + + See DataGenerator for details on testing data generation + From pig-commitscarterdavid@example.net Mon Aug 03 21:17:57 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 5190 invoked from network); 3 Aug 2009 21:17:56 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 3 Aug 2009 21:17:56 -0000 Received: (qmail 88911 invoked by uid 500); 3 Aug 2009 21:18:02 -0000 Delivered-To: carterdavid@example.net Received: (qmail 88883 invoked by uid 500); 3 Aug 2009 21:18:01 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 88874 invoked by uid 500); 3 Aug 2009 21:18:01 -0000 Delivered-To: carterdavid@example.net Received: (qmail 88871 invoked by uid 99); 3 Aug 2009 21:18:01 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 21:18:01 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eos.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 21:18:00 +0000 Received: from eos.apache.org (localhost [83.161.120.155]) by eos.apache.org (Postfix) with ESMTP id 9E09A11154 for <carterdavid@example.net>; Mon, 3 Aug 2009 21:17:39 +0000 (GMT) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Apache Wiki <carterdavid@example.net> To: carterdavid@example.net Date: Mon, 03 Aug 2009 21:17:39 -0000 Message-ID: <carterdavid@example.net> Subject: [Pig Wiki] Update of "PigMix" by yinghe X-Virus-Checked: Checked by ClamAV on apache.org Dear Wiki user, You have subscribed to a wiki page or wiki category on "Pig Wiki" for change notification. The following page has been changed by yinghe: http://wiki.apache.org/pig/PigMix ------------------------------------------------------------------------------ == Data Generation == If you want to run this queires yourself, please, see https://issues.apache.org/jira/browse/PIG-200 on how to generate the data. + See DataGeneratorHadoop for information on how to run data generator in hadoop mode. From pig-commitscarterdavid@example.net Mon Aug 03 21:25:01 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 8592 invoked from network); 3 Aug 2009 21:25:00 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 3 Aug 2009 21:25:00 -0000 Received: (qmail 8619 invoked by uid 500); 3 Aug 2009 21:25:06 -0000 Delivered-To: carterdavid@example.net Received: (qmail 8600 invoked by uid 500); 3 Aug 2009 21:25:06 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 8591 invoked by uid 500); 3 Aug 2009 21:25:06 -0000 Delivered-To: carterdavid@example.net Received: (qmail 8588 invoked by uid 99); 3 Aug 2009 21:25:06 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 21:25:06 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eos.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Aug 2009 21:25:02 +0000 Received: from eos.apache.org (localhost [83.161.120.155]) by eos.apache.org (Postfix) with ESMTP id 7DB0011154 for <carterdavid@example.net>; Mon, 3 Aug 2009 21:24:41 +0000 (GMT) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Apache Wiki <carterdavid@example.net> To: carterdavid@example.net Date: Mon, 03 Aug 2009 21:24:41 -0000 Message-ID: <carterdavid@example.net> Subject: [Pig Wiki] Update of "DataGeneratorHadoop" by yinghe X-Virus-Checked: Checked by ClamAV on apache.org Dear Wiki user, You have subscribed to a wiki page or wiki category on "Pig Wiki" for change notification. The following page has been changed by yinghe: http://wiki.apache.org/pig/DataGeneratorHadoop New page: == Make DataGenerator A Hadoop Job == === Introduction === The current data generator runs on a single box and is single threaded. Its execution time is linear to the amount of data to be generated. When the amount of data reaches hundreds of gigabytes, the time required becomes unacceptable. In other words, this application is not scalable to deal with large amount of data. The goal is to be able to generate data in parallel, so the time can be greatly reduced. === Algorithm === Tuples generated by data generator can contain fields that are uniformly distributed or Zipf distributed. Both types of fields can be split into multiple processors with each processor generating a fraction of total rows. If M rows are to be generated by N processors, then each processor shall generate M/N rows. When the data from each processor are combined together, the result should still be uniformly distributed or zipf distributed. * Uniform distributed fields: A random number is generated using Java Random class with specified cardinality. The random numbers are uniformly distributed. For integer and long types, this random number is returned as the value of the tuple field. If N processors use the same cardinality, the combined result is still uniformly distributed. For float, double, and string, this random number is used as a seed to generate corresponding float/double/string. We need to make sure for the same seed, the same float/double/string is returned. When running across N processors, this can be achieved by generating a mapping between random number to actual float/double/string in advance. Then each processor loads this information during startup and uses this data mapping to generate float/double/string fields. * Zipf distributed fields: A 3rd party library is used to generate random numbers between 1 and &quot;cardinality&quot; following zipf distributed. Given a cardinality, this library generates numbers with fixed density. For integer and long types, this number is returned as field value. Therefore, combining data from multiple processors together, the result should have the same density distribution. For float, double and string types, this number is used as a seed to return a float/double/string. We need to make sure for the same number, the same float/double/string is returned across multiple processors. This can be achieved by generating a mapping between random number to actual float/double/string in advance. Then each processor loads this information during startup and uses this data mapping to generate string fields. === Design === Data generator is modified to be a hadoop job. * Command line change * An option -m is added to specify the number of mappers to run this job. Reducer is not required. * Option -f is required to specify the output directory. * When DataGenerator is running in hadoop mode, -e (for seed) is disable. Because multiple mappers are running to generate data, if they share the same seed, the data generated by multiple mappers would be duplicated. * The sequence of execution * Data generator pre-generates a mapping file for each string/double/float field for zipf or uniform distribution. * Create a config file which contains data type, length, cardinality, distribution type, and percentage of NULLs for each field. Attach the name of the mapping file at the end if created from step 1. The name of this config file is passed to each mapper through JobConf. * If input file is not configured, then * Create N input files for N mappers. Each input file only has one row. It contains the number of rows to be generated by a mapper. * Mark the job that there is not input file * If input file is configured, then * Set the input file as input path. * Mark the job that there is input file * Start map-reduce job, and load in field config. For the fields that have mapping file associated with it, build an internal hash for lookups. When mapper gets the input tuple, depending on input type: * If there is no input file, the tuple that mapper receives is the number of rows to be generated. Therefore, it generates the specified number of rows. * If there is an input file, the tuple that mapper receives is an tuple from input file, append it with other fields. === Future Works === This implementation is constrained by the memory availability. For now, we assume the cardinality of a field that need a mapping file is less than 2M, and the number of such fields is not more than 5. In this case, the memory required should be less than 1G for most settings. To work with bigger cardinality or more of string fields, the DataGenerator has to generate data with random numbers and then does an explicit join between the mapping file and the data file. From pig-commitscarterdavid@example.net Tue Aug 04 17:12:19 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 50669 invoked from network); 4 Aug 2009 17:12:19 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 4 Aug 2009 17:12:19 -0000 Received: (qmail 15163 invoked by uid 500); 4 Aug 2009 17:12:24 -0000 Delivered-To: carterdavid@example.net Received: (qmail 15124 invoked by uid 500); 4 Aug 2009 17:12:24 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 15115 invoked by uid 500); 4 Aug 2009 17:12:24 -0000 Delivered-To: carterdavid@example.net Received: (qmail 15112 invoked by uid 99); 4 Aug 2009 17:12:24 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 17:12:24 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 17:12:22 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id E336B238886D; Tue, 4 Aug 2009 17:12:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r800877 - in /hadoop/pig/branches/branch-0.3: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ Date: Tue, 04 Aug 2009 17:12:01 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: pradeepkth Date: Tue Aug 4 17:12:01 2009 New Revision: 800877 URL: http://svn.apache.org/viewvc?rev=800877&view=rev Log: PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to serialized PigContext (pradeepkth) Modified: hadoop/pig/branches/branch-0.3/CHANGES.txt hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java Modified: hadoop/pig/branches/branch-0.3/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/CHANGES.txt?rev=800877&r1=800876&r2=800877&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/CHANGES.txt (original) +++ hadoop/pig/branches/branch-0.3/CHANGES.txt Tue Aug 4 17:12:01 2009 @@ -22,6 +22,9 @@ BUG FIXES +PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to +serialized PigContext (pradeepkth) + PIG-878: Pig is returning too many blocks in the input split. (arunc via gates). Release 0.3.0 - 06/16/09 Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java?rev=800877&r1=800876&r2=800877&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java Tue Aug 4 17:12:01 2009 @@ -18,10 +18,12 @@ package org.apache.pig; +import java.io.Serializable; + /** * The type of query execution */ -public enum ExecType { +public enum ExecType implements Serializable { /** * Run everything on the local machine */ Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=800877&r1=800876&r2=800877&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java Tue Aug 4 17:12:01 2009 @@ -233,7 +233,7 @@ } Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName()); for (Slice split : pigs) { - splits.add(new SliceWrapper(split, pigContext, i, fs, inpTargets.get(i))); + splits.add(new SliceWrapper(split, pigContext.getExecType(), i, fs, inpTargets.get(i))); } } catch (ExecException ee) { throw ee; Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java?rev=800877&r1=800876&r2=800877&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java Tue Aug 4 17:12:01 2009 @@ -60,7 +60,7 @@ public class SliceWrapper implements InputSplit { private int index; - private PigContext pigContext; + private ExecType execType; private Slice wrapped; private transient FileSystem fs;// transient so it isn't serialized private transient JobConf lastConf; @@ -70,9 +70,9 @@ // for deserialization } - public SliceWrapper(Slice slice, PigContext context, int index, FileSystem fs, ArrayList<OperatorKey> targetOps) { + public SliceWrapper(Slice slice, ExecType execType, int index, FileSystem fs, ArrayList<OperatorKey> targetOps) { this.wrapped = slice; - this.pigContext = context; + this.execType = execType; this.index = index; this.fs = fs; this.targetOps = targetOps; @@ -122,7 +122,7 @@ DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job)); // if the execution is against Mapred DFS, set // working dir to /user/<userid> - if(pigContext.getExecType() == ExecType.MAPREDUCE) + if(execType == ExecType.MAPREDUCE) store.setActiveContainer(store.asContainer("/user/" + job.getUser())); wrapped.init(store); @@ -165,7 +165,7 @@ } public void readFields(DataInput is) throws IOException { - pigContext = (PigContext) readObject(is); + execType = (ExecType) readObject(is); targetOps = (ArrayList<OperatorKey>) readObject(is); index = is.readInt(); wrapped = (Slice) readObject(is); @@ -192,7 +192,7 @@ } public void write(DataOutput os) throws IOException { - writeObject(pigContext, os); + writeObject(execType, os); writeObject(targetOps, os); os.writeInt(index); writeObject(wrapped, os); From pig-commitscarterdavid@example.net Tue Aug 04 18:41:59 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 99514 invoked from network); 4 Aug 2009 18:41:59 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 4 Aug 2009 18:41:59 -0000 Received: (qmail 50952 invoked by uid 500); 4 Aug 2009 18:42:04 -0000 Delivered-To: carterdavid@example.net Received: (qmail 50929 invoked by uid 500); 4 Aug 2009 18:42:04 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 50920 invoked by uid 500); 4 Aug 2009 18:42:04 -0000 Delivered-To: carterdavid@example.net Received: (qmail 50917 invoked by uid 99); 4 Aug 2009 18:42:04 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 18:42:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eos.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 18:42:02 +0000 Received: from eos.apache.org (localhost [83.161.120.155]) by eos.apache.org (Postfix) with ESMTP id D7ADF11376 for <carterdavid@example.net>; Tue, 4 Aug 2009 18:41:41 +0000 (GMT) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Apache Wiki <carterdavid@example.net> To: carterdavid@example.net Date: Tue, 04 Aug 2009 18:41:41 -0000 Message-ID: <carterdavid@example.net> Subject: [Pig Wiki] Update of "DataGeneratorHadoop" by yinghe X-Virus-Checked: Checked by ClamAV on apache.org Dear Wiki user, You have subscribed to a wiki page or wiki category on "Pig Wiki" for change notification. The following page has been changed by yinghe: http://wiki.apache.org/pig/DataGeneratorHadoop The comment on the change is: ------------------------------------------------------------------------------ - == Make DataGenerator A Hadoop Job == - === Introduction === + === Background === + Data generator is provided to generate tuples with specified number of fields for testing purpose. You can configure the datatype, length, cardinality, distribution and the percentage of NULL values for each field. Data generator generates random values that matches the configuration. Two types of distribution are supported: uniform and zipf distribution. + - The current data generator runs on a single box and is single threaded. Its execution time is linear to the amount of data to be generated. When the amount of data reaches hundreds of gigabytes, the time required becomes unacceptable. In other words, this application is not scalable to deal with large amount of data. The goal is to be able to generate data in parallel, so the time can be greatly reduced. + The current implementation runs on a single box and is single threaded. Its execution time is linear to the amount of data to be generated. When the amount of data reaches hundreds of gigabytes, the time required becomes unacceptable. In other words, this application is not scalable to deal with large amount of data. + + The newer version of implementation allows to generate data in hadoop mode. You can specify the number of mappers, each mapper only needs to generate a fraction of data. This can greatly reduce the execution time. === Algorithm === Tuples generated by data generator can contain fields that are uniformly distributed or Zipf distributed. Both types of fields can be split into multiple processors with each processor generating a fraction of total rows. If M rows are to be generated by N processors, then each processor shall generate M/N rows. When the data from each processor are combined together, the result should still be uniformly distributed or zipf distributed. @@ -33, +36 @@ * If there is no input file, the tuple that mapper receives is the number of rows to be generated. Therefore, it generates the specified number of rows. * If there is an input file, the tuple that mapper receives is an tuple from input file, append it with other fields. + + === Usage === + + Define following env variables: + * $pigjar: pig.jar + * $zipfjar: sdsuLibJKD12.jar + * $datagenjar: jar file that contains DataGenerator class + * $conf_file: hadoop-site.xml for your cluster + + export HADOOP_CLASSPATH=$pigjar:$zipfjar:$datagenjar + + hadoop jar -libjars $zipfjar $datagenjar org.apache.pig.test.utils.datagen.DataGenerator -conf $conf_file [options] colspec... + + * options: + * -m: number of mappers to run concurrently to generate data. If not configured or equal to 0, it runs in local mode. + * -e: seed value for random numbers, can not be configured if -m is greater than 0. Optional for local mode. + * -f: for local mode, output file and optional with default to stdout, for hadoop model output directory and required + * -i: optional, input file, lines will be read from. + * -r: number of rows to output, not required if -i is configured + * -s: optional, separator character, default is ^A + + * colspec Format: columntype:average_size:cardinality:distribution_type:percent_null + * columntype: + * i = int + * l = long + * f = float + * d = double + * s = string + * m = map + * bx = bag of x, where x is a columntype + * distribution_type: + * u = uniform + * z = zipf + * average_size: average size for string types + + Examples: + s:20:16000:z:7 specifies a String field whose length is 20, cardinality is 16000, it has zipf distribution and about 7% of NULL values. + + i:1:20:u:0 specifies an Integer field whose cardinality is 20, it has uniform distribution and no NULL values. + + To run it locally, besides using hadoop jar without -m, you can also use: + + java -cp $pigjar:$zipfjar:$datagenjar org.apache.pig.test.utils.datagen.DataGenerator [options] colspec... + + === Future Works === This implementation is constrained by the memory availability. For now, we assume the cardinality of a field that need a mapping file is less than 2M, and the number of such fields is not more than 5. In this case, the memory required should be less than 1G for most settings. To work with bigger cardinality or more of string fields, the DataGenerator has to generate data with random numbers and then does an explicit join between the mapping file and the data file. From pig-commitscarterdavid@example.net Tue Aug 04 20:49:29 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 55171 invoked from network); 4 Aug 2009 20:49:29 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 4 Aug 2009 20:49:29 -0000 Received: (qmail 91026 invoked by uid 500); 4 Aug 2009 20:49:34 -0000 Delivered-To: carterdavid@example.net Received: (qmail 90986 invoked by uid 500); 4 Aug 2009 20:49:34 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 90977 invoked by uid 500); 4 Aug 2009 20:49:34 -0000 Delivered-To: carterdavid@example.net Received: (qmail 90974 invoked by uid 99); 4 Aug 2009 20:49:34 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 20:49:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 20:49:32 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 6408623888ED; Tue, 4 Aug 2009 20:49:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r800963 - in /hadoop/pig/branches/branch-0.3/test: ./ org/apache/pig/test/MiniCluster.java Date: Tue, 04 Aug 2009 20:49:12 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: olga Date: Tue Aug 4 20:49:11 2009 New Revision: 800963 URL: http://svn.apache.org/viewvc?rev=800963&view=rev Log: update to svn external due to hadoop project split Modified: hadoop/pig/branches/branch-0.3/test/ (props changed) hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java Propchange: hadoop/pig/branches/branch-0.3/test/ ------------------------------------------------------------------------------ --- svn:externals (original) +++ svn:externals Tue Aug 4 20:49:11 2009 @@ -1 +1 @@ -bin http://svn.apache.org/repos/asf/hadoop/common/nightly/test-patch +bin http://svn.apache.org/repos/asf/hadoop/nightly/test-patch Modified: hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java?rev=800963&r1=800962&r2=800963&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java (original) +++ hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java Tue Aug 4 20:49:11 2009 @@ -20,7 +20,7 @@ import java.io.*; import java.util.Properties; -import org.apache.hadoop.dfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.conf.Configuration; @@ -68,7 +68,7 @@ m_conf.setInt("mapred.submit.replication", 2); m_conf.set("dfs.datanode.address", "83.161.120.155:0"); m_conf.set("dfs.datanode.http.address", "83.161.120.155:0"); - m_conf.write(new FileOutputStream(conf_file)); + m_conf.writeXml(new FileOutputStream(conf_file)); // Set the system properties needed by Pig System.setProperty("cluster", m_conf.get("mapred.job.tracker")); From pig-commitscarterdavid@example.net Tue Aug 04 23:40:26 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 41958 invoked from network); 4 Aug 2009 23:40:26 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 4 Aug 2009 23:40:26 -0000 Received: (qmail 2818 invoked by uid 500); 4 Aug 2009 23:40:31 -0000 Delivered-To: carterdavid@example.net Received: (qmail 2807 invoked by uid 500); 4 Aug 2009 23:40:31 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 2798 invoked by uid 500); 4 Aug 2009 23:40:31 -0000 Delivered-To: carterdavid@example.net Received: (qmail 2795 invoked by uid 99); 4 Aug 2009 23:40:31 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 23:40:31 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 23:40:29 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id EDD8723888E2; Tue, 4 Aug 2009 23:40:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r801025 - in /hadoop/pig/branches/branch-0.3: ./ bin/ src/org/apache/pig/backend/hadoop/datastorage/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/impl/io/ src/org/apache/pig/tools/pigstats/ test/org/a... Date: Tue, 04 Aug 2009 23:40:08 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: olga Date: Tue Aug 4 23:40:08 2009 New Revision: 801025 URL: http://svn.apache.org/viewvc?rev=801025&view=rev Log: reverted unintentional commit Modified: hadoop/pig/branches/branch-0.3/bin/pig hadoop/pig/branches/branch-0.3/build.xml hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java Modified: hadoop/pig/branches/branch-0.3/bin/pig URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/bin/pig?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/bin/pig (original) +++ hadoop/pig/branches/branch-0.3/bin/pig Tue Aug 4 23:40:08 2009 @@ -34,7 +34,7 @@ # # PIG_ROOT_LOGGER The root appender. Default is INFO,console # -# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 18 (0.18). +# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 20 (0.20). cygwin=false case "`uname`" in @@ -136,7 +136,7 @@ done # Set the version for Hadoop, default to 17 -PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-18}" +PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-20}" # add libs to CLASSPATH. There can be more than one version of the hadoop # libraries in the lib dir, so don't blindly add them all. Only add the one # that matche PIG_HADOOP_VERSION. Modified: hadoop/pig/branches/branch-0.3/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/build.xml?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/build.xml (original) +++ hadoop/pig/branches/branch-0.3/build.xml Tue Aug 4 23:40:08 2009 @@ -47,14 +47,14 @@ <!-- property name="build.encoding" value="ISO-8859-1" / --> <property name="build.encoding" value="UTF8" /> <!-- TODO with only one version of hadoop in the lib folder we do not need that anymore --> - <property name="hadoop.jarfile" value="hadoop18.jar" /> + <property name="hadoop.jarfile" value="hadoop20.jar" /> <property name="hbase.jarfile" value="hbase-0.18.1.jar" /> <property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" /> <!-- javac properties --> <property name="javac.debug" value="on" /> <property name="javac.optimize" value="on" /> - <property name="javac.deprecation" value="on" /> + <property name="javac.deprecation" value="off" /> <property name="javac.version" value="1.5" /> <property name="javac.args" value="" /> <!-- default warnings option --> @@ -446,6 +446,7 @@ <!-- Excluded under Windows.--> <exclude name="**/TestHBaseStorage.java" if="isWindows" /> <!-- Excluced because we don't want to run them --> + <exclude name="**/TestHBaseStorage.java" /> <exclude name="**/PigExecTestCase.java" /> <exclude name="**/TypeCheckingTestUtil.java" /> <exclude name="**/TypeGraphPrinter.java" /> Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java Tue Aug 4 23:40:08 2009 @@ -23,7 +23,6 @@ import java.util.Enumeration; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.JobConf; import java.util.Properties; Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java Tue Aug 4 23:40:08 2009 @@ -30,7 +30,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.dfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.pig.PigException; import org.apache.pig.backend.datastorage.ContainerDescriptor; Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Tue Aug 4 23:40:08 2009 @@ -30,6 +30,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.JobConf; import org.apache.pig.FuncSpec; import org.apache.pig.PigException; import org.apache.pig.PigWarning; @@ -1323,10 +1324,10 @@ ExecutionEngine eng = pigContext.getExecutionEngine(); if(eng instanceof HExecutionEngine){ try { - val = Math.round(0.9f * ((HExecutionEngine)eng).getJobClient().getDefaultReduces()); + val = ((JobConf)((HExecutionEngine)eng).getJobClient().getConf()).getNumReduceTasks(); if(val<=0) val = 1; - } catch (IOException e) { + } catch (Exception e) { int errCode = 6015; String msg = "Problem getting the default number of reduces from the Job Client."; throw new MRCompilerException(msg, errCode, PigException.REMOTE_ENVIRONMENT, e); Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java Tue Aug 4 23:40:08 2009 @@ -39,6 +39,6 @@ public Object getValueAsPigType() { BytesWritable bw = (BytesWritable)mValue; - return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize()); + return isNull() ? null : new DataByteArray(bw.getBytes(), 0, bw.getLength()); } } Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java Tue Aug 4 23:40:08 2009 @@ -148,13 +148,14 @@ try { counters = rj.getCounters(); Counters.Group taskgroup = counters.getGroup("org.apache.hadoop.mapred.Task$Counter"); - Counters.Group hdfsgroup = counters.getGroup("org.apache.hadoop.mapred.Task$FileSystemCounter"); + Counters.Group hdfsgroup = counters.getGroup("FileSystemCounters"); jobStats.put("PIG_STATS_MAP_INPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("MAP_INPUT_RECORDS").getCounter())).toString()); jobStats.put("PIG_STATS_MAP_OUTPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("MAP_OUTPUT_RECORDS").getCounter())).toString()); jobStats.put("PIG_STATS_REDUCE_INPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("REDUCE_INPUT_RECORDS").getCounter())).toString()); jobStats.put("PIG_STATS_REDUCE_OUTPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("REDUCE_OUTPUT_RECORDS").getCounter())).toString()); - jobStats.put("PIG_STATS_BYTES_WRITTEN", (Long.valueOf(hdfsgroup.getCounterForName("HDFS_WRITE").getCounter())).toString()); + jobStats.put("PIG_STATS_BYTES_WRITTEN", (Long.valueOf(hdfsgroup.getCounterForName("HDFS_BYTES_WRITTEN").getCounter())).toString()); + } catch (IOException e) { // TODO Auto-generated catch block String error = "Unable to get the counters."; Modified: hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java?rev=801025&r1=801024&r2=801025&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java (original) +++ hadoop/pig/branches/branch-0.3/test/org/apache/pig/test/MiniCluster.java Tue Aug 4 23:40:08 2009 @@ -20,7 +20,7 @@ import java.io.*; import java.util.Properties; -import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.dfs.MiniDFSCluster; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.conf.Configuration; @@ -68,7 +68,7 @@ m_conf.setInt("mapred.submit.replication", 2); m_conf.set("dfs.datanode.address", "83.161.120.155:0"); m_conf.set("dfs.datanode.http.address", "83.161.120.155:0"); - m_conf.writeXml(new FileOutputStream(conf_file)); + m_conf.write(new FileOutputStream(conf_file)); // Set the system properties needed by Pig System.setProperty("cluster", m_conf.get("mapred.job.tracker")); From pig-commitscarterdavid@example.net Tue Aug 04 23:56:47 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 48580 invoked from network); 4 Aug 2009 23:56:47 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 4 Aug 2009 23:56:47 -0000 Received: (qmail 19726 invoked by uid 500); 4 Aug 2009 23:56:52 -0000 Delivered-To: carterdavid@example.net Received: (qmail 19707 invoked by uid 500); 4 Aug 2009 23:56:52 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 19698 invoked by uid 500); 4 Aug 2009 23:56:52 -0000 Delivered-To: carterdavid@example.net Received: (qmail 19695 invoked by uid 99); 4 Aug 2009 23:56:52 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 23:56:52 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Aug 2009 23:56:47 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 950352388857; Tue, 4 Aug 2009 23:56:26 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r801031 - in /hadoop/pig/branches/branch-0.3: ./ bin/ src/org/apache/pig/backend/hadoop/datastorage/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/impl/io/ src/org/apache/pig/tools/pigstats/ test/ Date: Tue, 04 Aug 2009 23:56:26 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: pradeepkth Date: Tue Aug 4 23:56:25 2009 New Revision: 801031 URL: http://svn.apache.org/viewvc?rev=801031&view=rev Log: rolling back to revision 800877 to revert accidental changes Modified: hadoop/pig/branches/branch-0.3/bin/pig hadoop/pig/branches/branch-0.3/build.xml hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java hadoop/pig/branches/branch-0.3/test/ (props changed) Modified: hadoop/pig/branches/branch-0.3/bin/pig URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/bin/pig?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/bin/pig (original) +++ hadoop/pig/branches/branch-0.3/bin/pig Tue Aug 4 23:56:25 2009 @@ -34,7 +34,7 @@ # # PIG_ROOT_LOGGER The root appender. Default is INFO,console # -# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 20 (0.20). +# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 18 (0.18). cygwin=false case "`uname`" in @@ -136,7 +136,7 @@ done # Set the version for Hadoop, default to 17 -PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-20}" +PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-18}" # add libs to CLASSPATH. There can be more than one version of the hadoop # libraries in the lib dir, so don't blindly add them all. Only add the one # that matche PIG_HADOOP_VERSION. Modified: hadoop/pig/branches/branch-0.3/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/build.xml?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/build.xml (original) +++ hadoop/pig/branches/branch-0.3/build.xml Tue Aug 4 23:56:25 2009 @@ -47,14 +47,14 @@ <!-- property name="build.encoding" value="ISO-8859-1" / --> <property name="build.encoding" value="UTF8" /> <!-- TODO with only one version of hadoop in the lib folder we do not need that anymore --> - <property name="hadoop.jarfile" value="hadoop20.jar" /> + <property name="hadoop.jarfile" value="hadoop18.jar" /> <property name="hbase.jarfile" value="hbase-0.18.1.jar" /> <property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" /> <!-- javac properties --> <property name="javac.debug" value="on" /> <property name="javac.optimize" value="on" /> - <property name="javac.deprecation" value="off" /> + <property name="javac.deprecation" value="on" /> <property name="javac.version" value="1.5" /> <property name="javac.args" value="" /> <!-- default warnings option --> @@ -446,7 +446,6 @@ <!-- Excluded under Windows.--> <exclude name="**/TestHBaseStorage.java" if="isWindows" /> <!-- Excluced because we don't want to run them --> - <exclude name="**/TestHBaseStorage.java" /> <exclude name="**/PigExecTestCase.java" /> <exclude name="**/TypeCheckingTestUtil.java" /> <exclude name="**/TypeGraphPrinter.java" /> Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java Tue Aug 4 23:56:25 2009 @@ -23,6 +23,7 @@ import java.util.Enumeration; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; import java.util.Properties; Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java Tue Aug 4 23:56:25 2009 @@ -30,7 +30,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.dfs.DistributedFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.pig.PigException; import org.apache.pig.backend.datastorage.ContainerDescriptor; Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Tue Aug 4 23:56:25 2009 @@ -30,7 +30,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.mapred.JobConf; import org.apache.pig.FuncSpec; import org.apache.pig.PigException; import org.apache.pig.PigWarning; @@ -1324,10 +1323,10 @@ ExecutionEngine eng = pigContext.getExecutionEngine(); if(eng instanceof HExecutionEngine){ try { - val = ((JobConf)((HExecutionEngine)eng).getJobClient().getConf()).getNumReduceTasks(); + val = Math.round(0.9f * ((HExecutionEngine)eng).getJobClient().getDefaultReduces()); if(val<=0) val = 1; - } catch (Exception e) { + } catch (IOException e) { int errCode = 6015; String msg = "Problem getting the default number of reduces from the Job Client."; throw new MRCompilerException(msg, errCode, PigException.REMOTE_ENVIRONMENT, e); Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java Tue Aug 4 23:56:25 2009 @@ -39,6 +39,6 @@ public Object getValueAsPigType() { BytesWritable bw = (BytesWritable)mValue; - return isNull() ? null : new DataByteArray(bw.getBytes(), 0, bw.getLength()); + return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize()); } } Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java?rev=801031&r1=801030&r2=801031&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java (original) +++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java Tue Aug 4 23:56:25 2009 @@ -148,14 +148,13 @@ try { counters = rj.getCounters(); Counters.Group taskgroup = counters.getGroup("org.apache.hadoop.mapred.Task$Counter"); - Counters.Group hdfsgroup = counters.getGroup("FileSystemCounters"); + Counters.Group hdfsgroup = counters.getGroup("org.apache.hadoop.mapred.Task$FileSystemCounter"); jobStats.put("PIG_STATS_MAP_INPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("MAP_INPUT_RECORDS").getCounter())).toString()); jobStats.put("PIG_STATS_MAP_OUTPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("MAP_OUTPUT_RECORDS").getCounter())).toString()); jobStats.put("PIG_STATS_REDUCE_INPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("REDUCE_INPUT_RECORDS").getCounter())).toString()); jobStats.put("PIG_STATS_REDUCE_OUTPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("REDUCE_OUTPUT_RECORDS").getCounter())).toString()); - jobStats.put("PIG_STATS_BYTES_WRITTEN", (Long.valueOf(hdfsgroup.getCounterForName("HDFS_BYTES_WRITTEN").getCounter())).toString()); - + jobStats.put("PIG_STATS_BYTES_WRITTEN", (Long.valueOf(hdfsgroup.getCounterForName("HDFS_WRITE").getCounter())).toString()); } catch (IOException e) { // TODO Auto-generated catch block String error = "Unable to get the counters."; Propchange: hadoop/pig/branches/branch-0.3/test/ ------------------------------------------------------------------------------ --- svn:externals (original) +++ svn:externals Tue Aug 4 23:56:25 2009 @@ -1 +1 @@ -bin http://svn.apache.org/repos/asf/hadoop/nightly/test-patch +bin http://svn.apache.org/repos/asf/hadoop/common/nightly/test-patch From pig-commitscarterdavid@example.net Wed Aug 05 00:02:09 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 51501 invoked from network); 5 Aug 2009 00:02:08 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 5 Aug 2009 00:02:08 -0000 Received: (qmail 26969 invoked by uid 500); 5 Aug 2009 00:02:13 -0000 Delivered-To: carterdavid@example.net Received: (qmail 26934 invoked by uid 500); 5 Aug 2009 00:02:13 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 26925 invoked by uid 500); 5 Aug 2009 00:02:13 -0000 Delivered-To: carterdavid@example.net Received: (qmail 26922 invoked by uid 99); 5 Aug 2009 00:02:13 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Aug 2009 00:02:13 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Aug 2009 00:02:11 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id AF6292388872; Wed, 5 Aug 2009 00:01:51 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r801032 - /hadoop/pig/branches/branch-0.3/test/ Date: Wed, 05 Aug 2009 00:01:51 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: olga Date: Wed Aug 5 00:01:51 2009 New Revision: 801032 URL: http://svn.apache.org/viewvc?rev=801032&view=rev Log: property change due to hadoop project split Modified: hadoop/pig/branches/branch-0.3/test/ (props changed) Propchange: hadoop/pig/branches/branch-0.3/test/ ------------------------------------------------------------------------------ --- svn:externals (original) +++ svn:externals Wed Aug 5 00:01:51 2009 @@ -1 +1 @@ -bin http://svn.apache.org/repos/asf/hadoop/common/nightly/test-patch +bin http://svn.apache.org/repos/asf/hadoop/nightly/test-patch From pig-commitscarterdavid@example.net Wed Aug 05 23:18:49 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 89177 invoked from network); 5 Aug 2009 23:18:49 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 5 Aug 2009 23:18:49 -0000 Received: (qmail 64556 invoked by uid 500); 5 Aug 2009 23:18:57 -0000 Delivered-To: carterdavid@example.net Received: (qmail 64511 invoked by uid 500); 5 Aug 2009 23:18:56 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 64502 invoked by uid 500); 5 Aug 2009 23:18:56 -0000 Delivered-To: carterdavid@example.net Received: (qmail 64499 invoked by uid 99); 5 Aug 2009 23:18:56 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Aug 2009 23:18:56 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Aug 2009 23:18:52 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 017B1238887A; Wed, 5 Aug 2009 23:18:31 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r801460 - in /hadoop/pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ test/org/apache/pig/test/ Date: Wed, 05 Aug 2009 23:18:30 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: pradeepkth Date: Wed Aug 5 23:18:30 2009 New Revision: 801460 URL: http://svn.apache.org/viewvc?rev=801460&view=rev Log: PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to serialized PigContext (pradeepkth) Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/ExecType.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=801460&r1=801459&r2=801460&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Wed Aug 5 23:18:30 2009 @@ -43,6 +43,9 @@ PIG-792: skew join implementation (sriranjan via olgan) BUG FIXES + + PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to + serialized PigContext (pradeepkth) PIG-882: log level not propogated to loggers (daijy) Modified: hadoop/pig/trunk/src/org/apache/pig/ExecType.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/ExecType.java?rev=801460&r1=801459&r2=801460&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/ExecType.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/ExecType.java Wed Aug 5 23:18:30 2009 @@ -18,10 +18,12 @@ package org.apache.pig; +import java.io.Serializable; + /** * The type of query execution */ -public enum ExecType { +public enum ExecType implements Serializable { /** * Run everything on the local machine */ Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=801460&r1=801459&r2=801460&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java Wed Aug 5 23:18:30 2009 @@ -234,7 +234,7 @@ } Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName()); for (Slice split : pigs) { - splits.add(new SliceWrapper(split, pigContext, i, fs, inpTargets.get(i))); + splits.add(new SliceWrapper(split, pigContext.getExecType(), i, fs, inpTargets.get(i))); } } catch (ExecException ee) { throw ee; Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java?rev=801460&r1=801459&r2=801460&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java Wed Aug 5 23:18:30 2009 @@ -60,7 +60,7 @@ public class SliceWrapper implements InputSplit { private int index; - private PigContext pigContext; + private ExecType execType; private Slice wrapped; private transient FileSystem fs;// transient so it isn't serialized private transient JobConf lastConf; @@ -70,9 +70,9 @@ // for deserialization } - public SliceWrapper(Slice slice, PigContext context, int index, FileSystem fs, ArrayList<OperatorKey> targetOps) { + public SliceWrapper(Slice slice, ExecType execType, int index, FileSystem fs, ArrayList<OperatorKey> targetOps) { this.wrapped = slice; - this.pigContext = context; + this.execType = execType; this.index = index; this.fs = fs; this.targetOps = targetOps; @@ -122,7 +122,7 @@ DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job)); // if the execution is against Mapred DFS, set // working dir to /user/<userid> - if(pigContext.getExecType() == ExecType.MAPREDUCE) + if(execType == ExecType.MAPREDUCE) store.setActiveContainer(store.asContainer("/user/" + job.getUser())); PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list"))); wrapped.init(store); @@ -166,7 +166,7 @@ } public void readFields(DataInput is) throws IOException { - pigContext = (PigContext) readObject(is); + execType = (ExecType) readObject(is); targetOps = (ArrayList<OperatorKey>) readObject(is); index = is.readInt(); wrapped = (Slice) readObject(is); @@ -193,7 +193,7 @@ } public void write(DataOutput os) throws IOException { - writeObject(pigContext, os); + writeObject(execType, os); writeObject(targetOps, os); os.writeInt(index); writeObject(wrapped, os); Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java?rev=801460&r1=801459&r2=801460&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java Wed Aug 5 23:18:30 2009 @@ -103,6 +103,7 @@ @Test // See PIG-832 public void testImportList() throws Exception { + String FILE_SEPARATOR = System.getProperty("file.separator"); File tmpDir = File.createTempFile("test", ""); tmpDir.delete(); @@ -112,37 +113,51 @@ Util.deleteDirectory(tempDir); File udf1Dir = new File(tmpDir.getAbsolutePath()+FILE_SEPARATOR+"com"+FILE_SEPARATOR+"xxx"+FILE_SEPARATOR+"udf1"); udf1Dir.mkdirs(); - File udf1JavaSrc = new File(udf1Dir.getAbsolutePath()+FILE_SEPARATOR+"TestUDF.java"); + File udf2Dir = new File(tmpDir.getAbsolutePath()+FILE_SEPARATOR+"com"+FILE_SEPARATOR+"xxx"+FILE_SEPARATOR+"udf2"); + udf2Dir.mkdirs(); + File udf1JavaSrc = new File(udf1Dir.getAbsolutePath()+FILE_SEPARATOR+"TestUDF1.java"); + File udf2JavaSrc = new File(udf2Dir.getAbsolutePath()+FILE_SEPARATOR+"TestUDF2.java"); + String udf1Src = new String("package com.xxx.udf1;\n"+ "import java.io.IOException;\n"+ "import org.apache.pig.EvalFunc;\n"+ "import org.apache.pig.data.Tuple;\n"+ - "public class TestUDF extends EvalFunc<Integer>{\n"+ + "public class TestUDF1 extends EvalFunc<Integer>{\n"+ "public Integer exec(Tuple input) throws IOException {\n"+ "return 1;}\n"+ "}"); + String udf2Src = new String("package com.xxx.udf2;\n"+ + "import org.apache.pig.builtin.PigStorage;\n" + + "public class TestUDF2 extends PigStorage { }\n"); + // generate java file - FileOutputStream outStream = + FileOutputStream outStream1 = new FileOutputStream(udf1JavaSrc); - - OutputStreamWriter outWriter = new OutputStreamWriter(outStream); - outWriter.write(udf1Src); - outWriter.close(); + OutputStreamWriter outWriter1 = new OutputStreamWriter(outStream1); + outWriter1.write(udf1Src); + outWriter1.close(); + + FileOutputStream outStream2 = + new FileOutputStream(udf2JavaSrc); + OutputStreamWriter outWriter2 = new OutputStreamWriter(outStream2); + outWriter2.write(udf2Src); + outWriter2.close(); // compile int status; status = Util.executeShellCommand("javac -cp "+System.getProperty("java.class.path") + " " + udf1JavaSrc); - + status = Util.executeShellCommand("javac -cp "+System.getProperty("java.class.path") + " " + udf2JavaSrc); + // generate jar file - String jarName = "TestUDFJar1.jar"; + String jarName = "TestUDFJar.jar"; status = Util.executeShellCommand("jar -cf " + tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName + " -C " + tmpDir.getAbsolutePath() + " " + "com"); assertTrue(status==0); - + PigServer pig = new PigServer(pigContext); pig.registerJar(tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName); - + PigContext.initializeImportList("com.xxx.udf1:com.xxx.udf2."); ArrayList<String> importList = PigContext.getPackageImportList(); assertTrue(importList.size()==5); @@ -152,9 +167,9 @@ assertTrue(importList.get(3).equals("org.apache.pig.builtin.")); assertTrue(importList.get(4).equals("org.apache.pig.impl.builtin.")); - Object udf = PigContext.instantiateFuncFromSpec("TestUDF"); - assertTrue(udf.getClass().toString().endsWith("com.xxx.udf1.TestUDF")); - + Object udf = PigContext.instantiateFuncFromSpec("TestUDF1"); + assertTrue(udf.getClass().toString().endsWith("com.xxx.udf1.TestUDF1")); + int LOOP_COUNT = 40; File tmpFile = File.createTempFile("test", "txt"); PrintStream ps = new PrintStream(new FileOutputStream(tmpFile)); @@ -168,11 +183,10 @@ FileLocalizer.deleteTempFiles(); PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); - pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "' AS (num:chararray);"); - pigServer.registerQuery("B = foreach A generate TestUDF(num);"); + pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "' using TestUDF2() AS (num:chararray);"); + pigServer.registerQuery("B = foreach A generate TestUDF1(num);"); Iterator<Tuple> iter = pigServer.openIterator("B"); if(!iter.hasNext()) fail("No output found"); - int numIdentity = 0; while(iter.hasNext()){ Tuple t = iter.next(); assertTrue(t.get(0) instanceof Integer); From pig-commitscarterdavid@example.net Thu Aug 06 15:18:14 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 25186 invoked from network); 6 Aug 2009 15:18:14 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 6 Aug 2009 15:18:14 -0000 Received: (qmail 45671 invoked by uid 500); 6 Aug 2009 14:59:26 -0000 Delivered-To: carterdavid@example.net Received: (qmail 45666 invoked by uid 500); 6 Aug 2009 14:59:25 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 45651 invoked by uid 500); 6 Aug 2009 14:59:25 -0000 Delivered-To: carterdavid@example.net Received: (qmail 45648 invoked by uid 99); 6 Aug 2009 14:59:25 -0000 Received: from nike.apache.org (HELO nike.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Aug 2009 14:59:25 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eos.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Aug 2009 14:59:22 +0000 Received: from eos.apache.org (localhost [83.161.120.155]) by eos.apache.org (Postfix) with ESMTP id 2AE8911376 for <carterdavid@example.net>; Thu, 6 Aug 2009 14:59:01 +0000 (GMT) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Apache Wiki <carterdavid@example.net> To: carterdavid@example.net Date: Thu, 06 Aug 2009 14:59:00 -0000 Message-ID: <carterdavid@example.net> Subject: [Pig Wiki] Update of "PigErrorHandlingFunctionalSpecification" by AshutoshChauhan X-Virus-Checked: Checked by ClamAV on apache.org Dear Wiki user, You have subscribed to a wiki page or wiki category on "Pig Wiki" for change notification. The following page has been changed by AshutoshChauhan: http://wiki.apache.org/pig/PigErrorHandlingFunctionalSpecification The comment on the change is: Errors related to Merge Join ------------------------------------------------------------------------------ ||1098||Node index cannot be negative.|| ||1099||Node to be replaced cannot be null.|| ||1100||Replacement node cannot be null.|| + ||1101||Merge Join must have exactly two inputs. Found : + <size> + inputs || + ||1102||Error encountered while doing sort merge join. Check if data on right side is sorted. Last two tuples encountered were: \n"+ <previous tuple>+ "\n" + <current tuple>|| ||2000||Internal error. Mismatch in group by arities. Expected: <schema>. Found: <schema>|| ||2001||Unable to clone plan before compiling|| ||2002||The output file(s): <filename> already exists|| From pig-commitscarterdavid@example.net Fri Aug 07 02:19:11 2009 Return-Path: <pig-commitscarterdavid@example.net> Delivered-To: carterdavid@example.net Received: (qmail 93143 invoked from network); 7 Aug 2009 02:19:11 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (83.161.120.155) by minotaur.apache.org with SMTP; 7 Aug 2009 02:19:11 -0000 Received: (qmail 343 invoked by uid 500); 7 Aug 2009 02:19:18 -0000 Delivered-To: carterdavid@example.net Received: (qmail 200 invoked by uid 500); 7 Aug 2009 02:19:18 -0000 Mailing-List: contact carterdavid@example.net; run by ezmlm Precedence: bulk List-Help: <mailto:carterdavid@example.net> List-Unsubscribe: <mailto:carterdavid@example.net> List-Post: <mailto:carterdavid@example.net> List-Id: <pig-commits.hadoop.apache.org> Reply-To: carterdavid@example.net Delivered-To: mailing list carterdavid@example.net Received: (qmail 191 invoked by uid 500); 7 Aug 2009 02:19:18 -0000 Delivered-To: carterdavid@example.net Received: (qmail 188 invoked by uid 99); 7 Aug 2009 02:19:18 -0000 Received: from athena.apache.org (HELO athena.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Aug 2009 02:19:18 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [83.161.120.155] (HELO eris.apache.org) (83.161.120.155) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Aug 2009 02:19:14 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 33D3F2388895; Fri, 7 Aug 2009 02:18:54 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r801863 - in /hadoop/pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/impl/logicalLayer/ src/org/apache/pig/impl/logicalLayer/optimizer/ src/org/apache/pig/impl/logicalLay... Date: Fri, 07 Aug 2009 02:18:53 -0000 To: carterdavid@example.net From: carterdavid@example.net X-Mailer: svnmailer-1.0.8 Message-Id: <carterdavid@example.net> X-Virus-Checked: Checked by ClamAV on apache.org Author: daijy Date: Fri Aug 7 02:18:52 2009 New Revision: 801863 URL: http://svn.apache.org/viewvc?rev=801863&view=rev Log: PIG-697: Proposed improvements to pig's optimizer, Phase5 Removed: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOFRJoin.java Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/PigServer.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/DotLOPrinter.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOJoin.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOPrinter.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/PlanSetter.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ProjectStarTranslator.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ProjectionMapCalculator.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ProjectionMapRemover.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushDownForeachFlatten.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaCalculator.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java hadoop/pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java hadoop/pig/trunk/test/org/apache/pig/test/TestProjectionMap.java hadoop/pig/trunk/test/org/apache/pig/test/TestPushDownForeachFlatten.java hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java hadoop/pig/trunk/test/org/apache/pig/test/TestRequiredFields.java hadoop/pig/trunk/test/org/apache/pig/test/TestRewire.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=801863&r1=801862&r2=801863&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Aug 7 02:18:52 2009 @@ -26,6 +26,8 @@ IMPROVEMENTS +PIG-697: Proposed improvements to pig's optimizer, Phase5 (daijy) + PIG-895: Default parallel for Pig (daijy) PIG-820: Change RandomSampleLoader to take a LoadFunc instead of extending Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=801863&r1=801862&r2=801863&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Fri Aug 7 02:18:52 2009 @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.Collection; -import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -55,17 +54,13 @@ import org.apache.pig.impl.PigContext; import org.apache.pig.impl.io.FileLocalizer; import
{'source': 'Apache Software Foundation Public Mail Archives'}