<NME> run-all-examples
<BEF> #!/usr/bin/env bash
# This script is used to run all of the examples. Mostly to be used by travis for testing
# Output the commands we run
set -x
# If any command fails, fail
set -e
# Build everything
./sbt/sbt compile package assembly > sbtlog || (echo "sbt failed" && cat ./sbtlog && exit 1)
KAFKA_ROOT=./kafka_2.9.2-0.8.1.1
SPARK_SUBMIT_SCRIPT=$SPARK_HOME/bin/spark-submit
ASSEMBLY_JAR=./target/scala-2.10/learning-spark-examples-assembly-0.0.1.jar
# Mini cleanup
rm -rf /tmp/py; mkdir -p /tmp/py
rm -rf /tmp/java; mkdir -p /tmp/java
rm -rf /tmp/scala; mkdir -p /tmp/scala
# setup cassandra
# cqlsh --file ./files/cqlsh_setup &
# Scala
echo "Running Scala programs"
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.LoadJsonWithSparkSQL $ASSEMBLY_JAR local ./files/pandainfo.json 
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.ChapterSixExample  $ASSEMBLY_JAR local ./files/callsigns ./files/callsigns /tmp/scala/ch6out
TWITTER_DATA=./files/testweet.json
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.SparkSQLTwitter  $ASSEMBLY_JAR  "$TWITTER_DATA"  /tmp/scala/tweetout
#$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.BasicQueryCassandra $ASSEMBLY_JAR local localhost
echo "Running Scala streaming program"
./bin/fakelogs.sh &
sleep 1
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]
echo "Running Scala Kafka streaming example"
$SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-readers pandas 1
echo "panda\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas
# Python
echo "Running Python programs"
$SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local
$SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.FlumeInput $ASSEMBLY_JAR localhost 7788 &
FLUME_PID=$!
sleep 1
echo "panda\nerror panda\n" | nc localhost 44444
sleep 3
echo "panda2\nerror panda2\n" | nc localhost 44444
wait $FLUME_PID
# Python
echo "Running Python programs"
$SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local
$SPARK_SUBMIT_SCRIPT ./src/python/BasicAvg.py local
$SPARK_SUBMIT_SCRIPT ./src/python/BasicFilterMap.py local
$SPARK_SUBMIT_SCRIPT ./src/python/BasicKeyValueMapFilter.py local
$SPARK_SUBMIT_SCRIPT ./src/python/BasicMapPartitions.py local
$SPARK_SUBMIT_SCRIPT ./src/python/BasicMap.py local
$SPARK_SUBMIT_SCRIPT ./src/python/ChapterSixExample.py local ./files/callsigns /tmp/py/pandaout
$SPARK_SUBMIT_SCRIPT ./src/python/SparkSQLTwitter.py ./files/testweet.json /tmp/py/tweetout
$SPARK_SUBMIT_SCRIPT ./src/python/LoadCsv.py local ./files/favourite_animals.csv /tmp/py/panda_lovers.csv
$SPARK_SUBMIT_SCRIPT ./src/python/MakeHiveTable.py local ./files/int_string.csv pandaplural
# Temporarily disabled due to API changes
#$SPARK_SUBMIT_SCRIPT ./src/python/LoadHive.py local pandaplural
$SPARK_SUBMIT_SCRIPT ./src/python/LoadJson.py local ./files/pandainfo.json /tmp/py/loadjsonout
$SPARK_SUBMIT_SCRIPT ./src/python/PerKeyAvg.py local
sleep 1
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR 
echo "panda\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas

echo "Done running all programs :)"
echo "Running Java programs"
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.LoadJsonWithSparkSQL $ASSEMBLY_JAR local ./files/pandainfo.json
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.ChapterSixExample $ASSEMBLY_JAR local ./files/callsigns ./files/callsigns /tmp/java/ch6out
./sbt/sbt assembly && $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.SparkSQLTwitter  $ASSEMBLY_JAR  ./files/testweet.json  /tmp/java/tweetout
#$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.BasicQueryCassandra $ASSEMBLY_JAR local localhost
echo "Running Java streaming program"
./bin/fakelogs.sh &
sleep 1
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]
sleep 5
echo "Running Java Kafka streaming example"
$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-java-readers
echo "panda\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas

echo "Done running all programs :)"

<MSG> typos suck

<DFF> @@ -29,7 +29,7 @@ sleep 1
 $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]
 echo "Running Scala Kafka streaming example"
 $SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-readers pandas 1
-echo "panda\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas
+echo "panda\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas
 # Python
 echo "Running Python programs"
 $SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local
@@ -59,6 +59,6 @@ echo "Running Java streaming program"
 sleep 1
 $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]
 $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR 
-echo "panda\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas
+echo "panda\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas
 
 echo "Done running all programs :)"

{'commit': 'diff --git a/run-all-examples b/run-all-examples\nindex 2f4cf7a..8e91d82 100755\n--- a/run-all-examples\n+++ b/run-all-examples\n@@ -29,7 +29,7 @@ sleep 1\n $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]\n echo "Running Scala Kafka streaming example"\n $SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-readers pandas 1\n-echo "panda\\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n+echo "panda\\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n # Python\n echo "Running Python programs"\n $SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local\n@@ -59,6 +59,6 @@ echo "Running Java streaming program"\n sleep 1\n $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]\n $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR \n-echo "panda\\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n+echo "panda\\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n \n echo "Done running all programs :)"\n', 'commit_subject': 'typos suck', 'repo_name': 'databricks/learning-spark', 'committer': {'name': 'Holden Karau', 'email': '6be69bb5cf6cb8dd624e7f174d5aba3c49eb4343@pigscanfly.ca', 'time_sec': '1412202093', 'tz_offset': '-420', 'date': {'seconds': '1412202093'}}, 'commit_tree': '566d7c4d68da3cf8a16eade555536a3c6cc01dcd', 'commit_message': 'typos suck\n', 'language_name': 'Python', 'license': 'mit', 'hunks': '@@ -29,7 +29,7 @@ sleep 1\n $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]\n echo "Running Scala Kafka streaming example"\n $SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-readers pandas 1\n-echo "panda\\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n+echo "panda\\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n # Python\n echo "Running Python programs"\n $SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local\n@@ -59,6 +59,6 @@ echo "Running Java streaming program"\n sleep 1\n $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]\n $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR \n-echo "panda\\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n+echo "panda\\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n \n echo "Done running all programs :)"\n', 'addition_count': 2, 'deletion_count': 2, 'src_file': 'a/run-all-examples', 'tgt_file': 'b/run-all-examples', 'file_extension': '', 'before_file': '#!/usr/bin/env bash\n# This script is used to run all of the examples. Mostly to be used by travis for testing\n# Output the commands we run\nset -x\n# If any command fails, fail\nset -e\n# Build everything\n./sbt/sbt compile package assembly > sbtlog || (echo "sbt failed" && cat ./sbtlog && exit 1)\nKAFKA_ROOT=./kafka_2.9.2-0.8.1.1\nSPARK_SUBMIT_SCRIPT=$SPARK_HOME/bin/spark-submit\nASSEMBLY_JAR=./target/scala-2.10/learning-spark-examples-assembly-0.0.1.jar\n# Mini cleanup\nrm -rf /tmp/py; mkdir -p /tmp/py\nrm -rf /tmp/java; mkdir -p /tmp/java\nrm -rf /tmp/scala; mkdir -p /tmp/scala\n# setup cassandra\n# cqlsh --file ./files/cqlsh_setup &\n# Scala\necho "Running Scala programs"\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.LoadJsonWithSparkSQL $ASSEMBLY_JAR local ./files/pandainfo.json \n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.ChapterSixExample  $ASSEMBLY_JAR local ./files/callsigns ./files/callsigns /tmp/scala/ch6out\nTWITTER_DATA=./files/testweet.json\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.SparkSQLTwitter  $ASSEMBLY_JAR  "$TWITTER_DATA"  /tmp/scala/tweetout\n#$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.BasicQueryCassandra $ASSEMBLY_JAR local localhost\necho "Running Scala streaming program"\n./bin/fakelogs.sh &\nsleep 1\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.scala.StreamingLogInput $ASSEMBLY_JAR local[4]\necho "Running Scala Kafka streaming example"\n$SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-readers pandas 1\necho "panda\\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n# Python\necho "Running Python programs"\n$SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local\n$SPARK_SUBMIT_SCRIPT --master local[4] --class com.oreilly.learningsparkexamples.scala.FlumeInput $ASSEMBLY_JAR localhost 7788 &\nFLUME_PID=$!\nsleep 1\necho "panda\\nerror panda\\n" | nc localhost 44444\nsleep 3\necho "panda2\\nerror panda2\\n" | nc localhost 44444\nwait $FLUME_PID\n# Python\necho "Running Python programs"\n$SPARK_SUBMIT_SCRIPT ./src/python/AvgMapPartitions.py local\n$SPARK_SUBMIT_SCRIPT ./src/python/BasicAvg.py local\n$SPARK_SUBMIT_SCRIPT ./src/python/BasicFilterMap.py local\n$SPARK_SUBMIT_SCRIPT ./src/python/BasicKeyValueMapFilter.py local\n$SPARK_SUBMIT_SCRIPT ./src/python/BasicMapPartitions.py local\n$SPARK_SUBMIT_SCRIPT ./src/python/BasicMap.py local\n$SPARK_SUBMIT_SCRIPT ./src/python/ChapterSixExample.py local ./files/callsigns /tmp/py/pandaout\n$SPARK_SUBMIT_SCRIPT ./src/python/SparkSQLTwitter.py ./files/testweet.json /tmp/py/tweetout\n$SPARK_SUBMIT_SCRIPT ./src/python/LoadCsv.py local ./files/favourite_animals.csv /tmp/py/panda_lovers.csv\n$SPARK_SUBMIT_SCRIPT ./src/python/MakeHiveTable.py local ./files/int_string.csv pandaplural\n# Temporarily disabled due to API changes\n#$SPARK_SUBMIT_SCRIPT ./src/python/LoadHive.py local pandaplural\n$SPARK_SUBMIT_SCRIPT ./src/python/LoadJson.py local ./files/pandainfo.json /tmp/py/loadjsonout\n$SPARK_SUBMIT_SCRIPT ./src/python/PerKeyAvg.py local\nsleep 1\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR \necho "panda\\nerror panda" | $KAFAK_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n\necho "Done running all programs :)"\necho "Running Java programs"\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.LoadJsonWithSparkSQL $ASSEMBLY_JAR local ./files/pandainfo.json\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.ChapterSixExample $ASSEMBLY_JAR local ./files/callsigns ./files/callsigns /tmp/java/ch6out\n./sbt/sbt assembly && $SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.SparkSQLTwitter  $ASSEMBLY_JAR  ./files/testweet.json  /tmp/java/tweetout\n#$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.BasicQueryCassandra $ASSEMBLY_JAR local localhost\necho "Running Java streaming program"\n./bin/fakelogs.sh &\nsleep 1\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.StreamingLogInput $ASSEMBLY_JAR local[4]\nsleep 5\necho "Running Java Kafka streaming example"\n$SPARK_SUBMIT_SCRIPT --class com.oreilly.learningsparkexamples.java.KafkaInput $ASSEMBLY_JAR localhost:2181 spark-java-readers\necho "panda\\nerror panda" | $KAFKA_ROOT/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic pandas\n\necho "Done running all programs :)"\n', 'message': 'typos suck\n', 'author_name': 'Holden Karau', 'lang': 'run-all-examples'}