from pyspark.sql import SQLContext, Row

lines = sc.textFile("/user/admin/Wikipedia/*")
tokens = lines.map(lambda l: l.split("\t"))
data = tokens.map(lambda t: Row(year=int(t[0]), month=int(t[1]), day=int(t[2]), hour=int(t[3]), page=t[4], hits=int(t[5])))


sqlContext = SQLContext(sc)
wtDataFrame = sqlContext.createDataFrame(data)
wtDataFrame.registerTempTable("wt")

hitCountsRDD = sqlContext.sql("SELECT hits, COUNT(*) AS c FROM wt GROUP BY hits ORDER BY hits").cache()
hitCounts = hitCountsRDD.collect()

from pyspark.sql import SQLContext, Row

lines = sc.textFile("/user/admin/Wikipedia/*")
def parse_line(line):
    tokens = line.split('\t')
    return Row(page=tokens[4], hits=int(tokens[5]))
data = lines.map(parse_line)

sqlContext = SQLContext(sc)
wtDataFrame = sqlContext.createDataFrame(data)
wtDataFrame.registerTempTable("wt")

hitCountsRDD = sqlContext.sql("SELECT hits, COUNT(*) AS c FROM wt GROUP BY hits ORDER BY hits").cache()
hitCounts = hitCountsRDD.collect()

Use parse_line function like in later sections
Use parse_line function like in later sections
