spark_sql.py

from pyspark.sql import SparkSession
from pyspark.sql import Row

# Create a SparkSession
spark = SparkSession.builder.appName('SparkSQL').getOrCreate()


def mapper(line):
    fields = line.split(',')
    return Row(id=int(fields[0]), name=str(fields[1].encode('utf-8')),
               age=int(fields[2]), num_friends=int(fields[3]))


lines = spark.sparkContext.textFile('fakefriends.csv')
people = lines.map(mapper)

# Infer the schema, and register the DataFrame as a table.
schema_people = spark.createDataFrame(people).cache()
schema_people.createOrReplaceTempView('people')

# SQL can be run over DataFrames that have been registered as a table.
teenagers = spark.sql('SELECT * FROM people WHERE age >= 13 AND age <= 19')

# The results of SQL queries are RDDs and support all the normal RDD operations.
for teen in teenagers.collect():
    print(teen)

# We can also use functions instead of SQL queries:
schema_people.groupBy('age').count().orderBy('age').show()

spark.stop()