![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# **Split the Data into Training and Testing Sets**| \n", + " | Week of pending | \n", + "Mean days listing to pending | \n", + "Mean price reduction percentage | \n", + "
|---|---|---|---|
| 0 | \n", + "1517011200000000000 | \n", + "49 | \n", + "0.045619 | \n", + "
| 1 | \n", + "1517616000000000000 | \n", + "48 | \n", + "0.046532 | \n", + "
| 2 | \n", + "1518220800000000000 | \n", + "47 | \n", + "0.048536 | \n", + "
| 3 | \n", + "1518825600000000000 | \n", + "46 | \n", + "0.049169 | \n", + "
| 4 | \n", + "1519430400000000000 | \n", + "43 | \n", + "0.048168 | \n", + "
| \n", + " | Week of pending | \n", + "Mean days listing to pending | \n", + "
|---|---|---|
| 0 | \n", + "1517011200000000000 | \n", + "49 | \n", + "
| 1 | \n", + "1517616000000000000 | \n", + "48 | \n", + "
| 2 | \n", + "1518220800000000000 | \n", + "47 | \n", + "
| 3 | \n", + "1518825600000000000 | \n", + "46 | \n", + "
| 4 | \n", + "1519430400000000000 | \n", + "43 | \n", + "
"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "nTCutDQGoOxY"
- },
- "outputs": [],
- "source": [
- "!pip install keras-tuner --upgrade\n",
- "%matplotlib inline\n",
- "\n",
- "# Import our dependencies\n",
- "from sklearn.model_selection import train_test_split\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "import pandas as pd\n",
- "import tensorflow as tf\n",
- "\n",
- "# Import and read the charity_data.csv.\n",
- "import pandas as pd\n",
- "application_df = pd.read_csv(\"https://static.bc-edx.com/data/dl-1-2/m21/lms/starter/charity_data.csv\")\n",
- "application_df.head()\n",
- "application_df.tail()"
- ]
- },
- {
- "cell_type": "code",
- "source": [
- "# Drop the non-beneficial ID columns, 'EIN' and 'NAME'.\n",
- "application_df = application_df.drop(columns = ['EIN', 'NAME'],axis=1)\n",
- "application_df.head()\n",
- "application_df.tail()"
- ],
- "metadata": {
- "id": "hL-nfZhKovLI"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Determine the number of unique values in each column.\n",
- "application_df.nunique()"
- ],
- "metadata": {
- "id": "VhrRNiptovy9"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Look at APPLICATION_TYPE value counts for binning\n",
- "application_df_counts = application_df['APPLICATION_TYPE'].value_counts()\n",
- "application_df_counts"
- ],
- "metadata": {
- "id": "54sMSuG0ozWV"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Choose a cutoff value and create a list of application types to be replaced\n",
- "# use the variable name `application_types_to_replace`\n",
- "application_types_to_replace = list(application_df_counts[application_df_counts<500].index)\n",
- "\n",
- "# Replace in dataframe\n",
- "for app in application_types_to_replace:\n",
- " application_df['APPLICATION_TYPE'] = application_df['APPLICATION_TYPE'].replace(app,\"Other\")\n",
- "\n",
- "# Check to make sure binning was successful\n",
- "application_df['APPLICATION_TYPE'].value_counts()"
- ],
- "metadata": {
- "id": "Mkrjf-6Lo2IL"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Look at CLASSIFICATION value counts for binning\n",
- "class_counts = application_df['CLASSIFICATION'].value_counts()\n",
- "class_counts"
- ],
- "metadata": {
- "id": "lhtrnl4Jo4oE"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# You may find it helpful to look at CLASSIFICATION value counts >1\n",
- "class_type = class_counts.loc[class_counts > 1]\n",
- "class_type"
- ],
- "metadata": {
- "id": "cCrl-BZ5o66g"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Choose a cutoff value and create a list of classifications to be replaced\n",
- "# use the variable name `classifications_to_replace`\n",
- "classifications_to_replace = list(class_counts[class_counts < 1000].index)\n",
- "\n",
- "# Replace in dataframe\n",
- "for cls in classifications_to_replace:\n",
- " application_df['CLASSIFICATION'] = application_df['CLASSIFICATION'].replace(cls,\"Other\")\n",
- "\n",
- "# Check to make sure binning was successful\n",
- "application_df['CLASSIFICATION'].value_counts()"
- ],
- "metadata": {
- "id": "NQA9sL5Ho679"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Convert categorical data to numeric with `pd.get_dummies`\n",
- "dummies_df = pd.get_dummies(application_df)\n",
- "dummies_df.head()\n"
- ],
- "metadata": {
- "id": "ZLaV6OVVo9wu"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Split our preprocessed data into our features and target arrays\n",
- "y = dummies_df['IS_SUCCESSFUL'].values\n",
- "X = dummies_df.drop('IS_SUCCESSFUL', axis=1).values\n",
- "\n",
- "# Split the preprocessed data into a training and testing dataset\n",
- "X_train, X_test, y_train, y_test = train_test_split(X,y, random_state=78, test_size=.2)"
- ],
- "metadata": {
- "id": "JiE71OCRpBK8"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "print(f\"X_train Shape: {X_train.shape}, X_test Shape: {X_test.shape}\")"
- ],
- "metadata": {
- "id": "F-DkLL-epDuv"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Create a StandardScaler instances\n",
- "scaler = StandardScaler()\n",
- "\n",
- "# Fit the StandardScaler\n",
- "X_scaler = scaler.fit(X_train)\n",
- "\n",
- "# Scale the data\n",
- "X_train_scaled = X_scaler.transform(X_train)\n",
- "X_test_scaled = X_scaler.transform(X_test)"
- ],
- "metadata": {
- "id": "ruk8SGQzpFtM"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# installing kera_tuner upgrade\n",
- "# !pip install keras-tuner --upgrade\n",
- "\n",
- "# Import the kerastuner library\n",
- "#import keras_tuner as kt"
- ],
- "metadata": {
- "id": "l5_243-lpFu0"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "import warnings\n",
- "warnings.filterwarnings(\"ignore\")"
- ],
- "metadata": {
- "id": "xYT_dwpMpKTA"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Define the model - deep neural net, i.e., the number of input features and hidden nodes for each layer.\n",
- "input_features = len(X_train_scaled[0])\n",
- "hidden_nodes_layer1 = 80\n",
- "hidden_nodes_layer2 = 30\n",
- "hidden_nodes_layer3 = 1\n",
- "\n",
- "nn = tf.keras.models.Sequential()\n",
- "\n",
- "# First hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer1,\n",
- " input_dim=input_features, activation=\"relu\"))\n",
- "\n",
- "# Second hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer2, activation=\"sigmoid\"))\n",
- "\n",
- "# Output layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer3, activation=\"sigmoid\"))\n",
- "\n",
- "\n",
- "# Check the structure of the model\n",
- "nn.summary()"
- ],
- "metadata": {
- "id": "6frBap78pKUU"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Compile the model\n",
- "nn.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])"
- ],
- "metadata": {
- "id": "7Jul0GDHpKXc"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Train the model\n",
- "fit_model = nn.fit(X_train_scaled, y_train, epochs=100)"
- ],
- "metadata": {
- "id": "wckfODjopKaT"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Evaluate the model using the test data\n",
- "model_loss, model_accuracy = nn.evaluate(X_test_scaled,y_test,verbose=2)\n",
- "print(f\"Loss: {model_loss}, Accuracy: {model_accuracy}\")"
- ],
- "metadata": {
- "id": "mG26GVMKpKc7"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Export our model to HDF5 file\n",
- "nn.save(\"AlphabetSoupCharity.h5\")"
- ],
- "metadata": {
- "id": "jQFjjkWFpKfx"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "!pip install keras-tuner --upgrade\n",
- "%matplotlib inline\n",
- "\n",
- "!pip install tensorflow\n",
- "\n",
- "# Import our dependencies\n",
- "from sklearn.model_selection import train_test_split\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "import pandas as pd\n",
- "import tensorflow as tf\n",
- "\n",
- "# Import and read the charity_data.csv.\n",
- "import pandas as pd\n",
- "application_df = pd.read_csv(\"https://static.bc-edx.com/data/dl-1-2/m21/lms/starter/charity_data.csv\")\n",
- "application_df.head()\n",
- "application_df.tail()"
- ],
- "metadata": {
- "id": "MJJBDZhxpKky"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "\n",
- "# Drop the non-beneficial ID columns, 'EIN' and 'NAME'.\n",
- "application_df = application_df.drop(columns = ['EIN', 'NAME'],axis=1)\n",
- "application_df.head()\n",
- "application_df.tail()\n"
- ],
- "metadata": {
- "id": "GvPAjnEMpKnF"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Determine the number of unique values in each column.\n",
- "application_df.nunique()\n"
- ],
- "metadata": {
- "id": "3f859pSRpKpq"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Look at APPLICATION_TYPE value counts for binning\n",
- "application_df_counts = application_df['APPLICATION_TYPE'].value_counts()\n",
- "application_df_counts"
- ],
- "metadata": {
- "id": "u73oTiqApKsT"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Choose a cutoff value and create a list of application types to be replaced\n",
- "# use the variable name `application_types_to_replace`\n",
- "application_types_to_replace = list(application_df_counts[application_df_counts<500].index)\n",
- "\n",
- "# Replace in dataframe\n",
- "for app in application_types_to_replace:\n",
- " application_df['APPLICATION_TYPE'] = application_df['APPLICATION_TYPE'].replace(app,\"Other\")\n",
- "\n",
- "# Check to make sure binning was successful\n",
- "application_df['APPLICATION_TYPE'].value_counts()"
- ],
- "metadata": {
- "id": "HvTxcJmRpKup"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Look at CLASSIFICATION value counts for binning\n",
- "class_counts = application_df['CLASSIFICATION'].value_counts()\n",
- "class_counts"
- ],
- "metadata": {
- "id": "MQSfOqmUpjL5"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# You may find it helpful to look at CLASSIFICATION value counts >1\n",
- "class_type = class_counts.loc[class_counts > 1]\n",
- "class_type"
- ],
- "metadata": {
- "id": "BF9VYHRJpjOw"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- " Choose a cutoff value and create a list of classifications to be replaced\n",
- "# use the variable name `classifications_to_replace`\n",
- "classifications_to_replace = list(class_counts[class_counts < 1000].index)\n",
- "\n",
- "# Replace in dataframe\n",
- "for cls in classifications_to_replace:\n",
- " application_df['CLASSIFICATION'] = application_df['CLASSIFICATION'].replace(cls,\"Other\")\n",
- "\n",
- "# Check to make sure binning was successful\n",
- "application_df['CLASSIFICATION'].value_counts()"
- ],
- "metadata": {
- "id": "LsF3Ozx6pjRV"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Convert categorical data to numeric with `pd.get_dummies`\n",
- "dummies_df = pd.get_dummies(application_df)\n",
- "dummies_df.head()"
- ],
- "metadata": {
- "id": "YFcUF3hzpjUx"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Split our preprocessed data into our features and target arrays\n",
- "y = dummies_df['IS_SUCCESSFUL'].values\n",
- "X = dummies_df.drop('IS_SUCCESSFUL', axis=1).values\n",
- "\n",
- "# Split the preprocessed data into a training and testing dataset\n",
- "X_train, X_test, y_train, y_test = train_test_split(X,y, random_state=78, test_size=.2)\n"
- ],
- "metadata": {
- "id": "izZ1YXSlpjXF"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "print(f\"X_train Shape: {X_train.shape}, X_test Shape: {X_test.shape}\")\n"
- ],
- "metadata": {
- "id": "Nb3Xn_MOpjZa"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Create a StandardScaler instances\n",
- "scaler = StandardScaler()\n",
- "\n",
- "# Fit the StandardScaler\n",
- "X_scaler = scaler.fit(X_train)\n",
- "\n",
- "# Scale the data\n",
- "X_train_scaled = X_scaler.transform(X_train)\n",
- "X_test_scaled = X_scaler.transform(X_test)"
- ],
- "metadata": {
- "id": "Pv3LdlUOpjcR"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "#Import the kerastuner library\n",
- "import keras_tuner as kt"
- ],
- "metadata": {
- "id": "ZpjXPBbLpje3"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "#FIRST ATTEMPT\n",
- "# Define the model - deep neural net, i.e., the number of input features and hidden nodes for each layer.\n",
- "input_features = len(X_train_scaled[0])\n",
- "hidden_nodes_layer1 = 80\n",
- "hidden_nodes_layer2 = 30\n",
- "hidden_nodes_layer3 = 1\n",
- "\n",
- "nn = tf.keras.models.Sequential()\n",
- "\n",
- "# First hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer1,\n",
- " input_dim=input_features, activation=\"tanh\"))\n",
- "\n",
- "# Second hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer2, activation=\"sigmoid\"))\n",
- "\n",
- "# Third layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer3, activation=\"relu\"))\n",
- "\n",
- "# Outer Layer\n",
- "nn.add(tf.keras.layers.Dense(units=1, activation=\"sigmoid\"))\n",
- "\n",
- "\n",
- "# Check the structure of the model\n",
- "nn.summary()"
- ],
- "metadata": {
- "id": "WT1DSYt6pjhw"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *FIRST ATTEMPT* Compile the model\n",
- "nn.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])"
- ],
- "metadata": {
- "id": "xi9sevJPpjj0"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *FIRST ATTEMPT* Train the model\n",
- "fit_model = nn.fit(X_train_scaled, y_train, epochs=100)"
- ],
- "metadata": {
- "id": "osGv-I_Vpjms"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *FIRST ATTEMPT* Evaluate the model using the test data\n",
- "model_loss, model_accuracy = nn.evaluate(X_test_scaled,y_test,verbose=2)\n",
- "print(f\"Loss: {model_loss}, Accuracy: {model_accuracy}\")"
- ],
- "metadata": {
- "id": "2iblbOWSpjpF"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *FIRST ATTEMPT* ACCURACY\n",
- "First Attempt Accuracy = 72.5%"
- ],
- "metadata": {
- "id": "z3UZkZgxpjrp"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "#SECOND ATTEMPT\n",
- "# Define the model - deep neural net, i.e., the number of input features and hidden nodes for each layer.\n",
- "input_features = len(X_train_scaled[0])\n",
- "hidden_nodes_layer1 = 100\n",
- "hidden_nodes_layer2 = 30\n",
- "hidden_nodes_layer3 = 1\n",
- "\n",
- "nn = tf.keras.models.Sequential()\n",
- "\n",
- "# First hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer1,\n",
- " input_dim=input_features, activation=\"relu\"))\n",
- "\n",
- "# Second hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer2, activation=\"sigmoid\"))\n",
- "\n",
- "# Third layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer3, activation=\"relu\"))\n",
- "\n",
- "# Outer Layer\n",
- "nn.add(tf.keras.layers.Dense(units=1, activation=\"sigmoid\"))\n",
- "\n",
- "\n",
- "# Check the structure of the model\n",
- "nn.summary()"
- ],
- "metadata": {
- "id": "xrhb6QMfpjuP"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *SECOND ATTEMPT* Compile the model\n",
- "nn.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])"
- ],
- "metadata": {
- "id": "NCXuyAQJpjw2"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *SECOND ATTEMPT* Train the model\n",
- "fit_model = nn.fit(X_train_scaled, y_train, epochs=100)"
- ],
- "metadata": {
- "id": "VRxl-SKnpjzc"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *SECOND ATTEMPT* Evaluate the model using the test data\n",
- "model_loss, model_accuracy = nn.evaluate(X_test_scaled,y_test,verbose=2)\n",
- "print(f\"Loss: {model_loss}, Accuracy: {model_accuracy}\")"
- ],
- "metadata": {
- "id": "t0rABw3Gpj2F"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *SECOND* ACCURACY\n",
- "#Changes for second attempt: first hidden layer activation from \"tahn\" to \"relu\"\n",
- "# increase hidden layer nodes 1 from 80 to 100\n",
- "Second Attempt Accuracy = 72.1% (vs first attempt of 72.5%)"
- ],
- "metadata": {
- "id": "9DftVHWPpj4Z"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *THIRD ATTEMPT*\n",
- "# Define the model - deep neural net, i.e., the number of input features and hidden nodes for each layer.\n",
- "input_features = len(X_train_scaled[0])\n",
- "hidden_nodes_layer1 = 80\n",
- "hidden_nodes_layer2 = 25\n",
- "hidden_nodes_layer3 = 2\n",
- "\n",
- "nn = tf.keras.models.Sequential()\n",
- "\n",
- "# First hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer1,\n",
- " input_dim=input_features, activation=\"relu\"))\n",
- "\n",
- "# Second hidden layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer2, activation=\"sigmoid\"))\n",
- "\n",
- "# Third layer\n",
- "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer3, activation=\"relu\"))\n",
- "\n",
- "# Outer Layer\n",
- "nn.add(tf.keras.layers.Dense(units=1, activation=\"sigmoid\"))\n",
- "\n",
- "\n",
- "# Check the structure of the model\n",
- "nn.summary()\n"
- ],
- "metadata": {
- "id": "J4zv1OSNqZwx"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *THIRD ATTEMPT* Compile the model\n",
- "nn.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n"
- ],
- "metadata": {
- "id": "oFw0EosHqZxr"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *THIRD ATTEMPT* Train the model\n",
- "fit_model = nn.fit(X_train_scaled, y_train, epochs=100)"
- ],
- "metadata": {
- "id": "xrKzAwGLqZ44"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *THIRD ATTEMPT* Evaluate the model using the test data\n",
- "model_loss, model_accuracy = nn.evaluate(X_test_scaled,y_test,verbose=2)\n",
- "print(f\"Loss: {model_loss}, Accuracy: {model_accuracy}\")"
- ],
- "metadata": {
- "id": "6-eqf6xCqZ7d"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# *THIRD* ACCURACY\n",
- "First Attempt Accuracy = 72.2%"
- ],
- "metadata": {
- "id": "NLzLER_jqZ-E"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "import os\n",
- "# Find the latest version of spark 3.x from http://www.apache.org/dist/spark/ and enter as the spark version\n",
- "# For example:\n",
- "# spark_version = 'spark-3.5.1'\n",
- "spark_version = 'spark-3.5.1'\n",
- "os.environ['SPARK_VERSION']=spark_version\n",
- "\n",
- "# Install Spark and Java\n",
- "!apt-get update\n",
- "!apt-get install openjdk-11-jdk-headless -qq > /dev/null\n",
- "!wget -q http://www.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop3.tgz\n",
- "!tar xf $SPARK_VERSION-bin-hadoop3.tgz\n",
- "!pip install -q findspark\n",
- "\n",
- "# Set Environment Variables\n",
- "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\n",
- "os.environ[\"SPARK_HOME\"] = f\"/content/{spark_version}-bin-hadoop3\"\n",
- "\n",
- "# Start a SparkSession\n",
- "import findspark\n",
- "findspark.init()"
- ],
- "metadata": {
- "id": "Wpwrv8_dqaAe"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# Import packages\n",
- "from pyspark.sql import SparkSession\n",
- "import time\n",
- "\n",
- "# Create a SparkSession\n",
- "spark = SparkSession.builder.appName(\"SparkSQL\").getOrCreate()"
- ],
- "metadata": {
- "id": "4gd5ZX0XqaCw"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 1. Read in the AWS S3 bucket into a DataFrame.\n",
- "from pyspark import SparkFiles\n",
- "url = \"https://2u-data-curriculum-team.s3.amazonaws.com/dataviz-classroom/v1.2/22-big-data/home_sales_revised.csv\"\n",
- "spark.sparkContext.addFile(url)\n",
- "df = spark.read.csv(SparkFiles.get(\"home_sales_revised.csv\"), sep=\",\", header=True)\n",
- "\n",
- "# Show DataFrame\n",
- "df.show()\n"
- ],
- "metadata": {
- "id": "8iLGqnOKqaFG"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "\n",
- "# Print our schema\n",
- "df.printSchema()# 2. Create a temporary view of the DataFrame.\n",
- "\n",
- "df.createOrReplaceTempView('home_sales')"
- ],
- "metadata": {
- "id": "2TKsOjUcqaHt"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 3. What is the average price for a four bedroom house sold per year, rounded to two decimal places?\n",
- "a = \"\"\"\n",
- "SELECT\n",
- " YEAR(date) AS YEAR,\n",
- " ROUND(AVG(price), 2) AS AVERAGE_PRICE\n",
- "FROM home_sales\n",
- "WHERE bedrooms = 4\n",
- "GROUP BY YEAR\n",
- "ORDER BY YEAR DESC\n",
- "\"\"\"\n",
- "spark.sql(a).show()"
- ],
- "metadata": {
- "id": "0pIPKxgTqaJy"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 4. What is the average price of a home for each year the home was built,\n",
- "# that have 3 bedrooms and 3 bathrooms, rounded to two decimal places?\n",
- "b = \"\"\"\n",
- "SELECT\n",
- " YEAR(date_built) AS YEAR,\n",
- " ROUND(AVG(price), 2) AS AVERAGE_PRICE\n",
- "FROM home_sales\n",
- "WHERE bedrooms = 3\n",
- "and bathrooms = 3\n",
- "GROUP BY YEAR(date_built)\n",
- "ORDER BY YEAR DESC\n",
- "\"\"\"\n",
- "spark.sql(b).show()\n"
- ],
- "metadata": {
- "id": "Yfr0e-XxqaMa"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 5. What is the average price of a home for each year the home was built,\n",
- "# that have 3 bedrooms, 3 bathrooms, with two floors,\n",
- "# and are greater than or equal to 2,000 square feet, rounded to two decimal places?\n",
- "\n",
- "\n",
- "c = \"\"\"\n",
- "SELECT\n",
- "YEAR(date_built) AS YEAR_BUILT,\n",
- "ROUND(AVG(price), 2) AS AVERAGE_PRICE\n",
- "FROM home_sales\n",
- "WHERE bedrooms = 3\n",
- "and bathrooms = 3\n",
- "and sqft_living >= 2000\n",
- "and floors = 2\n",
- "GROUP BY YEAR_BUILT\n",
- "ORDER BY YEAR_BUILT DESC\n",
- "\"\"\"\n",
- "spark.sql(c).show()"
- ],
- "metadata": {
- "id": "HrEbTABkqaOu"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 6. What is the average price of a home per \"view\" rating, rounded to two decimal places,\n",
- "# having an average home price greater than or equal to $350,000? Order by descending view rating.\n",
- "# Although this is a small dataset, determine the run time for this query.\n",
- "\n",
- "start_time = time.time()\n",
- "\n",
- "d = \"\"\"\n",
- "SELECT\n",
- "view,\n",
- "ROUND(AVG(price), 2) AS AVERAGE_PRICE\n",
- "FROM home_sales\n",
- "GROUP BY view\n",
- "HAVING AVG(price) >= 350000\n",
- "ORDER BY view desc\n",
- "\"\"\"\n",
- "spark.sql(d).show()\n",
- "\n",
- "print(\"--- %s seconds ---\" % (time.time() - start_time))"
- ],
- "metadata": {
- "id": "PpPgkgRxqzuG"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 7. Cache the the temporary table home_sales.\n",
- "spark.sql('cache table home_sales')"
- ],
- "metadata": {
- "id": "IBCdFP4Eqzvi"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 8. Check if the table is cached.\n",
- "spark.catalog.isCached('home_sales')"
- ],
- "metadata": {
- "id": "AqFsOiWwqz1S"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 9. Using the cached data, run the last query above, that calculates\n",
- "# the average price of a home per \"view\" rating, rounded to two decimal places,\n",
- "# having an average home price greater than or equal to $350,000.\n",
- "# Determine the runtime and compare it to the uncached runtime.\n",
- "\n",
- "start_time = time.time()\n",
- "\n",
- "e = \"\"\"\n",
- "SELECT\n",
- "view,\n",
- "ROUND(AVG(price), 2) AS AVERAGE_PRICE\n",
- "FROM home_sales\n",
- "GROUP BY view\n",
- "HAVING AVG(price) >= 350000\n",
- "ORDER BY view desc\n",
- "\"\"\"\n",
- "spark.sql(e).show()\n",
- "\n",
- "\n",
- "print(\"--- %s seconds ---\" % (time.time() - start_time))\n",
- "\n",
- "# d speed = 0.7024235725402832 seconds\n",
- "# e spped = 0.5843555927276611 seconds\n",
- "# cache-ing sped up the run time !\n"
- ],
- "metadata": {
- "id": "SJTRR5UHqz5_"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 10. Partition by the \"date_built\" field on the formatted parquet home sales data\n",
- "df.write.partitionBy('date_built').mode(\"overwrite\").parquet(\"p_home_sales\")"
- ],
- "metadata": {
- "id": "cSeQIJrFq6G4"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 11. Read the parquet formatted data.\n",
- "p_df = spark.read.parquet('p_home_sales')"
- ],
- "metadata": {
- "id": "LgELnGCNq6OJ"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 12. Create a temporary table for the parquet data.\n",
- "p_df.createOrReplaceTempView('parquet_temp_home')"
- ],
- "metadata": {
- "id": "bL8GVtuEq6Xj"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 13. Using the parquet DataFrame, run the last query above, that calculates\n",
- "# the average price of a home per \"view\" rating, rounded to two decimal places,\n",
- "# having an average home price greater than or equal to $350,000.\n",
- "# Determine the runtime and compare it to the cached runtime.\n",
- "\n",
- "start_time = time.time()\n",
- "\n",
- "f = \"\"\"\n",
- "SELECT\n",
- "view,\n",
- "ROUND(AVG(price), 2) AS AVERAGE_PRICE\n",
- "FROM parquet_temp_home\n",
- "GROUP BY view\n",
- "HAVING AVG(price) >= 350000\n",
- "ORDER BY view desc\n",
- "\"\"\"\n",
- "spark.sql(f).show()\n",
- "\n",
- "print(\"--- %s seconds ---\" % (time.time() - start_time))\n",
- "\n",
- "# d speed = 0.7024235725402832 seconds\n",
- "# e spped = 0.5843555927276611 seconds\n",
- "# cache-ing sped up the run time !\n",
- "\n",
- "# f speed = 0.6659867763519287 seconds\n",
- "# parquet_temp_home is faster than original time, but not as fast as cached time\n"
- ],
- "metadata": {
- "id": "G5ynaoJvq6fl"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 14. Uncache the home_sales temporary table.\n",
- "spark.sql('uncache table home_sales')"
- ],
- "metadata": {
- "id": "zAI0vZamq6mq"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "code",
- "source": [
- "# 15. Check if the home_sales is no longer cached\n",
- "if spark.catalog.isCached('home_sales'):\n",
- " print('home_sales remains cached')\n",
- "else:\n",
- " print('home_sales is no longer cached. ')\n"
- ],
- "metadata": {
- "id": "Yn38FqT8rI-o"
- },
- "execution_count": null,
- "outputs": []
- }
- ]
-}
\ No newline at end of file