From 0954341fa4ab5b8d63450c43cdc2a7c4858eaf5a Mon Sep 17 00:00:00 2001 From: vishhh29 <77625419+vishhh29@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:03:31 -0500 Subject: [PATCH 1/4] Add files via upload --- ...sion_with_Elastic_Net_Regularization.ipynb | 149 ++++++++++++++++++ Readme.unknown | 35 ++++ 2 files changed, 184 insertions(+) create mode 100644 Linear_Regression_with_Elastic_Net_Regularization.ipynb create mode 100644 Readme.unknown diff --git a/Linear_Regression_with_Elastic_Net_Regularization.ipynb b/Linear_Regression_with_Elastic_Net_Regularization.ipynb new file mode 100644 index 0000000..69ce264 --- /dev/null +++ b/Linear_Regression_with_Elastic_Net_Regularization.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "zc02sM5D6aG1" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "class ElasticNetLinearRegression:\n", + " def __init__(self, alpha=1.0, l1_ratio=0.5, learning_rate=0.01, iterations=1000):\n", + " \"\"\"\n", + " Elastic Net Linear Regression constructor.\n", + "\n", + " Parameters:\n", + " alpha : Regularization strength (alpha > 0). Higher alpha increases regularization.\n", + " l1_ratio : The ratio between L1 (Lasso) and L2 (Ridge) regularization.\n", + " l1_ratio = 0 corresponds to L2 penalty (Ridge),\n", + " l1_ratio = 1 corresponds to L1 penalty (Lasso),\n", + " and 0 < l1_ratio < 1 corresponds to Elastic Net.\n", + " learning_rate : The step size for gradient descent. Controls how much to adjust the model per iteration.\n", + " iterations : Number of iterations for the gradient descent optimization.\n", + " \"\"\"\n", + " self.alpha = alpha # Store regularization strength\n", + " self.l1_ratio = l1_ratio # Store ratio between L1 and L2 regularization\n", + " self.learning_rate = learning_rate # Store learning rate for gradient descent\n", + " self.iterations = iterations # Store number of iterations for gradient descent\n", + "\n", + " def fit(self, X, y):\n", + " \"\"\"\n", + " Fit the model using gradient descent to minimize the cost function.\n", + "\n", + " Parameters:\n", + " X : Input feature matrix (num_samples x num_features)\n", + " y : Target vector (num_samples,)\n", + " \"\"\"\n", + " self.m, self.n = X.shape # Number of samples (m) and number of features (n)\n", + " self.theta = np.zeros(self.n) # Initialize weights (coefficients) to zeros\n", + " self.bias = 0 # Initialize bias (intercept) to zero\n", + "\n", + " # Gradient descent optimization loop\n", + " for _ in range(self.iterations):\n", + " y_pred = self.predict(X) # Compute predictions based on current weights and bias\n", + "\n", + " # Compute gradients for weights (theta) using Elastic Net regularization\n", + " d_theta = (1 / self.m) * (X.T @ (y_pred - y)) + self.alpha * (\n", + " self.l1_ratio * np.sign(self.theta) + # L1 penalty (Lasso)\n", + " (1 - self.l1_ratio) * self.theta) # L2 penalty (Ridge)\n", + "\n", + " # Compute gradient for bias (intercept)\n", + " d_bias = (1 / self.m) * np.sum(y_pred - y)\n", + "\n", + " # Update weights and bias using the gradients\n", + " self.theta -= self.learning_rate * d_theta # Update weights\n", + " self.bias -= self.learning_rate * d_bias # Update bias\n", + "\n", + " def predict(self, X):\n", + " \"\"\"\n", + " Make predictions using the learned weights and bias.\n", + "\n", + " Parameters:\n", + " X : Input feature matrix (num_samples x num_features)\n", + "\n", + " Returns:\n", + " y_pred : Predictions (num_samples,)\n", + " \"\"\"\n", + " return X @ self.theta + self.bias # Linear prediction (X * theta + bias)\n", + "\n", + " def mse(self, y_true, y_pred):\n", + " \"\"\"\n", + " Compute Mean Squared Error (MSE) as a performance metric.\n", + "\n", + " Parameters:\n", + " y_true : True target values (ground truth)\n", + " y_pred : Predicted values from the model\n", + "\n", + " Returns:\n", + " mse : Mean Squared Error, a measure of how close the predictions are to the true values.\n", + " \"\"\"\n", + " return np.mean((y_true - y_pred) ** 2) # Calculate the average of the squared differences\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Kjky5fZ_7t_Z", + "outputId": "ded25616-d58f-4d74-a85d-a046ab012cb4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predictions: [ 5.79274902 7.77612778 9.03612325 11.01950201]\n", + "MSE: 0.023689238600062532\n" + ] + } + ], + "source": [ + "# Sample data\n", + "X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]])\n", + "y = np.array([6, 8, 9, 11])\n", + "\n", + "# Initialize model\n", + "model = ElasticNetLinearRegression(alpha=0.1, l1_ratio=0.5, learning_rate=0.01, iterations=1000)\n", + "\n", + "# Fit the model to data\n", + "model.fit(X, y)\n", + "\n", + "# Predict using the model\n", + "y_pred = model.predict(X)\n", + "\n", + "# Print predictions and the MSE\n", + "print(\"Predictions:\", y_pred)\n", + "print(\"MSE:\", model.mse(y, y_pred))\n" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Readme.unknown b/Readme.unknown new file mode 100644 index 0000000..ed89e34 --- /dev/null +++ b/Readme.unknown @@ -0,0 +1,35 @@ + +# Linear Regression with Elastic Net Regularization +Group Members: +Vishwas Reddy Dodle (A20562449) +Yaswanth Mopada (A20585424) +----------------------------------------------------------------------------------------------- +# Project Overview +This project implements a linear regression model with Elastic Net regularization, which is a combination of L1 (Lasso) and L2 (Ridge) penalties. Elastic Net is particularly useful in scenarios where features are correlated or when feature selection is required. +----------------------------------------------------------------------------------------------- +# Usage Instructions +## To use the model: + +I: Initialize the ElasticNetLinearRegression class with desired parameters. +II: Call the fit() method on your training data. +III: Use the predict() method to make predictions on new data. +IV: Evaluate the model using the mse() method for Mean Squared Error calculation. +----------------------------------------------------------------------------------------------- +# 1. What does the model you have implemented do and when should it be used? +This model performs linear regression with Elastic Net regularization, which is a combination of L1 (Lasso) and L2 (Ridge) regularization. It is designed to handle both overfitting and multicollinearity in datasets, where it penalizes large coefficients (L2) and encourages sparsity (L1). It is useful when features are correlated or when feature selection is necessary in the model. + +# 2. How did you test your model to determine if it is working reasonably correctly? +I tested the model using a small synthetic dataset where the linear relationship is known. I computed predictions and checked that the Mean Squared Error (MSE) decreased as the model optimized the parameters during gradient descent. The model was also checked for handling both Lasso (L1) and Ridge (L2) penalties by varying the l1_ratio parameter. + +# 3. What parameters have you exposed to users of your implementation in order to tune performance? +alpha: Controls the strength of regularization (penalty). +l1_ratio: Determines the balance between L1 and L2 regularization. +learning_rate: Adjusts the step size for gradient descent optimization. +iterations: Sets the number of iterations for gradient descent to converge. +These parameters can be tuned based on the dataset and model requirements to balance bias and variance. + +# 4. Are there specific inputs that your implementation has trouble with? Given more time, could you work around these, or is it fundamental to the model? +The implementation may struggle with: +Very large datasets due to the basic gradient descent approach. A more efficient optimization algorithm like stochastic gradient descent or coordinate descent could improve scalability. +Extreme multicollinearity: While Elastic Net helps with multicollinearity, very high correlations could still pose challenges. Further testing could involve adding features for automatic feature selection. +----------------------------------------------------------------------------------------------------- \ No newline at end of file From cd3f2de57a8c6ef51ea7de56ec5caf81278ed6f4 Mon Sep 17 00:00:00 2001 From: vishhh29 <77625419+vishhh29@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:07:42 -0500 Subject: [PATCH 2/4] Delete Readme.unknown --- Readme.unknown | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 Readme.unknown diff --git a/Readme.unknown b/Readme.unknown deleted file mode 100644 index ed89e34..0000000 --- a/Readme.unknown +++ /dev/null @@ -1,35 +0,0 @@ - -# Linear Regression with Elastic Net Regularization -Group Members: -Vishwas Reddy Dodle (A20562449) -Yaswanth Mopada (A20585424) ------------------------------------------------------------------------------------------------ -# Project Overview -This project implements a linear regression model with Elastic Net regularization, which is a combination of L1 (Lasso) and L2 (Ridge) penalties. Elastic Net is particularly useful in scenarios where features are correlated or when feature selection is required. ------------------------------------------------------------------------------------------------ -# Usage Instructions -## To use the model: - -I: Initialize the ElasticNetLinearRegression class with desired parameters. -II: Call the fit() method on your training data. -III: Use the predict() method to make predictions on new data. -IV: Evaluate the model using the mse() method for Mean Squared Error calculation. ------------------------------------------------------------------------------------------------ -# 1. What does the model you have implemented do and when should it be used? -This model performs linear regression with Elastic Net regularization, which is a combination of L1 (Lasso) and L2 (Ridge) regularization. It is designed to handle both overfitting and multicollinearity in datasets, where it penalizes large coefficients (L2) and encourages sparsity (L1). It is useful when features are correlated or when feature selection is necessary in the model. - -# 2. How did you test your model to determine if it is working reasonably correctly? -I tested the model using a small synthetic dataset where the linear relationship is known. I computed predictions and checked that the Mean Squared Error (MSE) decreased as the model optimized the parameters during gradient descent. The model was also checked for handling both Lasso (L1) and Ridge (L2) penalties by varying the l1_ratio parameter. - -# 3. What parameters have you exposed to users of your implementation in order to tune performance? -alpha: Controls the strength of regularization (penalty). -l1_ratio: Determines the balance between L1 and L2 regularization. -learning_rate: Adjusts the step size for gradient descent optimization. -iterations: Sets the number of iterations for gradient descent to converge. -These parameters can be tuned based on the dataset and model requirements to balance bias and variance. - -# 4. Are there specific inputs that your implementation has trouble with? Given more time, could you work around these, or is it fundamental to the model? -The implementation may struggle with: -Very large datasets due to the basic gradient descent approach. A more efficient optimization algorithm like stochastic gradient descent or coordinate descent could improve scalability. -Extreme multicollinearity: While Elastic Net helps with multicollinearity, very high correlations could still pose challenges. Further testing could involve adding features for automatic feature selection. ------------------------------------------------------------------------------------------------------ \ No newline at end of file From 8be7270348a9329f814f670fd20399ed785bd2d4 Mon Sep 17 00:00:00 2001 From: vishhh29 <77625419+vishhh29@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:16:01 -0500 Subject: [PATCH 3/4] Delete Linear_Regression_with_Elastic_Net_Regularization.ipynb --- ...sion_with_Elastic_Net_Regularization.ipynb | 149 ------------------ 1 file changed, 149 deletions(-) delete mode 100644 Linear_Regression_with_Elastic_Net_Regularization.ipynb diff --git a/Linear_Regression_with_Elastic_Net_Regularization.ipynb b/Linear_Regression_with_Elastic_Net_Regularization.ipynb deleted file mode 100644 index 69ce264..0000000 --- a/Linear_Regression_with_Elastic_Net_Regularization.ipynb +++ /dev/null @@ -1,149 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "zc02sM5D6aG1" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "class ElasticNetLinearRegression:\n", - " def __init__(self, alpha=1.0, l1_ratio=0.5, learning_rate=0.01, iterations=1000):\n", - " \"\"\"\n", - " Elastic Net Linear Regression constructor.\n", - "\n", - " Parameters:\n", - " alpha : Regularization strength (alpha > 0). Higher alpha increases regularization.\n", - " l1_ratio : The ratio between L1 (Lasso) and L2 (Ridge) regularization.\n", - " l1_ratio = 0 corresponds to L2 penalty (Ridge),\n", - " l1_ratio = 1 corresponds to L1 penalty (Lasso),\n", - " and 0 < l1_ratio < 1 corresponds to Elastic Net.\n", - " learning_rate : The step size for gradient descent. Controls how much to adjust the model per iteration.\n", - " iterations : Number of iterations for the gradient descent optimization.\n", - " \"\"\"\n", - " self.alpha = alpha # Store regularization strength\n", - " self.l1_ratio = l1_ratio # Store ratio between L1 and L2 regularization\n", - " self.learning_rate = learning_rate # Store learning rate for gradient descent\n", - " self.iterations = iterations # Store number of iterations for gradient descent\n", - "\n", - " def fit(self, X, y):\n", - " \"\"\"\n", - " Fit the model using gradient descent to minimize the cost function.\n", - "\n", - " Parameters:\n", - " X : Input feature matrix (num_samples x num_features)\n", - " y : Target vector (num_samples,)\n", - " \"\"\"\n", - " self.m, self.n = X.shape # Number of samples (m) and number of features (n)\n", - " self.theta = np.zeros(self.n) # Initialize weights (coefficients) to zeros\n", - " self.bias = 0 # Initialize bias (intercept) to zero\n", - "\n", - " # Gradient descent optimization loop\n", - " for _ in range(self.iterations):\n", - " y_pred = self.predict(X) # Compute predictions based on current weights and bias\n", - "\n", - " # Compute gradients for weights (theta) using Elastic Net regularization\n", - " d_theta = (1 / self.m) * (X.T @ (y_pred - y)) + self.alpha * (\n", - " self.l1_ratio * np.sign(self.theta) + # L1 penalty (Lasso)\n", - " (1 - self.l1_ratio) * self.theta) # L2 penalty (Ridge)\n", - "\n", - " # Compute gradient for bias (intercept)\n", - " d_bias = (1 / self.m) * np.sum(y_pred - y)\n", - "\n", - " # Update weights and bias using the gradients\n", - " self.theta -= self.learning_rate * d_theta # Update weights\n", - " self.bias -= self.learning_rate * d_bias # Update bias\n", - "\n", - " def predict(self, X):\n", - " \"\"\"\n", - " Make predictions using the learned weights and bias.\n", - "\n", - " Parameters:\n", - " X : Input feature matrix (num_samples x num_features)\n", - "\n", - " Returns:\n", - " y_pred : Predictions (num_samples,)\n", - " \"\"\"\n", - " return X @ self.theta + self.bias # Linear prediction (X * theta + bias)\n", - "\n", - " def mse(self, y_true, y_pred):\n", - " \"\"\"\n", - " Compute Mean Squared Error (MSE) as a performance metric.\n", - "\n", - " Parameters:\n", - " y_true : True target values (ground truth)\n", - " y_pred : Predicted values from the model\n", - "\n", - " Returns:\n", - " mse : Mean Squared Error, a measure of how close the predictions are to the true values.\n", - " \"\"\"\n", - " return np.mean((y_true - y_pred) ** 2) # Calculate the average of the squared differences\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Kjky5fZ_7t_Z", - "outputId": "ded25616-d58f-4d74-a85d-a046ab012cb4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predictions: [ 5.79274902 7.77612778 9.03612325 11.01950201]\n", - "MSE: 0.023689238600062532\n" - ] - } - ], - "source": [ - "# Sample data\n", - "X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]])\n", - "y = np.array([6, 8, 9, 11])\n", - "\n", - "# Initialize model\n", - "model = ElasticNetLinearRegression(alpha=0.1, l1_ratio=0.5, learning_rate=0.01, iterations=1000)\n", - "\n", - "# Fit the model to data\n", - "model.fit(X, y)\n", - "\n", - "# Predict using the model\n", - "y_pred = model.predict(X)\n", - "\n", - "# Print predictions and the MSE\n", - "print(\"Predictions:\", y_pred)\n", - "print(\"MSE:\", model.mse(y, y_pred))\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} From 296349e36d2d53e18ef908a8f544f9026b55e97a Mon Sep 17 00:00:00 2001 From: vishhh29 <77625419+vishhh29@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:17:23 -0500 Subject: [PATCH 4/4] Add files via upload --- ML Project/Linear-regression-Elasticnet.py | 91 ++++++++++++++++++++++ ML Project/README.md | 35 +++++++++ 2 files changed, 126 insertions(+) create mode 100644 ML Project/Linear-regression-Elasticnet.py create mode 100644 ML Project/README.md diff --git a/ML Project/Linear-regression-Elasticnet.py b/ML Project/Linear-regression-Elasticnet.py new file mode 100644 index 0000000..cad72cb --- /dev/null +++ b/ML Project/Linear-regression-Elasticnet.py @@ -0,0 +1,91 @@ +import numpy as np + +class ElasticNetLinearRegression: + def __init__(self, alpha=1.0, l1_ratio=0.5, learning_rate=0.01, iterations=1000): + """ + Elastic Net Linear Regression constructor. + + Parameters: + alpha : Regularization strength (alpha > 0). Higher alpha increases regularization. + l1_ratio : The ratio between L1 (Lasso) and L2 (Ridge) regularization. + l1_ratio = 0 corresponds to L2 penalty (Ridge), + l1_ratio = 1 corresponds to L1 penalty (Lasso), + and 0 < l1_ratio < 1 corresponds to Elastic Net. + learning_rate : The step size for gradient descent. Controls how much to adjust the model per iteration. + iterations : Number of iterations for the gradient descent optimization. + """ + self.alpha = alpha # Store regularization strength + self.l1_ratio = l1_ratio # Store ratio between L1 and L2 regularization + self.learning_rate = learning_rate # Store learning rate for gradient descent + self.iterations = iterations # Store number of iterations for gradient descent + + def fit(self, X, y): + """ + Fit the model using gradient descent to minimize the cost function. + + Parameters: + X : Input feature matrix (num_samples x num_features) + y : Target vector (num_samples,) + """ + self.m, self.n = X.shape # Number of samples (m) and number of features (n) + self.theta = np.zeros(self.n) # Initialize weights (coefficients) to zeros + self.bias = 0 # Initialize bias (intercept) to zero + + # Gradient descent optimization loop + for _ in range(self.iterations): + y_pred = self.predict(X) # Compute predictions based on current weights and bias + + # Compute gradients for weights (theta) using Elastic Net regularization + d_theta = (1 / self.m) * (X.T @ (y_pred - y)) + self.alpha * ( + self.l1_ratio * np.sign(self.theta) + # L1 penalty (Lasso) + (1 - self.l1_ratio) * self.theta) # L2 penalty (Ridge) + + # Compute gradient for bias (intercept) + d_bias = (1 / self.m) * np.sum(y_pred - y) + + # Update weights and bias using the gradients + self.theta -= self.learning_rate * d_theta # Update weights + self.bias -= self.learning_rate * d_bias # Update bias + + def predict(self, X): + """ + Make predictions using the learned weights and bias. + + Parameters: + X : Input feature matrix (num_samples x num_features) + + Returns: + y_pred : Predictions (num_samples,) + """ + return X @ self.theta + self.bias # Linear prediction (X * theta + bias) + + def mse(self, y_true, y_pred): + """ + Compute Mean Squared Error (MSE) as a performance metric. + + Parameters: + y_true : True target values (ground truth) + y_pred : Predicted values from the model + + Returns: + mse : Mean Squared Error, a measure of how close the predictions are to the true values. + """ + return np.mean((y_true - y_pred) ** 2) # Calculate the average of the squared differences + + +# Sample data +X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]]) +y = np.array([6, 8, 9, 11]) + +# Initialize model +model = ElasticNetLinearRegression(alpha=0.1, l1_ratio=0.5, learning_rate=0.01, iterations=1000) + +# Fit the model to data +model.fit(X, y) + +# Predict using the model +y_pred = model.predict(X) + +# Print predictions and the MSE +print("Predictions:", y_pred) +print("MSE:", model.mse(y, y_pred)) \ No newline at end of file diff --git a/ML Project/README.md b/ML Project/README.md new file mode 100644 index 0000000..cfe0d9a --- /dev/null +++ b/ML Project/README.md @@ -0,0 +1,35 @@ + +# Linear Regression with Elastic Net Regularization +Group Members: +Vishwas Reddy Dodle (A20562449) +Yaswanth Mopada (A20585424) +----------------------------------------------------------------------------------------------- +# Project Overview +This project implements a linear regression model with Elastic Net regularization, which is a combination of L1 (Lasso) and L2 (Ridge) penalties. Elastic Net is particularly useful in scenarios where features are correlated or when feature selection is required. +----------------------------------------------------------------------------------------------- +# Usage Instructions +## To use the model: + +I: Initialize the ElasticNetLinearRegression class with desired parameters. +II: Call the fit() method on your training data. +III: Use the predict() method to make predictions on new data. +IV: Evaluate the model using the mse() method for Mean Squared Error calculation. +----------------------------------------------------------------------------------------------- +# 1. What does the model you have implemented do and when should it be used? +This model performs linear regression with Elastic Net regularization, which is a combination of L1 (Lasso) and L2 (Ridge) regularization. It is designed to handle both overfitting and multicollinearity in datasets, where it penalizes large coefficients (L2) and encourages sparsity (L1). It is useful when features are correlated or when feature selection is necessary in the model. + +# 2. How did you test your model to determine if it is working reasonably correctly? +I tested the model using a small synthetic dataset where the linear relationship is known. I computed predictions and checked that the Mean Squared Error (MSE) decreased as the model optimized the parameters during gradient descent. The model was also checked for handling both Lasso (L1) and Ridge (L2) penalties by varying the l1_ratio parameter. + +# 3. What parameters have you exposed to users of your implementation in order to tune performance? +alpha: Controls the strength of regularization (penalty). +l1_ratio: Determines the balance between L1 and L2 regularization. +learning_rate: Adjusts the step size for gradient descent optimization. +iterations: Sets the number of iterations for gradient descent to converge. +These parameters can be tuned based on the dataset and model requirements to balance bias and variance. + +# 4. Are there specific inputs that your implementation has trouble with? Given more time, could you work around these, or is it fundamental to the model? +The implementation may struggle with: +Very large datasets due to the basic gradient descent approach. A more efficient optimization algorithm like stochastic gradient descent or coordinate descent could improve scalability. +Extreme multicollinearity: While Elastic Net helps with multicollinearity, very high correlations could still pose challenges. Further testing could involve adding features for automatic feature selection. +----------------------------------------------------------------------------------------------------- \ No newline at end of file