diff --git a/4_predictive_models.ipynb b/4_predictive_models.ipynb new file mode 100644 index 0000000..68bf9d1 --- /dev/null +++ b/4_predictive_models.ipynb @@ -0,0 +1,6367 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Predictive Models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## \"Housekeeping\"" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2018-09-05 22:14:29 CEST\n", + "\n", + "CPython 3.6.5\n", + "IPython 6.5.0\n", + "\n", + "numpy 1.15.1\n", + "pandas 0.23.4\n", + "sklearn 0.20rc1\n" + ] + } + ], + "source": [ + "% load_ext watermark\n", + "% watermark -d -t -v -z -p numpy,pandas,sklearn" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from sklearn.ensemble import RandomForestRegressor\n", + "from sklearn.linear_model import Lasso, LinearRegression, Ridge\n", + "from sklearn.metrics import (\n", + " make_scorer,\n", + " mean_absolute_error,\n", + " mean_squared_error,\n", + " r2_score,\n", + ")\n", + "from sklearn.model_selection import GridSearchCV, KFold\n", + "from sklearn.svm import SVR\n", + "from sklearn.utils import shuffle\n", + "from tqdm import tqdm_notebook as progress_bar\n", + "\n", + "from utils import (\n", + " CONTINUOUS_VARIABLES,\n", + " DISCRETE_VARIABLES,\n", + " NOMINAL_VARIABLES,\n", + " ORDINAL_VARIABLES,\n", + " TARGET_VARIABLES,\n", + " bias_score,\n", + " encode_ordinals,\n", + " load_clean_data,\n", + " max_deviation,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "random_state = np.random.RandomState(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "pd.set_option(\"display.max_columns\", 250)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext blackcellmagic" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Original Data\n", + "\n", + "The DataFrame `df1` holds the cleaned data from notebook 1 with the all the nominal and ordinal features automatically translated to factor variables and ordered integer values." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "df1 = load_clean_data(\"data/data_clean.csv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This cell basically **replaces** all the manual work that went into generating new and identifying \"interesting\" features in notebooks 2 and 3." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "df1 = pd.concat([\n", + " df1[CONTINUOUS_VARIABLES + DISCRETE_VARIABLES + ORDINAL_VARIABLES + TARGET_VARIABLES],\n", + " pd.get_dummies(df1[NOMINAL_VARIABLES], dtype=int),\n", + "], axis=1)\n", + "# Re-order the columns for convenience.\n", + "df1 = df1[sorted(set(df1.columns) - set(TARGET_VARIABLES)) + TARGET_VARIABLES]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "df1 = encode_ordinals(df1)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "df1 = shuffle(df1, random_state=random_state)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "MultiIndex: 2898 entries, (144, 535153070) to (867, 907253130)\n", + "Columns: 248 entries, 1st Flr SF to SalePrice\n", + "dtypes: float64(19), int64(229)\n", + "memory usage: 5.5 MB\n" + ] + } + ], + "source": [ + "df1.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
1st Flr SF2nd Flr SF3Ssn PorchAlley_GrvlAlley_NAAlley_PaveBedroom AbvGrBldg Type_1FamBldg Type_2FmConBldg Type_DuplxBldg Type_TwnhsEBldg Type_TwnhsIBsmt CondBsmt ExposureBsmt Full BathBsmt Half BathBsmt QualBsmt Unf SFBsmtFin SF 1BsmtFin SF 2BsmtFin Type 1BsmtFin Type 2Central Air_NCentral Air_YCondition 1_ArteryCondition 1_FeedrCondition 1_NormCondition 1_PosACondition 1_PosNCondition 1_RRAeCondition 1_RRAnCondition 1_RRNeCondition 1_RRNnCondition 2_ArteryCondition 2_FeedrCondition 2_NormCondition 2_PosACondition 2_PosNCondition 2_RRAeCondition 2_RRAnCondition 2_RRNeCondition 2_RRNnElectricalEnclosed PorchExter CondExter QualExterior 1st_AsbShngExterior 1st_AsphShnExterior 1st_BrkCommExterior 1st_BrkFaceExterior 1st_CBlockExterior 1st_CemntBdExterior 1st_HdBoardExterior 1st_ImStuccExterior 1st_MetalSdExterior 1st_OtherExterior 1st_PlywoodExterior 1st_PreCastExterior 1st_StoneExterior 1st_StuccoExterior 1st_VinylSdExterior 1st_Wd SdngExterior 1st_WdShingExterior 2nd_AsbShngExterior 2nd_AsphShnExterior 2nd_BrkCommExterior 2nd_BrkFaceExterior 2nd_CBlockExterior 2nd_CemntBdExterior 2nd_HdBoardExterior 2nd_ImStuccExterior 2nd_MetalSdExterior 2nd_OtherExterior 2nd_PlywoodExterior 2nd_PreCastExterior 2nd_StoneExterior 2nd_StuccoExterior 2nd_VinylSdExterior 2nd_Wd SdngExterior 2nd_WdShingFenceFireplace QuFireplacesFoundation_BrkTilFoundation_CBlockFoundation_PConcFoundation_SlabFoundation_StoneFoundation_WoodFull BathFunctionalGarage AreaGarage CarsGarage CondGarage FinishGarage QualGarage Type_2TypesGarage Type_AttchdGarage Type_BasmentGarage Type_BuiltInGarage Type_CarPortGarage Type_DetchdGarage Type_NAGr Liv AreaHalf BathHeating QCHeating_FloorHeating_GasAHeating_GasWHeating_GravHeating_OthWHeating_WallHouse Style_1.5FinHouse Style_1.5UnfHouse Style_1StoryHouse Style_2.5FinHouse Style_2.5UnfHouse Style_2StoryHouse Style_SFoyerHouse Style_SLvlKitchen AbvGrKitchen QualLand Contour_BnkLand Contour_HLSLand Contour_LowLand Contour_LvlLand SlopeLot AreaLot Config_CornerLot Config_CulDSacLot Config_FR2Lot Config_FR3Lot Config_InsideLot ShapeLow Qual Fin SFMS SubClass_020MS SubClass_030MS SubClass_040MS SubClass_045MS SubClass_050MS SubClass_060MS SubClass_070MS SubClass_075MS SubClass_080MS SubClass_085MS SubClass_090MS SubClass_120MS SubClass_150MS SubClass_160MS SubClass_180MS SubClass_190MS Zoning_AMS Zoning_CMS Zoning_FVMS Zoning_IMS Zoning_RHMS Zoning_RLMS Zoning_RMMS Zoning_RPMas Vnr AreaMas Vnr Type_BrkCmnMas Vnr Type_BrkFaceMas Vnr Type_CBlockMas Vnr Type_NoneMas Vnr Type_StoneMisc Feature_ElevMisc Feature_Gar2Misc Feature_NAMisc Feature_OthrMisc Feature_ShedMisc Feature_TenCMisc ValMo SoldNeighborhood_BlmngtnNeighborhood_BluesteNeighborhood_BrDaleNeighborhood_BrkSideNeighborhood_ClearCrNeighborhood_CollgCrNeighborhood_CrawforNeighborhood_EdwardsNeighborhood_GilbertNeighborhood_GreensNeighborhood_GrnHillNeighborhood_IDOTRRNeighborhood_LandmrkNeighborhood_MeadowVNeighborhood_MitchelNeighborhood_NPkVillNeighborhood_NWAmesNeighborhood_NamesNeighborhood_NoRidgeNeighborhood_NridgHtNeighborhood_OldTownNeighborhood_SWISUNeighborhood_SawyerNeighborhood_SawyerWNeighborhood_SomerstNeighborhood_StoneBrNeighborhood_TimberNeighborhood_VeenkerOpen Porch SFOverall CondOverall QualPaved DrivePool AreaPool QCRoof Matl_ClyTileRoof Matl_CompShgRoof Matl_MembranRoof Matl_MetalRoof Matl_RollRoof Matl_Tar&GrvRoof Matl_WdShakeRoof Matl_WdShnglRoof Style_FlatRoof Style_GableRoof Style_GambrelRoof Style_HipRoof Style_MansardRoof Style_ShedSale Condition_AbnormlSale Condition_AdjLandSale Condition_AllocaSale Condition_FamilySale Condition_NormalSale Condition_PartialSale Type_CODSale Type_CWDSale Type_ConSale Type_ConLDSale Type_ConLISale Type_ConLwSale Type_NewSale Type_OthSale Type_VWDSale Type_WDScreen PorchStreet_GrvlStreet_PaveTotRms AbvGrdTotal Bsmt SFUtilitiesWood Deck SFYear BuiltYear Remod/AddYr SoldSalePrice
OrderPID
1445351530701194.00.00.0010310000311031194.00.00.011010010000000010000004120.022000000001000000000000000010000000000001000017312.0132301000001194.0020100000010000012000128760.00000130.0100000000000000000000100220.0010000010000.0400000000000000000100000000000.05520.000100000000010000001000000000010.00161194.030.0195919592010148000.0
15749163800601537.00.00.001031000034105482.01036.00.0610100100000000100000040.023000000000000001000000000000000010000000100027788.0333301000001537.00401000000100000130100211563.00000120.0100000000000000000000100258.0000010010000.04000000000000000000000000001026.04720.000100000000010000001000000000010.00181518.030.0200620072008294000.0
490528290190774.0656.00.001031000031004384.00.00.0110100100000000100000040.022000000000000001000000000000000010003100100027400.0232300010001430.0140100000000000112000127750.00000130.00000000010000000000001000.0000100010000.0300000000100000000000000000000.04620.000100000001000000001000000000010.0017384.030.0199920002009156000.0
1730528218050783.0701.00.001031000031004783.00.00.0110100000010000100000040.023000000000000001000000000000000010004100100027393.0233301000001484.01401000000000100130001210237.00000130.00000010000000000000001000.0000100010000.07000000001000000000000000000072.04520.000100000001000000000100000010000.0018783.030.0200520072007178900.0
22769211280301824.00.00.0010310000340051824.00.00.0110100010000000010000040.024000000001000000000000000010000000004100100027932.0333301000001824.00401000000100000140100212633.00000120.0100000000000000000000100242.0010000010000.09000000000000000000000000001036.04920.00010000000001000000010000001000108.00181824.03160.0200620072007392000.0
\n", + "
" + ], + "text/plain": [ + " 1st Flr SF 2nd Flr SF 3Ssn Porch Alley_Grvl Alley_NA \\\n", + "Order PID \n", + "144 535153070 1194.0 0.0 0.0 0 1 \n", + "1574 916380060 1537.0 0.0 0.0 0 1 \n", + "490 528290190 774.0 656.0 0.0 0 1 \n", + "1730 528218050 783.0 701.0 0.0 0 1 \n", + "2276 921128030 1824.0 0.0 0.0 0 1 \n", + "\n", + " Alley_Pave Bedroom AbvGr Bldg Type_1Fam Bldg Type_2FmCon \\\n", + "Order PID \n", + "144 535153070 0 3 1 0 \n", + "1574 916380060 0 3 1 0 \n", + "490 528290190 0 3 1 0 \n", + "1730 528218050 0 3 1 0 \n", + "2276 921128030 0 3 1 0 \n", + "\n", + " Bldg Type_Duplx Bldg Type_TwnhsE Bldg Type_TwnhsI \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Bsmt Cond Bsmt Exposure Bsmt Full Bath Bsmt Half Bath \\\n", + "Order PID \n", + "144 535153070 3 1 1 0 \n", + "1574 916380060 3 4 1 0 \n", + "490 528290190 3 1 0 0 \n", + "1730 528218050 3 1 0 0 \n", + "2276 921128030 3 4 0 0 \n", + "\n", + " Bsmt Qual Bsmt Unf SF BsmtFin SF 1 BsmtFin SF 2 \\\n", + "Order PID \n", + "144 535153070 3 1194.0 0.0 0.0 \n", + "1574 916380060 5 482.0 1036.0 0.0 \n", + "490 528290190 4 384.0 0.0 0.0 \n", + "1730 528218050 4 783.0 0.0 0.0 \n", + "2276 921128030 5 1824.0 0.0 0.0 \n", + "\n", + " BsmtFin Type 1 BsmtFin Type 2 Central Air_N Central Air_Y \\\n", + "Order PID \n", + "144 535153070 1 1 0 1 \n", + "1574 916380060 6 1 0 1 \n", + "490 528290190 1 1 0 1 \n", + "1730 528218050 1 1 0 1 \n", + "2276 921128030 1 1 0 1 \n", + "\n", + " Condition 1_Artery Condition 1_Feedr Condition 1_Norm \\\n", + "Order PID \n", + "144 535153070 0 0 1 \n", + "1574 916380060 0 0 1 \n", + "490 528290190 0 0 1 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Condition 1_PosA Condition 1_PosN Condition 1_RRAe \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 1 0 0 \n", + "\n", + " Condition 1_RRAn Condition 1_RRNe Condition 1_RRNn \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 1 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Condition 2_Artery Condition 2_Feedr Condition 2_Norm \\\n", + "Order PID \n", + "144 535153070 0 0 1 \n", + "1574 916380060 0 0 1 \n", + "490 528290190 0 0 1 \n", + "1730 528218050 0 0 1 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Condition 2_PosA Condition 2_PosN Condition 2_RRAe \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 1 0 0 \n", + "\n", + " Condition 2_RRAn Condition 2_RRNe Condition 2_RRNn \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Electrical Enclosed Porch Exter Cond Exter Qual \\\n", + "Order PID \n", + "144 535153070 4 120.0 2 2 \n", + "1574 916380060 4 0.0 2 3 \n", + "490 528290190 4 0.0 2 2 \n", + "1730 528218050 4 0.0 2 3 \n", + "2276 921128030 4 0.0 2 4 \n", + "\n", + " Exterior 1st_AsbShng Exterior 1st_AsphShn \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_BrkComm Exterior 1st_BrkFace \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_CBlock Exterior 1st_CemntBd \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_HdBoard Exterior 1st_ImStucc \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_MetalSd Exterior 1st_Other \\\n", + "Order PID \n", + "144 535153070 1 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 1 0 \n", + "\n", + " Exterior 1st_Plywood Exterior 1st_PreCast \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_Stone Exterior 1st_Stucco \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_VinylSd Exterior 1st_Wd Sdng \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 1 0 \n", + "490 528290190 1 0 \n", + "1730 528218050 1 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 1st_WdShing Exterior 2nd_AsbShng \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_AsphShn Exterior 2nd_BrkComm \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_BrkFace Exterior 2nd_CBlock \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_CemntBd Exterior 2nd_HdBoard \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_ImStucc Exterior 2nd_MetalSd \\\n", + "Order PID \n", + "144 535153070 0 1 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 1 \n", + "\n", + " Exterior 2nd_Other Exterior 2nd_Plywood \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_PreCast Exterior 2nd_Stone \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_Stucco Exterior 2nd_VinylSd \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 1 \n", + "490 528290190 0 1 \n", + "1730 528218050 0 1 \n", + "2276 921128030 0 0 \n", + "\n", + " Exterior 2nd_Wd Sdng Exterior 2nd_WdShing Fence \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Fireplace Qu Fireplaces Foundation_BrkTil \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 3 1 0 \n", + "1730 528218050 4 1 0 \n", + "2276 921128030 4 1 0 \n", + "\n", + " Foundation_CBlock Foundation_PConc Foundation_Slab \\\n", + "Order PID \n", + "144 535153070 1 0 0 \n", + "1574 916380060 0 1 0 \n", + "490 528290190 0 1 0 \n", + "1730 528218050 0 1 0 \n", + "2276 921128030 0 1 0 \n", + "\n", + " Foundation_Stone Foundation_Wood Full Bath Functional \\\n", + "Order PID \n", + "144 535153070 0 0 1 7 \n", + "1574 916380060 0 0 2 7 \n", + "490 528290190 0 0 2 7 \n", + "1730 528218050 0 0 2 7 \n", + "2276 921128030 0 0 2 7 \n", + "\n", + " Garage Area Garage Cars Garage Cond Garage Finish \\\n", + "Order PID \n", + "144 535153070 312.0 1 3 2 \n", + "1574 916380060 788.0 3 3 3 \n", + "490 528290190 400.0 2 3 2 \n", + "1730 528218050 393.0 2 3 3 \n", + "2276 921128030 932.0 3 3 3 \n", + "\n", + " Garage Qual Garage Type_2Types Garage Type_Attchd \\\n", + "Order PID \n", + "144 535153070 3 0 1 \n", + "1574 916380060 3 0 1 \n", + "490 528290190 3 0 0 \n", + "1730 528218050 3 0 1 \n", + "2276 921128030 3 0 1 \n", + "\n", + " Garage Type_Basment Garage Type_BuiltIn \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 1 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Garage Type_CarPort Garage Type_Detchd Garage Type_NA \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Gr Liv Area Half Bath Heating QC Heating_Floor \\\n", + "Order PID \n", + "144 535153070 1194.0 0 2 0 \n", + "1574 916380060 1537.0 0 4 0 \n", + "490 528290190 1430.0 1 4 0 \n", + "1730 528218050 1484.0 1 4 0 \n", + "2276 921128030 1824.0 0 4 0 \n", + "\n", + " Heating_GasA Heating_GasW Heating_Grav Heating_OthW \\\n", + "Order PID \n", + "144 535153070 1 0 0 0 \n", + "1574 916380060 1 0 0 0 \n", + "490 528290190 1 0 0 0 \n", + "1730 528218050 1 0 0 0 \n", + "2276 921128030 1 0 0 0 \n", + "\n", + " Heating_Wall House Style_1.5Fin House Style_1.5Unf \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " House Style_1Story House Style_2.5Fin House Style_2.5Unf \\\n", + "Order PID \n", + "144 535153070 1 0 0 \n", + "1574 916380060 1 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 1 0 0 \n", + "\n", + " House Style_2Story House Style_SFoyer House Style_SLvl \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 1 \n", + "1730 528218050 1 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Kitchen AbvGr Kitchen Qual Land Contour_Bnk \\\n", + "Order PID \n", + "144 535153070 1 2 0 \n", + "1574 916380060 1 3 0 \n", + "490 528290190 1 2 0 \n", + "1730 528218050 1 3 0 \n", + "2276 921128030 1 4 0 \n", + "\n", + " Land Contour_HLS Land Contour_Low Land Contour_Lvl \\\n", + "Order PID \n", + "144 535153070 0 0 1 \n", + "1574 916380060 1 0 0 \n", + "490 528290190 0 0 1 \n", + "1730 528218050 0 0 1 \n", + "2276 921128030 1 0 0 \n", + "\n", + " Land Slope Lot Area Lot Config_Corner Lot Config_CulDSac \\\n", + "Order PID \n", + "144 535153070 2 8760.0 0 0 \n", + "1574 916380060 2 11563.0 0 0 \n", + "490 528290190 2 7750.0 0 0 \n", + "1730 528218050 2 10237.0 0 0 \n", + "2276 921128030 2 12633.0 0 0 \n", + "\n", + " Lot Config_FR2 Lot Config_FR3 Lot Config_Inside Lot Shape \\\n", + "Order PID \n", + "144 535153070 0 0 1 3 \n", + "1574 916380060 0 0 1 2 \n", + "490 528290190 0 0 1 3 \n", + "1730 528218050 0 0 1 3 \n", + "2276 921128030 0 0 1 2 \n", + "\n", + " Low Qual Fin SF MS SubClass_020 MS SubClass_030 \\\n", + "Order PID \n", + "144 535153070 0.0 1 0 \n", + "1574 916380060 0.0 1 0 \n", + "490 528290190 0.0 0 0 \n", + "1730 528218050 0.0 0 0 \n", + "2276 921128030 0.0 1 0 \n", + "\n", + " MS SubClass_040 MS SubClass_045 MS SubClass_050 \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " MS SubClass_060 MS SubClass_070 MS SubClass_075 \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 1 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " MS SubClass_080 MS SubClass_085 MS SubClass_090 \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 1 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " MS SubClass_120 MS SubClass_150 MS SubClass_160 \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " MS SubClass_180 MS SubClass_190 MS Zoning_A MS Zoning_C \\\n", + "Order PID \n", + "144 535153070 0 0 0 0 \n", + "1574 916380060 0 0 0 0 \n", + "490 528290190 0 0 0 0 \n", + "1730 528218050 0 0 0 0 \n", + "2276 921128030 0 0 0 0 \n", + "\n", + " MS Zoning_FV MS Zoning_I MS Zoning_RH MS Zoning_RL \\\n", + "Order PID \n", + "144 535153070 0 0 0 1 \n", + "1574 916380060 0 0 0 1 \n", + "490 528290190 0 0 0 1 \n", + "1730 528218050 0 0 0 1 \n", + "2276 921128030 0 0 0 1 \n", + "\n", + " MS Zoning_RM MS Zoning_RP Mas Vnr Area \\\n", + "Order PID \n", + "144 535153070 0 0 220.0 \n", + "1574 916380060 0 0 258.0 \n", + "490 528290190 0 0 0.0 \n", + "1730 528218050 0 0 0.0 \n", + "2276 921128030 0 0 242.0 \n", + "\n", + " Mas Vnr Type_BrkCmn Mas Vnr Type_BrkFace \\\n", + "Order PID \n", + "144 535153070 0 1 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 1 \n", + "\n", + " Mas Vnr Type_CBlock Mas Vnr Type_None Mas Vnr Type_Stone \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 1 \n", + "490 528290190 0 1 0 \n", + "1730 528218050 0 1 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Misc Feature_Elev Misc Feature_Gar2 Misc Feature_NA \\\n", + "Order PID \n", + "144 535153070 0 0 1 \n", + "1574 916380060 0 0 1 \n", + "490 528290190 0 0 1 \n", + "1730 528218050 0 0 1 \n", + "2276 921128030 0 0 1 \n", + "\n", + " Misc Feature_Othr Misc Feature_Shed Misc Feature_TenC \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Misc Val Mo Sold Neighborhood_Blmngtn \\\n", + "Order PID \n", + "144 535153070 0.0 4 0 \n", + "1574 916380060 0.0 4 0 \n", + "490 528290190 0.0 3 0 \n", + "1730 528218050 0.0 7 0 \n", + "2276 921128030 0.0 9 0 \n", + "\n", + " Neighborhood_Blueste Neighborhood_BrDale \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_BrkSide Neighborhood_ClearCr \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_CollgCr Neighborhood_Crawfor \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_Edwards Neighborhood_Gilbert \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 1 \n", + "1730 528218050 0 1 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_Greens Neighborhood_GrnHill \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_IDOTRR Neighborhood_Landmrk \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_MeadowV Neighborhood_Mitchel \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_NPkVill Neighborhood_NWAmes \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_Names Neighborhood_NoRidge \\\n", + "Order PID \n", + "144 535153070 1 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_NridgHt Neighborhood_OldTown \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_SWISU Neighborhood_Sawyer \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_SawyerW Neighborhood_Somerst \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Neighborhood_StoneBr Neighborhood_Timber \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 1 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 1 \n", + "\n", + " Neighborhood_Veenker Open Porch SF Overall Cond \\\n", + "Order PID \n", + "144 535153070 0 0.0 5 \n", + "1574 916380060 0 26.0 4 \n", + "490 528290190 0 0.0 4 \n", + "1730 528218050 0 72.0 4 \n", + "2276 921128030 0 36.0 4 \n", + "\n", + " Overall Qual Paved Drive Pool Area Pool QC \\\n", + "Order PID \n", + "144 535153070 5 2 0.0 0 \n", + "1574 916380060 7 2 0.0 0 \n", + "490 528290190 6 2 0.0 0 \n", + "1730 528218050 5 2 0.0 0 \n", + "2276 921128030 9 2 0.0 0 \n", + "\n", + " Roof Matl_ClyTile Roof Matl_CompShg Roof Matl_Membran \\\n", + "Order PID \n", + "144 535153070 0 1 0 \n", + "1574 916380060 0 1 0 \n", + "490 528290190 0 1 0 \n", + "1730 528218050 0 1 0 \n", + "2276 921128030 0 1 0 \n", + "\n", + " Roof Matl_Metal Roof Matl_Roll Roof Matl_Tar&Grv \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Roof Matl_WdShake Roof Matl_WdShngl Roof Style_Flat \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Roof Style_Gable Roof Style_Gambrel Roof Style_Hip \\\n", + "Order PID \n", + "144 535153070 0 0 1 \n", + "1574 916380060 0 0 1 \n", + "490 528290190 1 0 0 \n", + "1730 528218050 1 0 0 \n", + "2276 921128030 0 0 1 \n", + "\n", + " Roof Style_Mansard Roof Style_Shed Sale Condition_Abnorml \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Sale Condition_AdjLand Sale Condition_Alloca \\\n", + "Order PID \n", + "144 535153070 0 0 \n", + "1574 916380060 0 0 \n", + "490 528290190 0 0 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Sale Condition_Family Sale Condition_Normal \\\n", + "Order PID \n", + "144 535153070 0 1 \n", + "1574 916380060 0 1 \n", + "490 528290190 0 1 \n", + "1730 528218050 0 0 \n", + "2276 921128030 0 0 \n", + "\n", + " Sale Condition_Partial Sale Type_COD Sale Type_CWD \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 1 0 0 \n", + "2276 921128030 1 0 0 \n", + "\n", + " Sale Type_Con Sale Type_ConLD Sale Type_ConLI \\\n", + "Order PID \n", + "144 535153070 0 0 0 \n", + "1574 916380060 0 0 0 \n", + "490 528290190 0 0 0 \n", + "1730 528218050 0 0 0 \n", + "2276 921128030 0 0 0 \n", + "\n", + " Sale Type_ConLw Sale Type_New Sale Type_Oth Sale Type_VWD \\\n", + "Order PID \n", + "144 535153070 0 0 0 0 \n", + "1574 916380060 0 0 0 0 \n", + "490 528290190 0 0 0 0 \n", + "1730 528218050 0 1 0 0 \n", + "2276 921128030 0 1 0 0 \n", + "\n", + " Sale Type_WD Screen Porch Street_Grvl Street_Pave \\\n", + "Order PID \n", + "144 535153070 1 0.0 0 1 \n", + "1574 916380060 1 0.0 0 1 \n", + "490 528290190 1 0.0 0 1 \n", + "1730 528218050 0 0.0 0 1 \n", + "2276 921128030 0 108.0 0 1 \n", + "\n", + " TotRms AbvGrd Total Bsmt SF Utilities Wood Deck SF \\\n", + "Order PID \n", + "144 535153070 6 1194.0 3 0.0 \n", + "1574 916380060 8 1518.0 3 0.0 \n", + "490 528290190 7 384.0 3 0.0 \n", + "1730 528218050 8 783.0 3 0.0 \n", + "2276 921128030 8 1824.0 3 160.0 \n", + "\n", + " Year Built Year Remod/Add Yr Sold SalePrice \n", + "Order PID \n", + "144 535153070 1959 1959 2010 148000.0 \n", + "1574 916380060 2006 2007 2008 294000.0 \n", + "490 528290190 1999 2000 2009 156000.0 \n", + "1730 528218050 2005 2007 2007 178900.0 \n", + "2276 921128030 2006 2007 2007 392000.0 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Obtain the raw numpy arrays:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "X1 = df1.drop(columns=\"SalePrice\").values\n", + "y1 = df1[\"SalePrice\"].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Improved Data\n", + "\n", + "The DataFrame `df2` holds the data as manually processed in notebooks 2 and 3.\n", + "\n", + "New features like the *years_since_\\** columns were generated or derived from other variables like *has 2nd Flr* (from the continuous *2nd Flr SF*). Further, factor variables were created taking into account patterns in the visualizations. For example, *Bldg Type*'s (from `df1`) five categories were condensed into just three. In summary, `df2` has less than half as many dimensions as `df1` to accomodate for a potential curse of dimensionality." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = load_clean_data(\"data/data_clean_with_transformations_and_factors.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = encode_ordinals(df2)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = shuffle(df2, random_state=random_state)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "MultiIndex: 2883 entries, (2774, 907175030) to (2659, 902305090)\n", + "Columns: 106 entries, 1st Flr SF to SalePrice (box-cox-0.0)\n", + "dtypes: float64(24), int64(82)\n", + "memory usage: 2.4 MB\n" + ] + } + ], + "source": [ + "df2.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
1st Flr SF2nd Flr SF3Ssn PorchBedroom AbvGrBsmt CondBsmt ExposureBsmt Full BathBsmt Half BathBsmt QualBsmt Unf SFBsmtFin SF 1BsmtFin SF 2BsmtFin Type 1BsmtFin Type 2ElectricalEnclosed PorchFenceFireplace QuFireplacesFull BathFunctionalGarage AreaGarage CarsGarage CondGarage FinishGarage QualGr Liv AreaHalf BathKitchen AbvGrKitchen QualLand SlopeLot AreaLot ShapeLow Qual Fin SFMas Vnr AreaMisc ValMo SoldOpen Porch SFOverall CondOverall QualPaved DrivePool AreaPool QCScreen PorchTotRms AbvGrdTotal BathTotal Bsmt SFTotal Porch SFTotal SFTotal SF (box-cox-0.0)UtilitiesWood Deck SFabnormal_saleair_condbuild_type_1Fambuild_type_2Fambuild_type_Twnhsfound_BrkTilfound_CBlockfound_PConchas 2nd Flrhas Bsmthas Fireplacehas Garagehas Poolhas Porchmajor_streetnew_homenhood_Blmngtnnhood_Bluestenhood_BrDalenhood_BrkSidenhood_ClearCrnhood_CollgCrnhood_Crawfornhood_Edwardsnhood_Gilbertnhood_Greensnhood_GrnHillnhood_IDOTRRnhood_Landmrknhood_MeadowVnhood_Mitchelnhood_NPkVillnhood_NWAmesnhood_Namesnhood_NoRidgenhood_NridgHtnhood_OldTownnhood_SWISUnhood_Sawyernhood_SawyerWnhood_Somerstnhood_StoneBrnhood_Timbernhood_Veenkerparkpartial_salerailwayrecently_builtrecently_remodeledremodeledyears_since_builtyears_since_remodeledSalePriceSalePrice (box-cox-0.0)
OrderPID
27749071750301525.00.00.0333104125.01400.00.06140.000027541.023231525.001329100.030.0244.00.0936.04620.000.063.01525.0255.03050.08.0228973219.00110000101010100000001000000000000000000000000011066235000.012.367341
2139907202100907.00.00.0333104399.060.0417.03440.030017308.01323907.0012216300.020.00.00.010.03420.000.052.0876.00.01783.07.48605330.0011000100101000000000100000000000000000000000000003030130000.011.775290
1044527451290483.0504.00.023100396.0387.00.05140.000017264.01313987.011221680.030.0232.00.070.04520.000.041.5483.00.01470.07.29301830.011001010110100000010000000000000000000000000000000373785400.011.355101
6435353031101062.00.00.0331103572.0490.00.05140.000017297.013131062.001228128.020.080.00.020.06520.000.062.01062.00.02124.07.66105630.0011000100101000000000000000000000100000000000000005555131000.011.782953
1595353770901056.00.00.00312030.01056.00.06140.040007576.023131056.002226979.030.00.0600.0656.04520.000.042.01056.0320.02112.07.6553913264.0010100100101010000000000000000000000100000000000003030144000.011.877569
\n", + "
" + ], + "text/plain": [ + " 1st Flr SF 2nd Flr SF 3Ssn Porch Bedroom AbvGr Bsmt Cond \\\n", + "Order PID \n", + "2774 907175030 1525.0 0.0 0.0 3 3 \n", + "2139 907202100 907.0 0.0 0.0 3 3 \n", + "1044 527451290 483.0 504.0 0.0 2 3 \n", + "643 535303110 1062.0 0.0 0.0 3 3 \n", + "159 535377090 1056.0 0.0 0.0 0 3 \n", + "\n", + " Bsmt Exposure Bsmt Full Bath Bsmt Half Bath Bsmt Qual \\\n", + "Order PID \n", + "2774 907175030 3 1 0 4 \n", + "2139 907202100 3 1 0 4 \n", + "1044 527451290 1 0 0 3 \n", + "643 535303110 1 1 0 3 \n", + "159 535377090 1 2 0 3 \n", + "\n", + " Bsmt Unf SF BsmtFin SF 1 BsmtFin SF 2 BsmtFin Type 1 \\\n", + "Order PID \n", + "2774 907175030 125.0 1400.0 0.0 6 \n", + "2139 907202100 399.0 60.0 417.0 3 \n", + "1044 527451290 96.0 387.0 0.0 5 \n", + "643 535303110 572.0 490.0 0.0 5 \n", + "159 535377090 0.0 1056.0 0.0 6 \n", + "\n", + " BsmtFin Type 2 Electrical Enclosed Porch Fence \\\n", + "Order PID \n", + "2774 907175030 1 4 0.0 0 \n", + "2139 907202100 4 4 0.0 3 \n", + "1044 527451290 1 4 0.0 0 \n", + "643 535303110 1 4 0.0 0 \n", + "159 535377090 1 4 0.0 4 \n", + "\n", + " Fireplace Qu Fireplaces Full Bath Functional Garage Area \\\n", + "Order PID \n", + "2774 907175030 0 0 2 7 541.0 \n", + "2139 907202100 0 0 1 7 308.0 \n", + "1044 527451290 0 0 1 7 264.0 \n", + "643 535303110 0 0 1 7 297.0 \n", + "159 535377090 0 0 0 7 576.0 \n", + "\n", + " Garage Cars Garage Cond Garage Finish Garage Qual \\\n", + "Order PID \n", + "2774 907175030 2 3 2 3 \n", + "2139 907202100 1 3 2 3 \n", + "1044 527451290 1 3 1 3 \n", + "643 535303110 1 3 1 3 \n", + "159 535377090 2 3 1 3 \n", + "\n", + " Gr Liv Area Half Bath Kitchen AbvGr Kitchen Qual \\\n", + "Order PID \n", + "2774 907175030 1525.0 0 1 3 \n", + "2139 907202100 907.0 0 1 2 \n", + "1044 527451290 987.0 1 1 2 \n", + "643 535303110 1062.0 0 1 2 \n", + "159 535377090 1056.0 0 2 2 \n", + "\n", + " Land Slope Lot Area Lot Shape Low Qual Fin SF \\\n", + "Order PID \n", + "2774 907175030 2 9100.0 3 0.0 \n", + "2139 907202100 2 16300.0 2 0.0 \n", + "1044 527451290 2 1680.0 3 0.0 \n", + "643 535303110 2 8128.0 2 0.0 \n", + "159 535377090 2 6979.0 3 0.0 \n", + "\n", + " Mas Vnr Area Misc Val Mo Sold Open Porch SF Overall Cond \\\n", + "Order PID \n", + "2774 907175030 244.0 0.0 9 36.0 4 \n", + "2139 907202100 0.0 0.0 1 0.0 3 \n", + "1044 527451290 232.0 0.0 7 0.0 4 \n", + "643 535303110 80.0 0.0 2 0.0 6 \n", + "159 535377090 0.0 600.0 6 56.0 4 \n", + "\n", + " Overall Qual Paved Drive Pool Area Pool QC Screen Porch \\\n", + "Order PID \n", + "2774 907175030 6 2 0.0 0 0.0 \n", + "2139 907202100 4 2 0.0 0 0.0 \n", + "1044 527451290 5 2 0.0 0 0.0 \n", + "643 535303110 5 2 0.0 0 0.0 \n", + "159 535377090 5 2 0.0 0 0.0 \n", + "\n", + " TotRms AbvGrd Total Bath Total Bsmt SF Total Porch SF \\\n", + "Order PID \n", + "2774 907175030 6 3.0 1525.0 255.0 \n", + "2139 907202100 5 2.0 876.0 0.0 \n", + "1044 527451290 4 1.5 483.0 0.0 \n", + "643 535303110 6 2.0 1062.0 0.0 \n", + "159 535377090 4 2.0 1056.0 320.0 \n", + "\n", + " Total SF Total SF (box-cox-0.0) Utilities Wood Deck SF \\\n", + "Order PID \n", + "2774 907175030 3050.0 8.022897 3 219.0 \n", + "2139 907202100 1783.0 7.486053 3 0.0 \n", + "1044 527451290 1470.0 7.293018 3 0.0 \n", + "643 535303110 2124.0 7.661056 3 0.0 \n", + "159 535377090 2112.0 7.655391 3 264.0 \n", + "\n", + " abnormal_sale air_cond build_type_1Fam build_type_2Fam \\\n", + "Order PID \n", + "2774 907175030 0 1 1 0 \n", + "2139 907202100 0 1 1 0 \n", + "1044 527451290 1 1 0 0 \n", + "643 535303110 0 1 1 0 \n", + "159 535377090 0 1 0 1 \n", + "\n", + " build_type_Twnhs found_BrkTil found_CBlock found_PConc \\\n", + "Order PID \n", + "2774 907175030 0 0 0 1 \n", + "2139 907202100 0 0 1 0 \n", + "1044 527451290 1 0 1 0 \n", + "643 535303110 0 0 1 0 \n", + "159 535377090 0 0 1 0 \n", + "\n", + " has 2nd Flr has Bsmt has Fireplace has Garage has Pool \\\n", + "Order PID \n", + "2774 907175030 0 1 0 1 0 \n", + "2139 907202100 0 1 0 1 0 \n", + "1044 527451290 1 1 0 1 0 \n", + "643 535303110 0 1 0 1 0 \n", + "159 535377090 0 1 0 1 0 \n", + "\n", + " has Porch major_street new_home nhood_Blmngtn \\\n", + "Order PID \n", + "2774 907175030 1 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 1 0 0 0 \n", + "\n", + " nhood_Blueste nhood_BrDale nhood_BrkSide nhood_ClearCr \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 1 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_CollgCr nhood_Crawfor nhood_Edwards nhood_Gilbert \\\n", + "Order PID \n", + "2774 907175030 1 0 0 0 \n", + "2139 907202100 1 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_Greens nhood_GrnHill nhood_IDOTRR nhood_Landmrk \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_MeadowV nhood_Mitchel nhood_NPkVill nhood_NWAmes \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_Names nhood_NoRidge nhood_NridgHt nhood_OldTown \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 1 0 0 0 \n", + "159 535377090 0 0 0 1 \n", + "\n", + " nhood_SWISU nhood_Sawyer nhood_SawyerW nhood_Somerst \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_StoneBr nhood_Timber nhood_Veenker park \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " partial_sale railway recently_built recently_remodeled \\\n", + "Order PID \n", + "2774 907175030 0 0 1 1 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " remodeled years_since_built years_since_remodeled \\\n", + "Order PID \n", + "2774 907175030 0 6 6 \n", + "2139 907202100 0 30 30 \n", + "1044 527451290 0 37 37 \n", + "643 535303110 0 55 55 \n", + "159 535377090 0 30 30 \n", + "\n", + " SalePrice SalePrice (box-cox-0.0) \n", + "Order PID \n", + "2774 907175030 235000.0 12.367341 \n", + "2139 907202100 130000.0 11.775290 \n", + "1044 527451290 85400.0 11.355101 \n", + "643 535303110 131000.0 11.782953 \n", + "159 535377090 144000.0 11.877569 " + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df2.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Obtain the raw numpy arrays:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "X2 = df2.drop(columns=[\"SalePrice\", \"SalePrice (box-cox-0.0)\"]).values\n", + "y2 = df2[\"SalePrice\"].values\n", + "y2l = df2[\"SalePrice (box-cox-0.0)\"].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Manual Feature Pre-Selection\n", + "\n", + "Also, notebooks 2 and 3 collect variables that correlate either weakly ($0.33 < \\vert\\rho\\vert < 0.66$) or strongly ($\\vert\\rho\\vert > 0.66$) with the *SalePrice* (or the logarithm thereof) or that reveal \"interesting\" visual patterns. These variables serve as a \"naive\" feature pre-selection." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"data/correlated_variables.json\", \"r\") as file:\n", + " _ = json.loads(file.read())\n", + " weakly_correlated = _[\"weakly_correlated\"]\n", + " strongly_correlated = _[\"strongly_correlated\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"data/interesting_variables.json\", \"r\") as file:\n", + " interesting_variables = json.loads(file.read())" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "pre_selection = sorted(\n", + " set(weakly_correlated + strongly_correlated + interesting_variables)\n", + " & set(df2.columns)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `df3` DataFrame is just a subset of `df2` (71 columns)." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "df3 = df2[pre_selection + TARGET_VARIABLES]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "MultiIndex: 2883 entries, (2774, 907175030) to (2659, 902305090)\n", + "Columns: 71 entries, 1st Flr SF to SalePrice (box-cox-0.0)\n", + "dtypes: float64(13), int64(58)\n", + "memory usage: 1.6 MB\n" + ] + } + ], + "source": [ + "df3.info(verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
1st Flr SFBsmt ExposureBsmt QualBsmtFin SF 1BsmtFin Type 1Fireplace QuFireplacesFull BathGarage AreaGarage CarsGarage CondGarage FinishGarage QualGr Liv AreaHalf BathKitchen QualLot ShapeMas Vnr AreaOverall QualPaved DriveTotRms AbvGrdTotal BathTotal Bsmt SFTotal Porch SFTotal SFTotal SF (box-cox-0.0)Wood Deck SFair_condbuild_type_1Fambuild_type_2Fambuild_type_Twnhshas 2nd Flrhas Fireplacehas Garagemajor_streetnew_homenhood_Blmngtnnhood_Bluestenhood_BrDalenhood_BrkSidenhood_ClearCrnhood_CollgCrnhood_Crawfornhood_Edwardsnhood_Gilbertnhood_Greensnhood_GrnHillnhood_IDOTRRnhood_Landmrknhood_MeadowVnhood_Mitchelnhood_NPkVillnhood_NWAmesnhood_Namesnhood_NoRidgenhood_NridgHtnhood_OldTownnhood_SWISUnhood_Sawyernhood_SawyerWnhood_Somerstnhood_StoneBrnhood_Timbernhood_Veenkerrecently_builtrecently_remodeledremodeledyears_since_builtyears_since_remodeledSalePriceSalePrice (box-cox-0.0)
OrderPID
27749071750301525.0341400.06002541.023231525.0033244.06263.01525.0255.03050.08.022897219.0110000100000001000000000000000000000011066235000.012.367341
2139907202100907.03460.03001308.01323907.00220.04252.0876.00.01783.07.4860530.011000010000000100000000000000000000000003030130000.011.775290
1044527451290483.013387.05001264.01313987.0123232.05241.5483.00.01470.07.2930180.01001101000010000000000000000000000000000373785400.011.355101
6435353031101062.013490.05001297.013131062.002280.05262.01062.00.02124.07.6610560.011000010000000000000000000100000000000005555131000.011.782953
1595353770901056.0131056.06000576.023131056.00230.05242.01056.0320.02112.07.655391264.010100010000000000000000000000100000000003030144000.011.877569
\n", + "
" + ], + "text/plain": [ + " 1st Flr SF Bsmt Exposure Bsmt Qual BsmtFin SF 1 \\\n", + "Order PID \n", + "2774 907175030 1525.0 3 4 1400.0 \n", + "2139 907202100 907.0 3 4 60.0 \n", + "1044 527451290 483.0 1 3 387.0 \n", + "643 535303110 1062.0 1 3 490.0 \n", + "159 535377090 1056.0 1 3 1056.0 \n", + "\n", + " BsmtFin Type 1 Fireplace Qu Fireplaces Full Bath \\\n", + "Order PID \n", + "2774 907175030 6 0 0 2 \n", + "2139 907202100 3 0 0 1 \n", + "1044 527451290 5 0 0 1 \n", + "643 535303110 5 0 0 1 \n", + "159 535377090 6 0 0 0 \n", + "\n", + " Garage Area Garage Cars Garage Cond Garage Finish \\\n", + "Order PID \n", + "2774 907175030 541.0 2 3 2 \n", + "2139 907202100 308.0 1 3 2 \n", + "1044 527451290 264.0 1 3 1 \n", + "643 535303110 297.0 1 3 1 \n", + "159 535377090 576.0 2 3 1 \n", + "\n", + " Garage Qual Gr Liv Area Half Bath Kitchen Qual Lot Shape \\\n", + "Order PID \n", + "2774 907175030 3 1525.0 0 3 3 \n", + "2139 907202100 3 907.0 0 2 2 \n", + "1044 527451290 3 987.0 1 2 3 \n", + "643 535303110 3 1062.0 0 2 2 \n", + "159 535377090 3 1056.0 0 2 3 \n", + "\n", + " Mas Vnr Area Overall Qual Paved Drive TotRms AbvGrd \\\n", + "Order PID \n", + "2774 907175030 244.0 6 2 6 \n", + "2139 907202100 0.0 4 2 5 \n", + "1044 527451290 232.0 5 2 4 \n", + "643 535303110 80.0 5 2 6 \n", + "159 535377090 0.0 5 2 4 \n", + "\n", + " Total Bath Total Bsmt SF Total Porch SF Total SF \\\n", + "Order PID \n", + "2774 907175030 3.0 1525.0 255.0 3050.0 \n", + "2139 907202100 2.0 876.0 0.0 1783.0 \n", + "1044 527451290 1.5 483.0 0.0 1470.0 \n", + "643 535303110 2.0 1062.0 0.0 2124.0 \n", + "159 535377090 2.0 1056.0 320.0 2112.0 \n", + "\n", + " Total SF (box-cox-0.0) Wood Deck SF air_cond \\\n", + "Order PID \n", + "2774 907175030 8.022897 219.0 1 \n", + "2139 907202100 7.486053 0.0 1 \n", + "1044 527451290 7.293018 0.0 1 \n", + "643 535303110 7.661056 0.0 1 \n", + "159 535377090 7.655391 264.0 1 \n", + "\n", + " build_type_1Fam build_type_2Fam build_type_Twnhs \\\n", + "Order PID \n", + "2774 907175030 1 0 0 \n", + "2139 907202100 1 0 0 \n", + "1044 527451290 0 0 1 \n", + "643 535303110 1 0 0 \n", + "159 535377090 0 1 0 \n", + "\n", + " has 2nd Flr has Fireplace has Garage major_street \\\n", + "Order PID \n", + "2774 907175030 0 0 1 0 \n", + "2139 907202100 0 0 1 0 \n", + "1044 527451290 1 0 1 0 \n", + "643 535303110 0 0 1 0 \n", + "159 535377090 0 0 1 0 \n", + "\n", + " new_home nhood_Blmngtn nhood_Blueste nhood_BrDale \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 1 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_BrkSide nhood_ClearCr nhood_CollgCr nhood_Crawfor \\\n", + "Order PID \n", + "2774 907175030 0 0 1 0 \n", + "2139 907202100 0 0 1 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_Edwards nhood_Gilbert nhood_Greens nhood_GrnHill \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_IDOTRR nhood_Landmrk nhood_MeadowV nhood_Mitchel \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_NPkVill nhood_NWAmes nhood_Names nhood_NoRidge \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 1 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_NridgHt nhood_OldTown nhood_SWISU nhood_Sawyer \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 1 0 0 \n", + "\n", + " nhood_SawyerW nhood_Somerst nhood_StoneBr nhood_Timber \\\n", + "Order PID \n", + "2774 907175030 0 0 0 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " nhood_Veenker recently_built recently_remodeled remodeled \\\n", + "Order PID \n", + "2774 907175030 0 1 1 0 \n", + "2139 907202100 0 0 0 0 \n", + "1044 527451290 0 0 0 0 \n", + "643 535303110 0 0 0 0 \n", + "159 535377090 0 0 0 0 \n", + "\n", + " years_since_built years_since_remodeled SalePrice \\\n", + "Order PID \n", + "2774 907175030 6 6 235000.0 \n", + "2139 907202100 30 30 130000.0 \n", + "1044 527451290 37 37 85400.0 \n", + "643 535303110 55 55 131000.0 \n", + "159 535377090 30 30 144000.0 \n", + "\n", + " SalePrice (box-cox-0.0) \n", + "Order PID \n", + "2774 907175030 12.367341 \n", + "2139 907202100 11.775290 \n", + "1044 527451290 11.355101 \n", + "643 535303110 11.782953 \n", + "159 535377090 11.877569 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df3.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Obtain the raw numpy arrays:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "X3 = df3.drop(columns=[\"SalePrice\", \"SalePrice (box-cox-0.0)\"]).values\n", + "y3 = df3[\"SalePrice\"].values\n", + "y3l = df3[\"SalePrice (box-cox-0.0)\"].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Re-usable Components\n", + "\n", + "Define a function to run k-fold cross validation to obtain unbiased estimators for the following scores / errors:\n", + "- Bias\n", + "- Mean Absolute Error\n", + "- Maximum Deviation (just to see the worst case prediction of a model)\n", + "- R2 (coefficient of determination)\n", + "- Root Mean Squared Error (default for comparison)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "def cross_validation(X, y, *, model, k=10, log=False, desc=None):\n", + " \"\"\"Perform a k-fold cross validation.\"\"\"\n", + " bias, mae, max_dev, r2, rmse = [], [], [], [], []\n", + " # Iterate over the k folds.\n", + " for train, test in progress_bar(KFold(n_splits=k).split(X), desc=desc, total=k):\n", + " model.fit(X[train], y[train])\n", + " y_pred = model.predict(X[test])\n", + " # If the sales price is provided on a log scale,\n", + " # take the exponent first so that scores and\n", + " # errors are comparable to the non-logged counterparts.\n", + " if log:\n", + " y_true, y_pred = np.exp(y[test]), np.exp(y_pred) \n", + " else:\n", + " y_true, y_pred = y[test], y_pred\n", + " # Collect the scores/errors for each fold.\n", + " bias.append(bias_score(y_true, y_pred))\n", + " mae.append(mean_absolute_error(y_true, y_pred))\n", + " max_dev.append(max_deviation(y_true, y_pred))\n", + " r2.append(r2_score(y_true, y_pred))\n", + " rmse.append(mean_squared_error(y_true, y_pred))\n", + " # Round for convenience.\n", + " return {\n", + " \"bias\": np.round(np.mean(bias)),\n", + " \"mae\": np.round(np.mean(mae)),\n", + " \"max_dev\": np.round(np.mean(max_dev)),\n", + " \"r2\": np.round(np.mean(r2), 3),\n", + " \"rmse\": np.round(np.sqrt(np.mean(rmse))),\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use one dictionary to store all the results in a systematic way." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "results = {}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Linear Regression\n", + "\n", + "A plain OLS regression model serves as the base case for benchmarking." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "lm = LinearRegression()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Original Data\n", + "\n", + "Given the unprocessed data, the linear model is not able to make a good fit at all." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "718a516b1cb64ff188ca952e99fc11a0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': 43942720.0,\n", + " 'mae': 50106151.0,\n", + " 'max_dev': 13358158031.0,\n", + " 'r2': -664342652.78,\n", + " 'rmse': 2056998619.0}" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[('lm','o')] = cross_validation(X1, y1, model=lm)\n", + "results[('lm','o')]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Improved Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6bf68bfa85504d30ab2ecd507dccfb6c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -63.0,\n", + " 'mae': 15393.0,\n", + " 'max_dev': 122981.0,\n", + " 'r2': 0.918,\n", + " 'rmse': 21992.0}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[('lm','i')] = cross_validation(X2, y2, model=lm)\n", + "results[('lm','i')]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "22c04a812ac34a41848040d697db5395", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -887.0,\n", + " 'mae': 13090.0,\n", + " 'max_dev': 115077.0,\n", + " 'r2': 0.935,\n", + " 'rmse': 19526.0}" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[('lm','il')] = cross_validation(X2, y2l, model=lm, log=True)\n", + "results[('lm','il')]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Improved Data with pre-selected Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6076f36e3abc4e808116eb736ef1c981", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': 140914.0,\n", + " 'mae': 1144261.0,\n", + " 'max_dev': 254073652.0,\n", + " 'r2': -300003.791,\n", + " 'rmse': 38864649.0}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[('lm','p')] = cross_validation(X3, y3, model=lm)\n", + "results[('lm','p')]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "20342cfdbd1d40ee95e7c2391da5d459", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -1355.0,\n", + " 'mae': 14751.0,\n", + " 'max_dev': 149008.0,\n", + " 'r2': 0.913,\n", + " 'rmse': 22606.0}" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[('lm','pl')] = cross_validation(X3, y3l, model=lm, log=True)\n", + "results[('lm','pl')]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LASSO" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "tol = 0.1\n", + "grid_search = GridSearchCV(\n", + " estimator=Lasso(tol=tol, selection=\"random\", random_state=random_state),\n", + " param_grid={\"alpha\": [2 ** x for x in range(-8, 4)] + list(range(12, 65, 4))},\n", + " cv=KFold(n_splits=4),\n", + " n_jobs=-1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Original Data" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "20" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X1, y1)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "275f6d85a1da40e7ae8a123a8e9a2fd7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': 278.0,\n", + " 'mae': 20892.0,\n", + " 'max_dev': 269314.0,\n", + " 'r2': 0.817,\n", + " 'rmse': 33496.0}" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"lasso\", \"o\")] = cross_validation(X1, y1, model=Lasso(alpha=alpha, tol=tol))\n", + "results[(\"lasso\", \"o\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Improved Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "32" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X2, y2)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d58394144f554d13a8bf430baad544b7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -41.0,\n", + " 'mae': 17545.0,\n", + " 'max_dev': 135602.0,\n", + " 'r2': 0.897,\n", + " 'rmse': 24586.0}" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"lasso\", \"i\")] = cross_validation(X2, y2, model=Lasso(alpha=alpha, tol=tol))\n", + "results[(\"lasso\", \"i\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.00390625" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X2, y2l)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fc8fcc1e4e1b4cc4a46b7cb58a401a24", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -868.0,\n", + " 'mae': 14664.0,\n", + " 'max_dev': 126763.0,\n", + " 'r2': 0.921,\n", + " 'rmse': 21641.0}" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"lasso\", \"il\")] = cross_validation(X2, y2l, model=Lasso(alpha=alpha, tol=tol), log=True)\n", + "results[(\"lasso\", \"il\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Improved Data with pre-selected Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "52" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X3, y3)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "84cc328896c148a9bc4f2d6fff9d7e66", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -57.0,\n", + " 'mae': 20634.0,\n", + " 'max_dev': 152574.0,\n", + " 'r2': 0.859,\n", + " 'rmse': 28793.0}" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"lasso\", \"p\")] = cross_validation(X3, y3, model=Lasso(alpha=alpha, tol=tol))\n", + "results[(\"lasso\", \"p\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.00390625" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X3, y3l)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "98653a7e2a7d493b81431ffe1bb05424", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -1041.0,\n", + " 'mae': 15805.0,\n", + " 'max_dev': 131190.0,\n", + " 'r2': 0.91,\n", + " 'rmse': 23041.0}" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"lasso\", \"pl\")] = cross_validation(X3, y3l, model=Lasso(alpha=alpha, tol=tol), log=True)\n", + "results[(\"lasso\", \"pl\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Ridge Regression" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "grid_search = GridSearchCV(\n", + " estimator=Ridge(),\n", + " param_grid={\"alpha\": [2 ** x for x in range(-8, 4)] + list(range(12, 65, 4))},\n", + " cv=KFold(n_splits=4),\n", + " n_jobs=-1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Original Data" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.125" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X1, y1)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a88a9298736d4dbc9de2823db4f668a3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': 152.0,\n", + " 'mae': 17064.0,\n", + " 'max_dev': 263561.0,\n", + " 'r2': 0.853,\n", + " 'rmse': 29970.0}" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"ridge\", \"o\")] = cross_validation(X1, y1, model=Ridge(alpha=alpha))\n", + "results[(\"ridge\", \"o\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Improved Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.25" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X2, y2)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d37e164bda314ddb8282044d0976efcd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -49.0,\n", + " 'mae': 15371.0,\n", + " 'max_dev': 123212.0,\n", + " 'r2': 0.918,\n", + " 'rmse': 21971.0}" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"ridge\", \"i\")] = cross_validation(X2, y2, model=Ridge(alpha=alpha))\n", + "results[(\"ridge\", \"i\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X2, y2l)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0faab768c5504fb5997b462817f85ae7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -868.0,\n", + " 'mae': 13065.0,\n", + " 'max_dev': 114246.0,\n", + " 'r2': 0.936,\n", + " 'rmse': 19442.0}" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"ridge\", \"il\")] = cross_validation(X2, y2l, model=Ridge(alpha=alpha), log=True)\n", + "results[(\"ridge\", \"il\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Improved Data with pre-selected Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X3, y3)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2900ffb411e146dfa03672b755adc7ea", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -51.0,\n", + " 'mae': 16471.0,\n", + " 'max_dev': 126394.0,\n", + " 'r2': 0.908,\n", + " 'rmse': 23255.0}" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"ridge\", \"p\")] = cross_validation(X3, y3, model=Ridge(alpha=alpha))\n", + "results[(\"ridge\", \"p\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_search.fit(X3, y3l)\n", + "alpha = grid_search.best_params_[\"alpha\"]\n", + "alpha" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fcad8f88d4cb404389747b9e185f204f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -1114.0,\n", + " 'mae': 14457.0,\n", + " 'max_dev': 123534.0,\n", + " 'r2': 0.924,\n", + " 'rmse': 21169.0}" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"ridge\", \"pl\")] = cross_validation(X3, y3l, model=Ridge(alpha=alpha), log=True)\n", + "results[(\"ridge\", \"pl\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Random Forest" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "rf = RandomForestRegressor(\n", + " n_estimators=500,\n", + " n_jobs=-1, random_state=random_state\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Original Data" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5f57ede479174cacb7dd5be49972d446", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -26.0,\n", + " 'mae': 15322.0,\n", + " 'max_dev': 164505.0,\n", + " 'r2': 0.898,\n", + " 'rmse': 25354.0}" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"rf\", \"o\")] = cross_validation(X1, y1, model=rf)\n", + "results[(\"rf\", \"o\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Improved Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9bc68055dd914a26a96ac1882e0a6c90", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -75.0,\n", + " 'mae': 14957.0,\n", + " 'max_dev': 130725.0,\n", + " 'r2': 0.911,\n", + " 'rmse': 22960.0}" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"rf\", \"i\")] = cross_validation(X2, y2, model=rf)\n", + "results[(\"rf\", \"i\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "918c0c34b8e84d46bc2f0007114fd14f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -2061.0,\n", + " 'mae': 15023.0,\n", + " 'max_dev': 134360.0,\n", + " 'r2': 0.908,\n", + " 'rmse': 23260.0}" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"rf\", \"il\")] = cross_validation(X2, y2l, model=rf, log=True)\n", + "results[(\"rf\", \"il\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Improved Data with pre-selected Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### a) Normal Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5fe3529d68734889921e35d5c29bf5ee", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -430.0,\n", + " 'mae': 15447.0,\n", + " 'max_dev': 133420.0,\n", + " 'r2': 0.906,\n", + " 'rmse': 23572.0}" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"rf\", \"p\")] = cross_validation(X3, y3, model=rf)\n", + "results[(\"rf\", \"p\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### b) Log Scale" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "39aa0c8e2bc44ba294aee4711afeaa4c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "{'bias': -2343.0,\n", + " 'mae': 15626.0,\n", + " 'max_dev': 134375.0,\n", + " 'r2': 0.902,\n", + " 'rmse': 23973.0}" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results[(\"rf\", \"pl\")] = cross_validation(X3, y3l, model=rf, log=True)\n", + "results[(\"rf\", \"pl\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analysis of Results\n", + "\n", + "This notebook did not focus on hyper-parameter optimization. Therefore, the predictions by Lasso, Ridge, and the Random Forest can potentially be improved by fine-graining the grid search even more.\n", + "\n", + "In general, the manually \"improved\" data clearly outperform the data that were only cleaned with the minimum effort. Also, the result suggests to allow the model to select its features. The manually pre-selected features perform well but not as good as the full feature set." + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "def scores_by_source(source, score=\"rmse\", *, ascending=True):\n", + " rv = [\n", + " (model, scores[score])\n", + " for (model, data_source), scores in results.items()\n", + " if data_source == source\n", + " ]\n", + " return sorted(rv, key=lambda x: x[1], reverse=(not ascending))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Root Mean Squared Error" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('rf', 25354.0), ('ridge', 29970.0), ('lasso', 33496.0), ('lm', 2056998619.0)]" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"o\", \"rmse\")" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 21971.0), ('lm', 21992.0), ('rf', 22960.0), ('lasso', 24586.0)]" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"i\", \"rmse\")" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 19442.0), ('lm', 19526.0), ('lasso', 21641.0), ('rf', 23260.0)]" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"il\", \"rmse\")" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 23255.0), ('rf', 23572.0), ('lasso', 28793.0), ('lm', 38864649.0)]" + ] + }, + "execution_count": 67, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"p\", \"rmse\")" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 21169.0), ('lm', 22606.0), ('lasso', 23041.0), ('rf', 23973.0)]" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"pl\", \"rmse\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### R2" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('rf', 0.898), ('ridge', 0.853), ('lasso', 0.817), ('lm', -664342652.78)]" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"o\", \"r2\", ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('lm', 0.918), ('ridge', 0.918), ('rf', 0.911), ('lasso', 0.897)]" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"i\", \"r2\", ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 0.936), ('lm', 0.935), ('lasso', 0.921), ('rf', 0.908)]" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"il\", \"r2\", ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 0.908), ('rf', 0.906), ('lasso', 0.859), ('lm', -300003.791)]" + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"p\", \"r2\", ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('ridge', 0.924), ('lm', 0.913), ('lasso', 0.91), ('rf', 0.902)]" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_by_source(\"pl\", \"r2\", ascending=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Pipfile b/Pipfile index 37679b7..b3ba18d 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ seaborn = "*" missingno = "*" sklearn = "*" statsmodels = "*" +tqdm = "*" [dev-packages] black = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 2d15b42..b33e09e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9179f1a05ed984d241ed8fcfa9c66c135100dbc2137f22627ac15977ced9ce87" + "sha256": "cd46097e2ebd23accf453936aef1b898dfbf46ff9f2642f494da0e041b7a5992" }, "pipfile-spec": 6, "requires": { @@ -624,6 +624,14 @@ "markers": "python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.2.*'", "version": "==5.1" }, + "tqdm": { + "hashes": [ + "sha256:5ef526702c0d265d5a960a3b27f3971fac13c26cf0fb819294bfa71fc6026c88", + "sha256:a3364bd83ce4777320b862e3c8a93d7da91e20a95f06ef79bed7dd71c654cafa" + ], + "index": "pypi", + "version": "==4.25.0" + }, "traitlets": { "hashes": [ "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", diff --git a/utils.py b/utils.py index 6ffe4a0..072b35f 100644 --- a/utils.py +++ b/utils.py @@ -24,10 +24,10 @@ Implementation Note: This file defines the "constants" it exports dynamically. This is a bit advanced but intentional! """ -# pragma pylint:disable=global-statement import re +import numpy as np import pandas as pd import requests import tabulate @@ -140,19 +140,6 @@ def _populate_dicts_and_lists(): dictionaries and lists are considered derived from it and thus considered "secondary". """ - global ALL_VARIABLES - global CONTINUOUS_COLUMNS - global CONTINUOUS_VARIABLES - global DISCRETE_COLUMNS - global DISCRETE_VARIABLES - global NUMERIC_COLUMNS - global NUMERIC_VARIABLES - global NOMINAL_COLUMNS - global NOMINAL_VARIABLES - global ORDINAL_COLUMNS - global ORDINAL_VARIABLES - global LABEL_COLUMNS - global LABEL_VARIABLES # The global data structures are not re-assigned to so as to keep all # references in the Jupyter notebooks alive. Instead, they are emptied # and re-filled. @@ -175,6 +162,11 @@ def _populate_dicts_and_lists(): } ) DISCRETE_VARIABLES[:] = sorted(DISCRETE_COLUMNS) + FACTOR_VARIABLES[:] = [ + key + for (key, value) in ALL_COLUMNS.items() + if value["type"] == "factor" + ] NUMERIC_COLUMNS.clear() NUMERIC_COLUMNS.update({**CONTINUOUS_COLUMNS, **DISCRETE_COLUMNS}) NUMERIC_VARIABLES[:] = sorted(NUMERIC_COLUMNS) @@ -203,7 +195,6 @@ def _populate_dicts_and_lists(): def _rename_column(old_name, new_name): """Change the name of a column.""" - global ALL_COLUMNS ALL_COLUMNS[new_name] = ALL_COLUMNS[old_name] del ALL_COLUMNS[old_name] @@ -249,7 +240,6 @@ def update_column_descriptions(columns_to_be_kept, *, correct_columns=False): After dropping some columns from the DataFrame, these removals must be propagated to the helper data structures defined in this module. """ - global ALL_COLUMNS if correct_columns: correct_column_names(columns_to_be_kept, repopulate=False) columns_to_be_removed = list(set(ALL_COLUMNS) - set(columns_to_be_kept)) @@ -279,17 +269,15 @@ def print_column_list(subset=None): print(tabulate.tabulate(sorted(columns), tablefmt="plain")) -def load_clean_data(*, path=None, ordinal_encoded=False): +def load_clean_data(path=None): """Return the clean project data as a pandas DataFrame. This utility function ensures that each column is cast to its correct type. - It takes the following optional keyword-only arguments: - - 'path': path to the clean CSV file (defaults to "data/data_clean.csv") - - 'ordinal_encoded': can be set to True to obtain the ordinal columns that - are already encoded into ordered integers + It takes an optional path argument to a clean CSV file (defaults to + "data/data_clean.csv"). - The target variable "SalePrice" is always included as the last column. + The target variables are always included as the last columns. Implementation Notes: @@ -331,22 +319,30 @@ def load_clean_data(*, path=None, ordinal_encoded=False): derived_columns = set(df.columns) - set(ALL_VARIABLES + TARGET_VARIABLES) if derived_columns: for column in derived_columns: - # All derived variables are numeric (including factors). - df[column] = df[column].astype(float) # Check if the derived variable is a target variable. for target in TARGET_VARIABLES[:]: if column.startswith(target): + df[column] = df[column].astype(float) TARGET_VARIABLES.append(column) break else: + df[column] = df[column].astype(float) + is_int = (df[column] == df[column].astype(int)).all() + n_unique = len(df[column].unique()) + if is_int & (n_unique == 2): + df[column] = df[column].astype(int) + type_ = "factor" + elif is_int & (n_unique < 150): + df[column] = df[column].astype(int) + type_ = "discrete" + else: + df[column] = df[column].astype(float) + type_ = "continuous" ALL_COLUMNS[column] = { - "type": "continuous", + "type": type_, "description": "derived variable", } _populate_dicts_and_lists() - # Use integer encoding for ordinal variables. - if ordinal_encoded: - df = encode_ordinals(df) return df @@ -356,10 +352,24 @@ def encode_ordinals(df): df = df.copy() for column in df.columns: if column in ORDINAL_VARIABLES: - df[column] = df[column].cat.codes + df[column] = df[column].cat.codes.astype(int) return df +def bias_score(y_true, y_pred): + """Determine the bias of a prediction.""" + assert y_true.shape == y_pred.shape + assert y_true.ndim == 1 + return np.mean(y_pred - y_true) + + +def max_deviation(y_true, y_pred): + """Determine the maximum deviation of a prediction.""" + assert y_true.shape == y_pred.shape + assert y_true.ndim == 1 + return np.max(np.abs(y_pred - y_true)) + + # This code is executed once during import time and # populates all the "constants" directly or indirectly. _extract_meta_data(_get_lines())