{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Predictive Models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \"Housekeeping\"" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.linear_model import Lasso, LinearRegression, Ridge\n", "from sklearn.metrics import (\n", " make_scorer,\n", " mean_absolute_error,\n", " mean_squared_error,\n", " r2_score,\n", ")\n", "from sklearn.model_selection import GridSearchCV, KFold\n", "from sklearn.svm import SVR\n", "from sklearn.utils import shuffle\n", "from tqdm import tqdm as progress_bar\n", "\n", "from utils import (\n", " CONTINUOUS_VARIABLES,\n", " DISCRETE_VARIABLES,\n", " NOMINAL_VARIABLES,\n", " ORDINAL_VARIABLES,\n", " TARGET_VARIABLES,\n", " bias_score,\n", " encode_ordinals,\n", " load_clean_data,\n", " max_deviation,\n", ")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "random_state = np.random.RandomState(42)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "pd.set_option(\"display.max_columns\", 250)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load the Data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1. Original Data\n", "\n", "The DataFrame `df1` holds the cleaned data from notebook 1 with the all the nominal and ordinal features automatically translated to factor variables and ordered integer values." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "df1 = load_clean_data(\"data/data_clean.csv\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This cell basically **replaces** all the manual work that went into generating new and identifying \"interesting\" features in notebooks 2 and 3." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "df1 = pd.concat([\n", " df1[CONTINUOUS_VARIABLES + DISCRETE_VARIABLES + ORDINAL_VARIABLES + TARGET_VARIABLES],\n", " pd.get_dummies(df1[NOMINAL_VARIABLES], dtype=int),\n", "], axis=1)\n", "# Re-order the columns for convenience.\n", "df1 = df1[sorted(set(df1.columns) - set(TARGET_VARIABLES)) + TARGET_VARIABLES]" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "df1 = encode_ordinals(df1)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "df1 = shuffle(df1, random_state=random_state)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "MultiIndex: 2898 entries, (144, 535153070) to (867, 907253130)\n", "Columns: 248 entries, 1st Flr SF to SalePrice\n", "dtypes: float64(19), int64(229)\n", "memory usage: 5.7 MB\n" ] } ], "source": [ "df1.info()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
1st Flr SF2nd Flr SF3Ssn PorchAlley_GrvlAlley_NAAlley_PaveBedroom AbvGrBldg Type_1FamBldg Type_2FmConBldg Type_DuplxBldg Type_TwnhsEBldg Type_TwnhsIBsmt CondBsmt ExposureBsmt Full BathBsmt Half BathBsmt QualBsmt Unf SFBsmtFin SF 1BsmtFin SF 2BsmtFin Type 1BsmtFin Type 2Central Air_NCentral Air_YCondition 1_ArteryCondition 1_FeedrCondition 1_NormCondition 1_PosACondition 1_PosNCondition 1_RRAeCondition 1_RRAnCondition 1_RRNeCondition 1_RRNnCondition 2_ArteryCondition 2_FeedrCondition 2_NormCondition 2_PosACondition 2_PosNCondition 2_RRAeCondition 2_RRAnCondition 2_RRNeCondition 2_RRNnElectricalEnclosed PorchExter CondExter QualExterior 1st_AsbShngExterior 1st_AsphShnExterior 1st_BrkCommExterior 1st_BrkFaceExterior 1st_CBlockExterior 1st_CemntBdExterior 1st_HdBoardExterior 1st_ImStuccExterior 1st_MetalSdExterior 1st_OtherExterior 1st_PlywoodExterior 1st_PreCastExterior 1st_StoneExterior 1st_StuccoExterior 1st_VinylSdExterior 1st_Wd SdngExterior 1st_WdShingExterior 2nd_AsbShngExterior 2nd_AsphShnExterior 2nd_BrkCommExterior 2nd_BrkFaceExterior 2nd_CBlockExterior 2nd_CemntBdExterior 2nd_HdBoardExterior 2nd_ImStuccExterior 2nd_MetalSdExterior 2nd_OtherExterior 2nd_PlywoodExterior 2nd_PreCastExterior 2nd_StoneExterior 2nd_StuccoExterior 2nd_VinylSdExterior 2nd_Wd SdngExterior 2nd_WdShingFenceFireplace QuFireplacesFoundation_BrkTilFoundation_CBlockFoundation_PConcFoundation_SlabFoundation_StoneFoundation_WoodFull BathFunctionalGarage AreaGarage CarsGarage CondGarage FinishGarage QualGarage Type_2TypesGarage Type_AttchdGarage Type_BasmentGarage Type_BuiltInGarage Type_CarPortGarage Type_DetchdGarage Type_NAGr Liv AreaHalf BathHeating QCHeating_FloorHeating_GasAHeating_GasWHeating_GravHeating_OthWHeating_WallHouse Style_1.5FinHouse Style_1.5UnfHouse Style_1StoryHouse Style_2.5FinHouse Style_2.5UnfHouse Style_2StoryHouse Style_SFoyerHouse Style_SLvlKitchen AbvGrKitchen QualLand Contour_BnkLand Contour_HLSLand Contour_LowLand Contour_LvlLand SlopeLot AreaLot Config_CornerLot Config_CulDSacLot Config_FR2Lot Config_FR3Lot Config_InsideLot ShapeLow Qual Fin SFMS SubClass_020MS SubClass_030MS SubClass_040MS SubClass_045MS SubClass_050MS SubClass_060MS SubClass_070MS SubClass_075MS SubClass_080MS SubClass_085MS SubClass_090MS SubClass_120MS SubClass_150MS SubClass_160MS SubClass_180MS SubClass_190MS Zoning_AMS Zoning_CMS Zoning_FVMS Zoning_IMS Zoning_RHMS Zoning_RLMS Zoning_RMMS Zoning_RPMas Vnr AreaMas Vnr Type_BrkCmnMas Vnr Type_BrkFaceMas Vnr Type_CBlockMas Vnr Type_NoneMas Vnr Type_StoneMisc Feature_ElevMisc Feature_Gar2Misc Feature_NAMisc Feature_OthrMisc Feature_ShedMisc Feature_TenCMisc ValMo SoldNeighborhood_BlmngtnNeighborhood_BluesteNeighborhood_BrDaleNeighborhood_BrkSideNeighborhood_ClearCrNeighborhood_CollgCrNeighborhood_CrawforNeighborhood_EdwardsNeighborhood_GilbertNeighborhood_GreensNeighborhood_GrnHillNeighborhood_IDOTRRNeighborhood_LandmrkNeighborhood_MeadowVNeighborhood_MitchelNeighborhood_NPkVillNeighborhood_NWAmesNeighborhood_NamesNeighborhood_NoRidgeNeighborhood_NridgHtNeighborhood_OldTownNeighborhood_SWISUNeighborhood_SawyerNeighborhood_SawyerWNeighborhood_SomerstNeighborhood_StoneBrNeighborhood_TimberNeighborhood_VeenkerOpen Porch SFOverall CondOverall QualPaved DrivePool AreaPool QCRoof Matl_ClyTileRoof Matl_CompShgRoof Matl_MembranRoof Matl_MetalRoof Matl_RollRoof Matl_Tar&GrvRoof Matl_WdShakeRoof Matl_WdShnglRoof Style_FlatRoof Style_GableRoof Style_GambrelRoof Style_HipRoof Style_MansardRoof Style_ShedSale Condition_AbnormlSale Condition_AdjLandSale Condition_AllocaSale Condition_FamilySale Condition_NormalSale Condition_PartialSale Type_CODSale Type_CWDSale Type_ConSale Type_ConLDSale Type_ConLISale Type_ConLwSale Type_NewSale Type_OthSale Type_VWDSale Type_WDScreen PorchStreet_GrvlStreet_PaveTotRms AbvGrdTotal Bsmt SFUtilitiesWood Deck SFYear BuiltYear Remod/AddYr SoldSalePrice
OrderPID
1445351530701194.00.00.0010310000311031194.00.00.011010010000000010000004120.022000000001000000000000000010000000000001000017312.0132301000001194.0020100000010000012000128760.00000130.0100000000000000000000100220.0010000010000.0400000000000000000100000000000.05520.000100000000010000001000000000010.00161194.030.0195919592010148000.0
15749163800601537.00.00.001031000034105482.01036.00.0610100100000000100000040.023000000000000001000000000000000010000000100027788.0333301000001537.00401000000100000130100211563.00000120.0100000000000000000000100258.0000010010000.04000000000000000000000000001026.04720.000100000000010000001000000000010.00181518.030.0200620072008294000.0
490528290190774.0656.00.001031000031004384.00.00.0110100100000000100000040.022000000000000001000000000000000010003100100027400.0232300010001430.0140100000000000112000127750.00000130.00000000010000000000001000.0000100010000.0300000000100000000000000000000.04620.000100000001000000001000000000010.0017384.030.0199920002009156000.0
1730528218050783.0701.00.001031000031004783.00.00.0110100000010000100000040.023000000000000001000000000000000010004100100027393.0233301000001484.01401000000000100130001210237.00000130.00000010000000000000001000.0000100010000.07000000001000000000000000000072.04520.000100000001000000000100000010000.0018783.030.0200520072007178900.0
22769211280301824.00.00.0010310000340051824.00.00.0110100010000000010000040.024000000001000000000000000010000000004100100027932.0333301000001824.00401000000100000140100212633.00000120.0100000000000000000000100242.0010000010000.09000000000000000000000000001036.04920.00010000000001000000010000001000108.00181824.03160.0200620072007392000.0
\n", "
" ], "text/plain": [ " 1st Flr SF 2nd Flr SF 3Ssn Porch Alley_Grvl Alley_NA \\\n", "Order PID \n", "144 535153070 1194.0 0.0 0.0 0 1 \n", "1574 916380060 1537.0 0.0 0.0 0 1 \n", "490 528290190 774.0 656.0 0.0 0 1 \n", "1730 528218050 783.0 701.0 0.0 0 1 \n", "2276 921128030 1824.0 0.0 0.0 0 1 \n", "\n", " Alley_Pave Bedroom AbvGr Bldg Type_1Fam Bldg Type_2FmCon \\\n", "Order PID \n", "144 535153070 0 3 1 0 \n", "1574 916380060 0 3 1 0 \n", "490 528290190 0 3 1 0 \n", "1730 528218050 0 3 1 0 \n", "2276 921128030 0 3 1 0 \n", "\n", " Bldg Type_Duplx Bldg Type_TwnhsE Bldg Type_TwnhsI \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Bsmt Cond Bsmt Exposure Bsmt Full Bath Bsmt Half Bath \\\n", "Order PID \n", "144 535153070 3 1 1 0 \n", "1574 916380060 3 4 1 0 \n", "490 528290190 3 1 0 0 \n", "1730 528218050 3 1 0 0 \n", "2276 921128030 3 4 0 0 \n", "\n", " Bsmt Qual Bsmt Unf SF BsmtFin SF 1 BsmtFin SF 2 \\\n", "Order PID \n", "144 535153070 3 1194.0 0.0 0.0 \n", "1574 916380060 5 482.0 1036.0 0.0 \n", "490 528290190 4 384.0 0.0 0.0 \n", "1730 528218050 4 783.0 0.0 0.0 \n", "2276 921128030 5 1824.0 0.0 0.0 \n", "\n", " BsmtFin Type 1 BsmtFin Type 2 Central Air_N Central Air_Y \\\n", "Order PID \n", "144 535153070 1 1 0 1 \n", "1574 916380060 6 1 0 1 \n", "490 528290190 1 1 0 1 \n", "1730 528218050 1 1 0 1 \n", "2276 921128030 1 1 0 1 \n", "\n", " Condition 1_Artery Condition 1_Feedr Condition 1_Norm \\\n", "Order PID \n", "144 535153070 0 0 1 \n", "1574 916380060 0 0 1 \n", "490 528290190 0 0 1 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Condition 1_PosA Condition 1_PosN Condition 1_RRAe \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 1 0 0 \n", "\n", " Condition 1_RRAn Condition 1_RRNe Condition 1_RRNn \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 1 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Condition 2_Artery Condition 2_Feedr Condition 2_Norm \\\n", "Order PID \n", "144 535153070 0 0 1 \n", "1574 916380060 0 0 1 \n", "490 528290190 0 0 1 \n", "1730 528218050 0 0 1 \n", "2276 921128030 0 0 0 \n", "\n", " Condition 2_PosA Condition 2_PosN Condition 2_RRAe \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 1 0 0 \n", "\n", " Condition 2_RRAn Condition 2_RRNe Condition 2_RRNn \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Electrical Enclosed Porch Exter Cond Exter Qual \\\n", "Order PID \n", "144 535153070 4 120.0 2 2 \n", "1574 916380060 4 0.0 2 3 \n", "490 528290190 4 0.0 2 2 \n", "1730 528218050 4 0.0 2 3 \n", "2276 921128030 4 0.0 2 4 \n", "\n", " Exterior 1st_AsbShng Exterior 1st_AsphShn \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_BrkComm Exterior 1st_BrkFace \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_CBlock Exterior 1st_CemntBd \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_HdBoard Exterior 1st_ImStucc \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_MetalSd Exterior 1st_Other \\\n", "Order PID \n", "144 535153070 1 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 1 0 \n", "\n", " Exterior 1st_Plywood Exterior 1st_PreCast \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_Stone Exterior 1st_Stucco \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_VinylSd Exterior 1st_Wd Sdng \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 1 0 \n", "490 528290190 1 0 \n", "1730 528218050 1 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 1st_WdShing Exterior 2nd_AsbShng \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_AsphShn Exterior 2nd_BrkComm \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_BrkFace Exterior 2nd_CBlock \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_CemntBd Exterior 2nd_HdBoard \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_ImStucc Exterior 2nd_MetalSd \\\n", "Order PID \n", "144 535153070 0 1 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 1 \n", "\n", " Exterior 2nd_Other Exterior 2nd_Plywood \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_PreCast Exterior 2nd_Stone \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_Stucco Exterior 2nd_VinylSd \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 1 \n", "490 528290190 0 1 \n", "1730 528218050 0 1 \n", "2276 921128030 0 0 \n", "\n", " Exterior 2nd_Wd Sdng Exterior 2nd_WdShing Fence \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Fireplace Qu Fireplaces Foundation_BrkTil \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 3 1 0 \n", "1730 528218050 4 1 0 \n", "2276 921128030 4 1 0 \n", "\n", " Foundation_CBlock Foundation_PConc Foundation_Slab \\\n", "Order PID \n", "144 535153070 1 0 0 \n", "1574 916380060 0 1 0 \n", "490 528290190 0 1 0 \n", "1730 528218050 0 1 0 \n", "2276 921128030 0 1 0 \n", "\n", " Foundation_Stone Foundation_Wood Full Bath Functional \\\n", "Order PID \n", "144 535153070 0 0 1 7 \n", "1574 916380060 0 0 2 7 \n", "490 528290190 0 0 2 7 \n", "1730 528218050 0 0 2 7 \n", "2276 921128030 0 0 2 7 \n", "\n", " Garage Area Garage Cars Garage Cond Garage Finish \\\n", "Order PID \n", "144 535153070 312.0 1 3 2 \n", "1574 916380060 788.0 3 3 3 \n", "490 528290190 400.0 2 3 2 \n", "1730 528218050 393.0 2 3 3 \n", "2276 921128030 932.0 3 3 3 \n", "\n", " Garage Qual Garage Type_2Types Garage Type_Attchd \\\n", "Order PID \n", "144 535153070 3 0 1 \n", "1574 916380060 3 0 1 \n", "490 528290190 3 0 0 \n", "1730 528218050 3 0 1 \n", "2276 921128030 3 0 1 \n", "\n", " Garage Type_Basment Garage Type_BuiltIn \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 1 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Garage Type_CarPort Garage Type_Detchd Garage Type_NA \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Gr Liv Area Half Bath Heating QC Heating_Floor \\\n", "Order PID \n", "144 535153070 1194.0 0 2 0 \n", "1574 916380060 1537.0 0 4 0 \n", "490 528290190 1430.0 1 4 0 \n", "1730 528218050 1484.0 1 4 0 \n", "2276 921128030 1824.0 0 4 0 \n", "\n", " Heating_GasA Heating_GasW Heating_Grav Heating_OthW \\\n", "Order PID \n", "144 535153070 1 0 0 0 \n", "1574 916380060 1 0 0 0 \n", "490 528290190 1 0 0 0 \n", "1730 528218050 1 0 0 0 \n", "2276 921128030 1 0 0 0 \n", "\n", " Heating_Wall House Style_1.5Fin House Style_1.5Unf \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " House Style_1Story House Style_2.5Fin House Style_2.5Unf \\\n", "Order PID \n", "144 535153070 1 0 0 \n", "1574 916380060 1 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 1 0 0 \n", "\n", " House Style_2Story House Style_SFoyer House Style_SLvl \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 1 \n", "1730 528218050 1 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Kitchen AbvGr Kitchen Qual Land Contour_Bnk \\\n", "Order PID \n", "144 535153070 1 2 0 \n", "1574 916380060 1 3 0 \n", "490 528290190 1 2 0 \n", "1730 528218050 1 3 0 \n", "2276 921128030 1 4 0 \n", "\n", " Land Contour_HLS Land Contour_Low Land Contour_Lvl \\\n", "Order PID \n", "144 535153070 0 0 1 \n", "1574 916380060 1 0 0 \n", "490 528290190 0 0 1 \n", "1730 528218050 0 0 1 \n", "2276 921128030 1 0 0 \n", "\n", " Land Slope Lot Area Lot Config_Corner Lot Config_CulDSac \\\n", "Order PID \n", "144 535153070 2 8760.0 0 0 \n", "1574 916380060 2 11563.0 0 0 \n", "490 528290190 2 7750.0 0 0 \n", "1730 528218050 2 10237.0 0 0 \n", "2276 921128030 2 12633.0 0 0 \n", "\n", " Lot Config_FR2 Lot Config_FR3 Lot Config_Inside Lot Shape \\\n", "Order PID \n", "144 535153070 0 0 1 3 \n", "1574 916380060 0 0 1 2 \n", "490 528290190 0 0 1 3 \n", "1730 528218050 0 0 1 3 \n", "2276 921128030 0 0 1 2 \n", "\n", " Low Qual Fin SF MS SubClass_020 MS SubClass_030 \\\n", "Order PID \n", "144 535153070 0.0 1 0 \n", "1574 916380060 0.0 1 0 \n", "490 528290190 0.0 0 0 \n", "1730 528218050 0.0 0 0 \n", "2276 921128030 0.0 1 0 \n", "\n", " MS SubClass_040 MS SubClass_045 MS SubClass_050 \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " MS SubClass_060 MS SubClass_070 MS SubClass_075 \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 1 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " MS SubClass_080 MS SubClass_085 MS SubClass_090 \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 1 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " MS SubClass_120 MS SubClass_150 MS SubClass_160 \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " MS SubClass_180 MS SubClass_190 MS Zoning_A MS Zoning_C \\\n", "Order PID \n", "144 535153070 0 0 0 0 \n", "1574 916380060 0 0 0 0 \n", "490 528290190 0 0 0 0 \n", "1730 528218050 0 0 0 0 \n", "2276 921128030 0 0 0 0 \n", "\n", " MS Zoning_FV MS Zoning_I MS Zoning_RH MS Zoning_RL \\\n", "Order PID \n", "144 535153070 0 0 0 1 \n", "1574 916380060 0 0 0 1 \n", "490 528290190 0 0 0 1 \n", "1730 528218050 0 0 0 1 \n", "2276 921128030 0 0 0 1 \n", "\n", " MS Zoning_RM MS Zoning_RP Mas Vnr Area \\\n", "Order PID \n", "144 535153070 0 0 220.0 \n", "1574 916380060 0 0 258.0 \n", "490 528290190 0 0 0.0 \n", "1730 528218050 0 0 0.0 \n", "2276 921128030 0 0 242.0 \n", "\n", " Mas Vnr Type_BrkCmn Mas Vnr Type_BrkFace \\\n", "Order PID \n", "144 535153070 0 1 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 1 \n", "\n", " Mas Vnr Type_CBlock Mas Vnr Type_None Mas Vnr Type_Stone \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 1 \n", "490 528290190 0 1 0 \n", "1730 528218050 0 1 0 \n", "2276 921128030 0 0 0 \n", "\n", " Misc Feature_Elev Misc Feature_Gar2 Misc Feature_NA \\\n", "Order PID \n", "144 535153070 0 0 1 \n", "1574 916380060 0 0 1 \n", "490 528290190 0 0 1 \n", "1730 528218050 0 0 1 \n", "2276 921128030 0 0 1 \n", "\n", " Misc Feature_Othr Misc Feature_Shed Misc Feature_TenC \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Misc Val Mo Sold Neighborhood_Blmngtn \\\n", "Order PID \n", "144 535153070 0.0 4 0 \n", "1574 916380060 0.0 4 0 \n", "490 528290190 0.0 3 0 \n", "1730 528218050 0.0 7 0 \n", "2276 921128030 0.0 9 0 \n", "\n", " Neighborhood_Blueste Neighborhood_BrDale \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_BrkSide Neighborhood_ClearCr \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_CollgCr Neighborhood_Crawfor \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_Edwards Neighborhood_Gilbert \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 1 \n", "1730 528218050 0 1 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_Greens Neighborhood_GrnHill \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_IDOTRR Neighborhood_Landmrk \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_MeadowV Neighborhood_Mitchel \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_NPkVill Neighborhood_NWAmes \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_Names Neighborhood_NoRidge \\\n", "Order PID \n", "144 535153070 1 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_NridgHt Neighborhood_OldTown \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_SWISU Neighborhood_Sawyer \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_SawyerW Neighborhood_Somerst \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Neighborhood_StoneBr Neighborhood_Timber \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 1 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 1 \n", "\n", " Neighborhood_Veenker Open Porch SF Overall Cond \\\n", "Order PID \n", "144 535153070 0 0.0 5 \n", "1574 916380060 0 26.0 4 \n", "490 528290190 0 0.0 4 \n", "1730 528218050 0 72.0 4 \n", "2276 921128030 0 36.0 4 \n", "\n", " Overall Qual Paved Drive Pool Area Pool QC \\\n", "Order PID \n", "144 535153070 5 2 0.0 0 \n", "1574 916380060 7 2 0.0 0 \n", "490 528290190 6 2 0.0 0 \n", "1730 528218050 5 2 0.0 0 \n", "2276 921128030 9 2 0.0 0 \n", "\n", " Roof Matl_ClyTile Roof Matl_CompShg Roof Matl_Membran \\\n", "Order PID \n", "144 535153070 0 1 0 \n", "1574 916380060 0 1 0 \n", "490 528290190 0 1 0 \n", "1730 528218050 0 1 0 \n", "2276 921128030 0 1 0 \n", "\n", " Roof Matl_Metal Roof Matl_Roll Roof Matl_Tar&Grv \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Roof Matl_WdShake Roof Matl_WdShngl Roof Style_Flat \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Roof Style_Gable Roof Style_Gambrel Roof Style_Hip \\\n", "Order PID \n", "144 535153070 0 0 1 \n", "1574 916380060 0 0 1 \n", "490 528290190 1 0 0 \n", "1730 528218050 1 0 0 \n", "2276 921128030 0 0 1 \n", "\n", " Roof Style_Mansard Roof Style_Shed Sale Condition_Abnorml \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Sale Condition_AdjLand Sale Condition_Alloca \\\n", "Order PID \n", "144 535153070 0 0 \n", "1574 916380060 0 0 \n", "490 528290190 0 0 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Sale Condition_Family Sale Condition_Normal \\\n", "Order PID \n", "144 535153070 0 1 \n", "1574 916380060 0 1 \n", "490 528290190 0 1 \n", "1730 528218050 0 0 \n", "2276 921128030 0 0 \n", "\n", " Sale Condition_Partial Sale Type_COD Sale Type_CWD \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 1 0 0 \n", "2276 921128030 1 0 0 \n", "\n", " Sale Type_Con Sale Type_ConLD Sale Type_ConLI \\\n", "Order PID \n", "144 535153070 0 0 0 \n", "1574 916380060 0 0 0 \n", "490 528290190 0 0 0 \n", "1730 528218050 0 0 0 \n", "2276 921128030 0 0 0 \n", "\n", " Sale Type_ConLw Sale Type_New Sale Type_Oth Sale Type_VWD \\\n", "Order PID \n", "144 535153070 0 0 0 0 \n", "1574 916380060 0 0 0 0 \n", "490 528290190 0 0 0 0 \n", "1730 528218050 0 1 0 0 \n", "2276 921128030 0 1 0 0 \n", "\n", " Sale Type_WD Screen Porch Street_Grvl Street_Pave \\\n", "Order PID \n", "144 535153070 1 0.0 0 1 \n", "1574 916380060 1 0.0 0 1 \n", "490 528290190 1 0.0 0 1 \n", "1730 528218050 0 0.0 0 1 \n", "2276 921128030 0 108.0 0 1 \n", "\n", " TotRms AbvGrd Total Bsmt SF Utilities Wood Deck SF \\\n", "Order PID \n", "144 535153070 6 1194.0 3 0.0 \n", "1574 916380060 8 1518.0 3 0.0 \n", "490 528290190 7 384.0 3 0.0 \n", "1730 528218050 8 783.0 3 0.0 \n", "2276 921128030 8 1824.0 3 160.0 \n", "\n", " Year Built Year Remod/Add Yr Sold SalePrice \n", "Order PID \n", "144 535153070 1959 1959 2010 148000.0 \n", "1574 916380060 2006 2007 2008 294000.0 \n", "490 528290190 1999 2000 2009 156000.0 \n", "1730 528218050 2005 2007 2007 178900.0 \n", "2276 921128030 2006 2007 2007 392000.0 " ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df1.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Obtain the raw numpy arrays:" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "X1 = df1.drop(columns=\"SalePrice\").values\n", "y1 = df1[\"SalePrice\"].values" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2. Improved Data\n", "\n", "The DataFrame `df2` holds the data as manually processed in notebooks 2 and 3.\n", "\n", "New features like the *years_since_\\** columns were generated or derived from other variables like *has 2nd Flr* (from the continuous *2nd Flr SF*). Further, factor variables were created taking into account patterns in the visualizations. For example, *Bldg Type*'s (from `df1`) five categories were condensed into just three. In summary, `df2` has less than half as many dimensions as `df1` to accomodate for a potential curse of dimensionality." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "df2 = load_clean_data(\"data/data_clean_with_transformations_and_factors.csv\")" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "df2 = encode_ordinals(df2)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "df2 = shuffle(df2, random_state=random_state)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "MultiIndex: 2883 entries, (2775, 907175060) to (2660, 902325050)\n", "Columns: 109 entries, 1st Flr SF to SalePrice (box-cox-0)\n", "dtypes: float64(27), int64(82)\n", "memory usage: 2.6 MB\n" ] } ], "source": [ "df2.info()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
1st Flr SF1st Flr SF (box-cox-0)2nd Flr SF3Ssn PorchBedroom AbvGrBsmt CondBsmt ExposureBsmt Full BathBsmt Half BathBsmt QualBsmt Unf SFBsmtFin SF 1BsmtFin SF 2BsmtFin Type 1BsmtFin Type 2ElectricalEnclosed PorchFenceFireplace QuFireplacesFull BathFunctionalGarage AreaGarage CarsGarage CondGarage FinishGarage QualGr Liv AreaGr Liv Area (box-cox-0)Half BathKitchen AbvGrKitchen QualLand SlopeLot AreaLot Area (box-cox-0.1)Lot ShapeLow Qual Fin SFMas Vnr AreaMisc ValMo SoldOpen Porch SFOverall CondOverall QualPaved DrivePool AreaPool QCScreen PorchTotRms AbvGrdTotal BathTotal Bsmt SFTotal Porch SFTotal SFTotal SF (box-cox-0.2)UtilitiesWood Deck SFabnormal_saleair_condbuild_type_1Fambuild_type_2Fambuild_type_Twnhsfound_BrkTilfound_CBlockfound_PConchas 2nd Flrhas Bsmthas Fireplacehas Garagehas Poolhas Porchmajor_streetnew_homenhood_Blmngtnnhood_Bluestenhood_BrDalenhood_BrkSidenhood_ClearCrnhood_CollgCrnhood_Crawfornhood_Edwardsnhood_Gilbertnhood_Greensnhood_GrnHillnhood_IDOTRRnhood_Landmrknhood_MeadowVnhood_Mitchelnhood_NPkVillnhood_NWAmesnhood_Namesnhood_NoRidgenhood_NridgHtnhood_OldTownnhood_SWISUnhood_Sawyernhood_SawyerWnhood_Somerstnhood_StoneBrnhood_Timbernhood_Veenkerparkpartial_salerailwayrecently_builtrecently_remodeledremodeledyears_since_builtyears_since_remodeledSalePriceSalePrice (box-cox-0)
OrderPID
27759071750601494.07.3092120.00.0334104279.01200.00.06140.000027576.023231494.07.30921201329236.014.92002020.00.00.0727.04520.000.063.01479.0195.02973.019.7519253168.00110000101010100000001000000000000000000000000011099217000.012.287653
2140907202220879.06.7787850.00.0331003102.0330.0432.04340.000017440.02313879.06.77878501229675.015.03600930.00.00.0980.05420.000.051.0864.080.01743.017.24485030.0011000100101010000000100000000000000000000000000003232120000.011.695247
1046527451550630.06.445720672.00.0331003280.0350.00.03140.000027264.013131302.07.17165711221680.011.01507430.0356.00.0120.04420.000.062.5630.00.01932.017.70761030.0010010101101000000100000000000000000000000000000003636111750.011.624020
6445353031501319.07.1846290.00.03311030.0428.0180.04240.020015270.013131319.07.184629012210634.015.27374930.00.00.0110.05420.000.052.0608.066.01927.017.695845366.0011000010101010000000000000000000100000000000000005656123000.011.719940
1605354010801287.07.1600690.00.0331103733.072.0258.05340.004117576.023331287.07.16006901329830.015.07583220.00.00.0317.06420.00182.072.01063.0563.02350.018.6147623364.001100010011101000000000000000000010000000000000011514162000.011.995352
\n", "
" ], "text/plain": [ " 1st Flr SF 1st Flr SF (box-cox-0) 2nd Flr SF 3Ssn Porch \\\n", "Order PID \n", "2775 907175060 1494.0 7.309212 0.0 0.0 \n", "2140 907202220 879.0 6.778785 0.0 0.0 \n", "1046 527451550 630.0 6.445720 672.0 0.0 \n", "644 535303150 1319.0 7.184629 0.0 0.0 \n", "160 535401080 1287.0 7.160069 0.0 0.0 \n", "\n", " Bedroom AbvGr Bsmt Cond Bsmt Exposure Bsmt Full Bath \\\n", "Order PID \n", "2775 907175060 3 3 4 1 \n", "2140 907202220 3 3 1 0 \n", "1046 527451550 3 3 1 0 \n", "644 535303150 3 3 1 1 \n", "160 535401080 3 3 1 1 \n", "\n", " Bsmt Half Bath Bsmt Qual Bsmt Unf SF BsmtFin SF 1 \\\n", "Order PID \n", "2775 907175060 0 4 279.0 1200.0 \n", "2140 907202220 0 3 102.0 330.0 \n", "1046 527451550 0 3 280.0 350.0 \n", "644 535303150 0 3 0.0 428.0 \n", "160 535401080 0 3 733.0 72.0 \n", "\n", " BsmtFin SF 2 BsmtFin Type 1 BsmtFin Type 2 Electrical \\\n", "Order PID \n", "2775 907175060 0.0 6 1 4 \n", "2140 907202220 432.0 4 3 4 \n", "1046 527451550 0.0 3 1 4 \n", "644 535303150 180.0 4 2 4 \n", "160 535401080 258.0 5 3 4 \n", "\n", " Enclosed Porch Fence Fireplace Qu Fireplaces Full Bath \\\n", "Order PID \n", "2775 907175060 0.0 0 0 0 2 \n", "2140 907202220 0.0 0 0 0 1 \n", "1046 527451550 0.0 0 0 0 2 \n", "644 535303150 0.0 2 0 0 1 \n", "160 535401080 0.0 0 4 1 1 \n", "\n", " Functional Garage Area Garage Cars Garage Cond \\\n", "Order PID \n", "2775 907175060 7 576.0 2 3 \n", "2140 907202220 7 440.0 2 3 \n", "1046 527451550 7 264.0 1 3 \n", "644 535303150 5 270.0 1 3 \n", "160 535401080 7 576.0 2 3 \n", "\n", " Garage Finish Garage Qual Gr Liv Area \\\n", "Order PID \n", "2775 907175060 2 3 1494.0 \n", "2140 907202220 1 3 879.0 \n", "1046 527451550 1 3 1302.0 \n", "644 535303150 1 3 1319.0 \n", "160 535401080 3 3 1287.0 \n", "\n", " Gr Liv Area (box-cox-0) Half Bath Kitchen AbvGr \\\n", "Order PID \n", "2775 907175060 7.309212 0 1 \n", "2140 907202220 6.778785 0 1 \n", "1046 527451550 7.171657 1 1 \n", "644 535303150 7.184629 0 1 \n", "160 535401080 7.160069 0 1 \n", "\n", " Kitchen Qual Land Slope Lot Area Lot Area (box-cox-0.1) \\\n", "Order PID \n", "2775 907175060 3 2 9236.0 14.920020 \n", "2140 907202220 2 2 9675.0 15.036009 \n", "1046 527451550 2 2 1680.0 11.015074 \n", "644 535303150 2 2 10634.0 15.273749 \n", "160 535401080 3 2 9830.0 15.075832 \n", "\n", " Lot Shape Low Qual Fin SF Mas Vnr Area Misc Val Mo Sold \\\n", "Order PID \n", "2775 907175060 2 0.0 0.0 0.0 7 \n", "2140 907202220 3 0.0 0.0 0.0 9 \n", "1046 527451550 3 0.0 356.0 0.0 12 \n", "644 535303150 3 0.0 0.0 0.0 11 \n", "160 535401080 2 0.0 0.0 0.0 3 \n", "\n", " Open Porch SF Overall Cond Overall Qual Paved Drive \\\n", "Order PID \n", "2775 907175060 27.0 4 5 2 \n", "2140 907202220 80.0 5 4 2 \n", "1046 527451550 0.0 4 4 2 \n", "644 535303150 0.0 5 4 2 \n", "160 535401080 17.0 6 4 2 \n", "\n", " Pool Area Pool QC Screen Porch TotRms AbvGrd Total Bath \\\n", "Order PID \n", "2775 907175060 0.0 0 0.0 6 3.0 \n", "2140 907202220 0.0 0 0.0 5 1.0 \n", "1046 527451550 0.0 0 0.0 6 2.5 \n", "644 535303150 0.0 0 0.0 5 2.0 \n", "160 535401080 0.0 0 182.0 7 2.0 \n", "\n", " Total Bsmt SF Total Porch SF Total SF \\\n", "Order PID \n", "2775 907175060 1479.0 195.0 2973.0 \n", "2140 907202220 864.0 80.0 1743.0 \n", "1046 527451550 630.0 0.0 1932.0 \n", "644 535303150 608.0 66.0 1927.0 \n", "160 535401080 1063.0 563.0 2350.0 \n", "\n", " Total SF (box-cox-0.2) Utilities Wood Deck SF \\\n", "Order PID \n", "2775 907175060 19.751925 3 168.0 \n", "2140 907202220 17.244850 3 0.0 \n", "1046 527451550 17.707610 3 0.0 \n", "644 535303150 17.695845 3 66.0 \n", "160 535401080 18.614762 3 364.0 \n", "\n", " abnormal_sale air_cond build_type_1Fam build_type_2Fam \\\n", "Order PID \n", "2775 907175060 0 1 1 0 \n", "2140 907202220 0 1 1 0 \n", "1046 527451550 0 1 0 0 \n", "644 535303150 0 1 1 0 \n", "160 535401080 0 1 1 0 \n", "\n", " build_type_Twnhs found_BrkTil found_CBlock found_PConc \\\n", "Order PID \n", "2775 907175060 0 0 0 1 \n", "2140 907202220 0 0 1 0 \n", "1046 527451550 1 0 1 0 \n", "644 535303150 0 0 0 1 \n", "160 535401080 0 0 1 0 \n", "\n", " has 2nd Flr has Bsmt has Fireplace has Garage has Pool \\\n", "Order PID \n", "2775 907175060 0 1 0 1 0 \n", "2140 907202220 0 1 0 1 0 \n", "1046 527451550 1 1 0 1 0 \n", "644 535303150 0 1 0 1 0 \n", "160 535401080 0 1 1 1 0 \n", "\n", " has Porch major_street new_home nhood_Blmngtn \\\n", "Order PID \n", "2775 907175060 1 0 0 0 \n", "2140 907202220 1 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 1 0 0 0 \n", "160 535401080 1 0 0 0 \n", "\n", " nhood_Blueste nhood_BrDale nhood_BrkSide nhood_ClearCr \\\n", "Order PID \n", "2775 907175060 0 0 0 0 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 1 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 0 \n", "\n", " nhood_CollgCr nhood_Crawfor nhood_Edwards nhood_Gilbert \\\n", "Order PID \n", "2775 907175060 1 0 0 0 \n", "2140 907202220 1 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 0 \n", "\n", " nhood_Greens nhood_GrnHill nhood_IDOTRR nhood_Landmrk \\\n", "Order PID \n", "2775 907175060 0 0 0 0 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 0 \n", "\n", " nhood_MeadowV nhood_Mitchel nhood_NPkVill nhood_NWAmes \\\n", "Order PID \n", "2775 907175060 0 0 0 0 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 0 \n", "\n", " nhood_Names nhood_NoRidge nhood_NridgHt nhood_OldTown \\\n", "Order PID \n", "2775 907175060 0 0 0 0 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 1 0 0 0 \n", "160 535401080 1 0 0 0 \n", "\n", " nhood_SWISU nhood_Sawyer nhood_SawyerW nhood_Somerst \\\n", "Order PID \n", "2775 907175060 0 0 0 0 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 0 \n", "\n", " nhood_StoneBr nhood_Timber nhood_Veenker park \\\n", "Order PID \n", "2775 907175060 0 0 0 0 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 0 \n", "\n", " partial_sale railway recently_built recently_remodeled \\\n", "Order PID \n", "2775 907175060 0 0 1 1 \n", "2140 907202220 0 0 0 0 \n", "1046 527451550 0 0 0 0 \n", "644 535303150 0 0 0 0 \n", "160 535401080 0 0 0 1 \n", "\n", " remodeled years_since_built years_since_remodeled \\\n", "Order PID \n", "2775 907175060 0 9 9 \n", "2140 907202220 0 32 32 \n", "1046 527451550 0 36 36 \n", "644 535303150 0 56 56 \n", "160 535401080 1 51 4 \n", "\n", " SalePrice SalePrice (box-cox-0) \n", "Order PID \n", "2775 907175060 217000.0 12.287653 \n", "2140 907202220 120000.0 11.695247 \n", "1046 527451550 111750.0 11.624020 \n", "644 535303150 123000.0 11.719940 \n", "160 535401080 162000.0 11.995352 " ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df2.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Obtain the raw numpy arrays:" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "X2 = df2.drop(columns=[\"SalePrice\", \"SalePrice (box-cox-0)\"]).values\n", "y2 = df2[\"SalePrice\"].values\n", "y2l = df2[\"SalePrice (box-cox-0)\"].values" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3. Manual Feature Pre-Selection\n", "\n", "Also, notebook 2 collects variables that correlate either weakly ($0.33 < \\vert\\rho\\vert < 0.66$) or strongly ($\\vert\\rho\\vert > 0.66$) with the *SalePrice* (or the logarithm thereof). These variables serve as a \"naive\" feature pre-selection." ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "with open(\"data/correlated_variables.json\", \"r\") as file:\n", " _ = json.loads(file.read())\n", " weakly_correlated = _[\"weakly_correlated\"]\n", " strongly_correlated = _[\"strongly_correlated\"]" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "pre_selection = sorted(set(weakly_correlated + strongly_correlated) & set(df2.columns))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The `df3` DataFrame is just a subset of `df2` (71 columns)." ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "df3 = df2[pre_selection + TARGET_VARIABLES]" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "MultiIndex: 2883 entries, (2775, 907175060) to (2660, 902325050)\n", "Columns: 32 entries, 1st Flr SF to SalePrice (box-cox-0)\n", "dtypes: float64(16), int64(16)\n", "memory usage: 906.2 KB\n" ] } ], "source": [ "df3.info(verbose=False)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
1st Flr SF1st Flr SF (box-cox-0)Bsmt ExposureBsmt QualBsmtFin SF 1BsmtFin Type 1Fireplace QuFireplacesFull BathGarage AreaGarage CarsGarage CondGarage FinishGarage QualGr Liv AreaGr Liv Area (box-cox-0)Half BathKitchen QualLot Area (box-cox-0.1)Lot ShapeMas Vnr AreaOverall QualPaved DriveTotRms AbvGrdTotal BathTotal Bsmt SFTotal Porch SFTotal SFTotal SF (box-cox-0.2)Wood Deck SFSalePriceSalePrice (box-cox-0)
OrderPID
27759071750601494.07.309212441200.06002576.023231494.07.3092120314.92002020.05263.01479.0195.02973.019.751925168.0217000.012.287653
2140907202220879.06.77878513330.04001440.02313879.06.7787850215.03600930.04251.0864.080.01743.017.2448500.0120000.011.695247
1046527451550630.06.44572013350.03002264.013131302.07.1716571211.0150743356.04262.5630.00.01932.017.7076100.0111750.011.624020
6445353031501319.07.18462913428.04001270.013131319.07.1846290215.27374930.04252.0608.066.01927.017.69584566.0123000.011.719940
1605354010801287.07.1600691372.05411576.023331287.07.1600690315.07583220.04272.01063.0563.02350.018.614762364.0162000.011.995352
\n", "
" ], "text/plain": [ " 1st Flr SF 1st Flr SF (box-cox-0) Bsmt Exposure Bsmt Qual \\\n", "Order PID \n", "2775 907175060 1494.0 7.309212 4 4 \n", "2140 907202220 879.0 6.778785 1 3 \n", "1046 527451550 630.0 6.445720 1 3 \n", "644 535303150 1319.0 7.184629 1 3 \n", "160 535401080 1287.0 7.160069 1 3 \n", "\n", " BsmtFin SF 1 BsmtFin Type 1 Fireplace Qu Fireplaces \\\n", "Order PID \n", "2775 907175060 1200.0 6 0 0 \n", "2140 907202220 330.0 4 0 0 \n", "1046 527451550 350.0 3 0 0 \n", "644 535303150 428.0 4 0 0 \n", "160 535401080 72.0 5 4 1 \n", "\n", " Full Bath Garage Area Garage Cars Garage Cond \\\n", "Order PID \n", "2775 907175060 2 576.0 2 3 \n", "2140 907202220 1 440.0 2 3 \n", "1046 527451550 2 264.0 1 3 \n", "644 535303150 1 270.0 1 3 \n", "160 535401080 1 576.0 2 3 \n", "\n", " Garage Finish Garage Qual Gr Liv Area \\\n", "Order PID \n", "2775 907175060 2 3 1494.0 \n", "2140 907202220 1 3 879.0 \n", "1046 527451550 1 3 1302.0 \n", "644 535303150 1 3 1319.0 \n", "160 535401080 3 3 1287.0 \n", "\n", " Gr Liv Area (box-cox-0) Half Bath Kitchen Qual \\\n", "Order PID \n", "2775 907175060 7.309212 0 3 \n", "2140 907202220 6.778785 0 2 \n", "1046 527451550 7.171657 1 2 \n", "644 535303150 7.184629 0 2 \n", "160 535401080 7.160069 0 3 \n", "\n", " Lot Area (box-cox-0.1) Lot Shape Mas Vnr Area \\\n", "Order PID \n", "2775 907175060 14.920020 2 0.0 \n", "2140 907202220 15.036009 3 0.0 \n", "1046 527451550 11.015074 3 356.0 \n", "644 535303150 15.273749 3 0.0 \n", "160 535401080 15.075832 2 0.0 \n", "\n", " Overall Qual Paved Drive TotRms AbvGrd Total Bath \\\n", "Order PID \n", "2775 907175060 5 2 6 3.0 \n", "2140 907202220 4 2 5 1.0 \n", "1046 527451550 4 2 6 2.5 \n", "644 535303150 4 2 5 2.0 \n", "160 535401080 4 2 7 2.0 \n", "\n", " Total Bsmt SF Total Porch SF Total SF \\\n", "Order PID \n", "2775 907175060 1479.0 195.0 2973.0 \n", "2140 907202220 864.0 80.0 1743.0 \n", "1046 527451550 630.0 0.0 1932.0 \n", "644 535303150 608.0 66.0 1927.0 \n", "160 535401080 1063.0 563.0 2350.0 \n", "\n", " Total SF (box-cox-0.2) Wood Deck SF SalePrice \\\n", "Order PID \n", "2775 907175060 19.751925 168.0 217000.0 \n", "2140 907202220 17.244850 0.0 120000.0 \n", "1046 527451550 17.707610 0.0 111750.0 \n", "644 535303150 17.695845 66.0 123000.0 \n", "160 535401080 18.614762 364.0 162000.0 \n", "\n", " SalePrice (box-cox-0) \n", "Order PID \n", "2775 907175060 12.287653 \n", "2140 907202220 11.695247 \n", "1046 527451550 11.624020 \n", "644 535303150 11.719940 \n", "160 535401080 11.995352 " ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df3.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Obtain the raw numpy arrays:" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "X3 = df3.drop(columns=[\"SalePrice\", \"SalePrice (box-cox-0)\"]).values\n", "y3 = df3[\"SalePrice\"].values\n", "y3l = df3[\"SalePrice (box-cox-0)\"].values" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Re-usable Components\n", "\n", "Define a function to run k-fold cross validation to obtain unbiased estimators for the following scores / errors:\n", "- Bias\n", "- Mean Absolute Error\n", "- Maximum Deviation (just to see the worst case prediction of a model)\n", "- R2 (coefficient of determination)\n", "- Root Mean Squared Error (default for comparison)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def cross_validation(X, y, *, model, k=10, log=False, desc=None):\n", " \"\"\"Perform a k-fold cross validation.\"\"\"\n", " bias, mae, max_dev, r2, rmse = [], [], [], [], []\n", " # Iterate over the k folds.\n", " for train, test in progress_bar(KFold(n_splits=k).split(X), desc=desc, total=k):\n", " model.fit(X[train], y[train])\n", " y_pred = model.predict(X[test])\n", " # If the sales price is provided on a log scale,\n", " # take the exponent first so that scores and\n", " # errors are comparable to the non-logged counterparts.\n", " if log:\n", " y_true, y_pred = np.exp(y[test]), np.exp(y_pred) \n", " else:\n", " y_true, y_pred = y[test], y_pred\n", " # Collect the scores/errors for each fold.\n", " bias.append(bias_score(y_true, y_pred))\n", " mae.append(mean_absolute_error(y_true, y_pred))\n", " max_dev.append(max_deviation(y_true, y_pred))\n", " r2.append(r2_score(y_true, y_pred))\n", " rmse.append(mean_squared_error(y_true, y_pred))\n", " # Round for convenience.\n", " return {\n", " \"bias\": np.round(np.mean(bias)),\n", " \"mae\": np.round(np.mean(mae)),\n", " \"max_dev\": np.round(np.mean(max_dev)),\n", " \"r2\": np.round(np.mean(r2), 3),\n", " \"rmse\": np.round(np.sqrt(np.mean(rmse))),\n", " }" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Use one dictionary to store all the results in a systematic way." ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "results = {}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Linear Regression\n", "\n", "A plain OLS regression model serves as the base case for benchmarking." ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [], "source": [ "lm = LinearRegression()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1. Original Data\n", "\n", "Given the unprocessed data, the linear model is not able to make a good fit at all." ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:01<00:00, 9.86it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': 14349804.0,\n", " 'mae': 18498417.0,\n", " 'max_dev': 5072252036.0,\n", " 'r2': -106245976.863,\n", " 'rmse': 820848886.0}" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[('lm','o')] = cross_validation(X1, y1, model=lm)\n", "results[('lm','o')]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2. Improved Data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 22.99it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -2.0,\n", " 'mae': 15277.0,\n", " 'max_dev': 126472.0,\n", " 'r2': 0.919,\n", " 'rmse': 21829.0}" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[('lm','i')] = cross_validation(X2, y2, model=lm)\n", "results[('lm','i')]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 19.21it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -820.0,\n", " 'mae': 12807.0,\n", " 'max_dev': 114579.0,\n", " 'r2': 0.938,\n", " 'rmse': 19106.0}" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[('lm','il')] = cross_validation(X2, y2l, model=lm, log=True)\n", "results[('lm','il')]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3. Improved Data with pre-selected Features" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 278.33it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': 4.0,\n", " 'mae': 18369.0,\n", " 'max_dev': 142946.0,\n", " 'r2': 0.887,\n", " 'rmse': 25682.0}" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[('lm','p')] = cross_validation(X3, y3, model=lm)\n", "results[('lm','p')]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 271.17it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -1342.0,\n", " 'mae': 16126.0,\n", " 'max_dev': 140216.0,\n", " 'r2': 0.906,\n", " 'rmse': 23335.0}" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[('lm','pl')] = cross_validation(X3, y3l, model=lm, log=True)\n", "results[('lm','pl')]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## LASSO" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "tol = 0.1\n", "grid_search = GridSearchCV(\n", " estimator=Lasso(tol=tol, selection=\"random\", random_state=random_state),\n", " param_grid={\"alpha\": [2 ** x for x in range(-8, 4)] + list(range(12, 65, 4))},\n", " cv=KFold(n_splits=4),\n", " n_jobs=-1,\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1. Original Data" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X1, y1)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 38.22it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': 278.0,\n", " 'mae': 20892.0,\n", " 'max_dev': 269314.0,\n", " 'r2': 0.817,\n", " 'rmse': 33496.0}" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"lasso\", \"o\")] = cross_validation(X1, y1, model=Lasso(alpha=alpha, tol=tol))\n", "results[(\"lasso\", \"o\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2. Improved Data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "40" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X2, y2)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 54.59it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -49.0,\n", " 'mae': 17488.0,\n", " 'max_dev': 134191.0,\n", " 'r2': 0.896,\n", " 'rmse': 24529.0}" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"lasso\", \"i\")] = cross_validation(X2, y2, model=Lasso(alpha=alpha, tol=tol))\n", "results[(\"lasso\", \"i\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.00390625" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X2, y2l)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 46.34it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -861.0,\n", " 'mae': 14346.0,\n", " 'max_dev': 129044.0,\n", " 'r2': 0.923,\n", " 'rmse': 21081.0}" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"lasso\", \"il\")] = cross_validation(X2, y2l, model=Lasso(alpha=alpha, tol=tol), log=True)\n", "results[(\"lasso\", \"il\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3. Improved Data with pre-selected Features" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "44" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X3, y3)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 328.75it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -11.0,\n", " 'mae': 23344.0,\n", " 'max_dev': 152485.0,\n", " 'r2': 0.827,\n", " 'rmse': 31672.0}" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"lasso\", \"p\")] = cross_validation(X3, y3, model=Lasso(alpha=alpha, tol=tol))\n", "results[(\"lasso\", \"p\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.00390625" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X3, y3l)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 299.57it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -877.0,\n", " 'mae': 16619.0,\n", " 'max_dev': 143135.0,\n", " 'r2': 0.899,\n", " 'rmse': 24205.0}" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"lasso\", \"pl\")] = cross_validation(X3, y3l, model=Lasso(alpha=alpha, tol=tol), log=True)\n", "results[(\"lasso\", \"pl\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Ridge Regression" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "grid_search = GridSearchCV(\n", " estimator=Ridge(),\n", " param_grid={\"alpha\": [2 ** x for x in range(-8, 4)] + list(range(12, 65, 4))},\n", " cv=KFold(n_splits=4),\n", " n_jobs=-1,\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1. Original Data" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.125" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X1, y1)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 41.08it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': 152.0,\n", " 'mae': 17064.0,\n", " 'max_dev': 263561.0,\n", " 'r2': 0.853,\n", " 'rmse': 29970.0}" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"ridge\", \"o\")] = cross_validation(X1, y1, model=Ridge(alpha=alpha))\n", "results[(\"ridge\", \"o\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2. Improved Data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "8" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X2, y2)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 69.82it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -6.0,\n", " 'mae': 15248.0,\n", " 'max_dev': 126366.0,\n", " 'r2': 0.919,\n", " 'rmse': 21835.0}" ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"ridge\", \"i\")] = cross_validation(X2, y2, model=Ridge(alpha=alpha))\n", "results[(\"ridge\", \"i\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.5" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X2, y2l)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 74.39it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -840.0,\n", " 'mae': 12803.0,\n", " 'max_dev': 113756.0,\n", " 'r2': 0.938,\n", " 'rmse': 19073.0}" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"ridge\", \"il\")] = cross_validation(X2, y2l, model=Ridge(alpha=alpha), log=True)\n", "results[(\"ridge\", \"il\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3. Improved Data with pre-selected Features" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "8" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X3, y3)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 64.44it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': 4.0,\n", " 'mae': 18342.0,\n", " 'max_dev': 142600.0,\n", " 'r2': 0.887,\n", " 'rmse': 25659.0}" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"ridge\", \"p\")] = cross_validation(X3, y3, model=Ridge(alpha=alpha))\n", "results[(\"ridge\", \"p\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.5" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.fit(X3, y3l)\n", "alpha = grid_search.best_params_[\"alpha\"]\n", "alpha" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:00<00:00, 60.47it/s]\n" ] }, { "data": { "text/plain": [ "{'bias': -1400.0,\n", " 'mae': 16105.0,\n", " 'max_dev': 139398.0,\n", " 'r2': 0.907,\n", " 'rmse': 23280.0}" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"ridge\", \"pl\")] = cross_validation(X3, y3l, model=Ridge(alpha=alpha), log=True)\n", "results[(\"ridge\", \"pl\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Random Forest" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [], "source": [ "rf = RandomForestRegressor(\n", " n_estimators=500,\n", " n_jobs=-1, random_state=random_state\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1. Original Data" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:19<00:00, 1.96s/it]\n" ] }, { "data": { "text/plain": [ "{'bias': -26.0,\n", " 'mae': 15322.0,\n", " 'max_dev': 164505.0,\n", " 'r2': 0.898,\n", " 'rmse': 25354.0}" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"rf\", \"o\")] = cross_validation(X1, y1, model=rf)\n", "results[(\"rf\", \"o\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2. Improved Data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:16<00:00, 1.69s/it]\n" ] }, { "data": { "text/plain": [ "{'bias': -70.0,\n", " 'mae': 14916.0,\n", " 'max_dev': 137911.0,\n", " 'r2': 0.91,\n", " 'rmse': 22813.0}" ] }, "execution_count": 55, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"rf\", \"i\")] = cross_validation(X2, y2, model=rf)\n", "results[(\"rf\", \"i\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:19<00:00, 1.97s/it]\n" ] }, { "data": { "text/plain": [ "{'bias': -1996.0,\n", " 'mae': 14952.0,\n", " 'max_dev': 141360.0,\n", " 'r2': 0.908,\n", " 'rmse': 23000.0}" ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"rf\", \"il\")] = cross_validation(X2, y2l, model=rf, log=True)\n", "results[(\"rf\", \"il\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3. Improved Data with pre-selected Features" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### a) Normal Scale" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:12<00:00, 1.29s/it]\n" ] }, { "data": { "text/plain": [ "{'bias': -165.0,\n", " 'mae': 16042.0,\n", " 'max_dev': 143762.0,\n", " 'r2': 0.898,\n", " 'rmse': 24274.0}" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"rf\", \"p\")] = cross_validation(X3, y3, model=rf)\n", "results[(\"rf\", \"p\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### b) Log Scale" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:11<00:00, 1.13s/it]\n" ] }, { "data": { "text/plain": [ "{'bias': -2250.0,\n", " 'mae': 16356.0,\n", " 'max_dev': 146888.0,\n", " 'r2': 0.893,\n", " 'rmse': 24785.0}" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[(\"rf\", \"pl\")] = cross_validation(X3, y3l, model=rf, log=True)\n", "results[(\"rf\", \"pl\")]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Analysis of Results\n", "\n", "This notebook did not focus on hyper-parameter optimization. Therefore, the predictions by Lasso, Ridge, and the Random Forest can potentially be improved by fine-graining the grid search even more.\n", "\n", "In general, the manually \"improved\" data clearly outperform the data that were only cleaned with the minimum effort. Also, the result suggests to allow the model to select its features. The manually pre-selected features perform well but not as good as the full feature set." ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "def scores_by_source(source, score=\"rmse\", *, ascending=True):\n", " rv = [\n", " (model, scores[score])\n", " for (model, data_source), scores in results.items()\n", " if data_source == source\n", " ]\n", " return sorted(rv, key=lambda x: x[1], reverse=(not ascending))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Root Mean Squared Error" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('rf', 25354.0), ('ridge', 29970.0), ('lasso', 33496.0), ('lm', 820848886.0)]" ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"o\", \"rmse\")" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('lm', 21829.0), ('ridge', 21835.0), ('rf', 22813.0), ('lasso', 24529.0)]" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"i\", \"rmse\")" ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('ridge', 19073.0), ('lm', 19106.0), ('lasso', 21081.0), ('rf', 23000.0)]" ] }, "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"il\", \"rmse\")" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('rf', 24274.0), ('ridge', 25659.0), ('lm', 25682.0), ('lasso', 31672.0)]" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"p\", \"rmse\")" ] }, { "cell_type": "code", "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('ridge', 23280.0), ('lm', 23335.0), ('lasso', 24205.0), ('rf', 24785.0)]" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"pl\", \"rmse\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### R2" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('rf', 0.898), ('ridge', 0.853), ('lasso', 0.817), ('lm', -106245976.863)]" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"o\", \"r2\", ascending=False)" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('lm', 0.919), ('ridge', 0.919), ('rf', 0.91), ('lasso', 0.896)]" ] }, "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"i\", \"r2\", ascending=False)" ] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('lm', 0.938), ('ridge', 0.938), ('lasso', 0.923), ('rf', 0.908)]" ] }, "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"il\", \"r2\", ascending=False)" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('rf', 0.898), ('lm', 0.887), ('ridge', 0.887), ('lasso', 0.827)]" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"p\", \"r2\", ascending=False)" ] }, { "cell_type": "code", "execution_count": 69, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('ridge', 0.907), ('lm', 0.906), ('lasso', 0.899), ('rf', 0.893)]" ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scores_by_source(\"pl\", \"r2\", ascending=False)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }