{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "[Sebastian Raschka](http://sebastianraschka.com), 2015\n", "\n", "https://github.com/rasbt/python-machine-learning-book" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Dealing with missing data" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABCD
01.02.03.04.0
15.06.0NaN8.0
210.011.012.0NaN
\n", "
" ], "text/plain": [ " A B C D\n", "0 1.0 2.0 3.0 4.0\n", "1 5.0 6.0 NaN 8.0\n", "2 10.0 11.0 12.0 NaN" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import numpy as np\n", "import pandas as pd\n", "from io import StringIO\n", "\n", "csv_data = '''A,B,C,D\n", "1.0,2.0,3.0,4.0\n", "5.0,6.0,,8.0\n", "10.0,11.0,12.0,'''\n", "\n", "# If you are using Python 2.7, you need\n", "# to convert the string to unicode:\n", "# csv_data = unicode(csv_data)\n", "\n", "df = pd.read_csv(StringIO(csv_data))\n", "df" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "A 0\n", "B 0\n", "C 1\n", "D 1\n", "dtype: int64" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# df.isna()\n", "# df.notna()\n", "df.isnull().sum()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Eliminating samples or features with missing values" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABCD
01.02.03.04.0
\n", "
" ], "text/plain": [ " A B C D\n", "0 1.0 2.0 3.0 4.0" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.dropna()\n", "# or\n", "df.dropna(axis=0)\n", "# or\n", "df.dropna(axis='index')" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
AB
01.02.0
15.06.0
210.011.0
\n", "
" ], "text/plain": [ " A B\n", "0 1.0 2.0\n", "1 5.0 6.0\n", "2 10.0 11.0" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.dropna(axis=1)\n", "# or\n", "df.dropna(axis='columns')" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABCD
01.02.03.04.0
15.06.0NaN8.0
210.011.012.0NaN
\n", "
" ], "text/plain": [ " A B C D\n", "0 1.0 2.0 3.0 4.0\n", "1 5.0 6.0 NaN 8.0\n", "2 10.0 11.0 12.0 NaN" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# only drop rows where all columns are NaN\n", "df.dropna(how='all') " ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABCD
01.02.03.04.0
\n", "
" ], "text/plain": [ " A B C D\n", "0 1.0 2.0 3.0 4.0" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# drop rows that have less than 4 non-NaN values\n", "df.dropna(thresh=4)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABCD
01.02.03.04.0
210.011.012.0NaN
\n", "
" ], "text/plain": [ " A B C D\n", "0 1.0 2.0 3.0 4.0\n", "2 10.0 11.0 12.0 NaN" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# only drop rows where NaN appear in specific columns (here: 'C')\n", "df.dropna(subset=['C'])" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABD
01.02.04.0
15.06.08.0
210.011.0NaN
\n", "
" ], "text/plain": [ " A B D\n", "0 1.0 2.0 4.0\n", "1 5.0 6.0 8.0\n", "2 10.0 11.0 NaN" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# only drop columns where NaN appear in specific rows\n", "df.dropna(axis=1,subset=[1])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Imputing missing values" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\ProgramData\\Miniconda3\\envs\\py3x\\lib\\site-packages\\sklearn\\utils\\deprecation.py:66: DeprecationWarning: Class Imputer is deprecated; Imputer was deprecated in version 0.20 and will be removed in 0.22. Import impute.SimpleImputer from sklearn instead.\n", " warnings.warn(msg, category=DeprecationWarning)\n" ] }, { "data": { "text/plain": [ "array([[ 1. , 2. , 3. , 4. ],\n", " [ 5. , 6. , 7.5, 8. ],\n", " [10. , 11. , 12. , 6. ]])" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.preprocessing import Imputer\n", "\n", "imr = Imputer(missing_values='NaN', strategy='mean', axis=0)\n", "imr = imr.fit(df)\n", "imputed_data = imr.transform(df.values)\n", "imputed_data\n" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 1. , 2. , 3. , 4. ],\n", " [ 5. , 6. , 7.5, 8. ],\n", " [10. , 11. , 12. , 6. ]])" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.impute import SimpleImputer\n", "\n", "imr = SimpleImputer(missing_values=np.nan, strategy='mean')\n", "# imr = SimpleImputer(missing_values=np.nan, strategy='mean',copy=False)\n", "imr = imr.fit(df)\n", "imputed_data = imr.transform(df.values)\n", "imputed_data" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 1., 2., 3., 4.],\n", " [ 5., 6., nan, 8.],\n", " [10., 11., 12., nan]])" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.values" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ABCDEF
01.02013-01-021.03.0testfoo
11.0NaT1.03.0trainfoo
21.02013-01-021.0NaNtestfoo
31.02013-01-021.03.0NaNfoo
\n", "
" ], "text/plain": [ " A B C D E F\n", "0 1.0 2013-01-02 1.0 3.0 test foo\n", "1 1.0 NaT 1.0 3.0 train foo\n", "2 1.0 2013-01-02 1.0 NaN test foo\n", "3 1.0 2013-01-02 1.0 3.0 NaN foo" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df2 = pd.DataFrame({'A': 1.,\n", " ...: 'B': pd.Timestamp('20130102'),\n", " ...: 'C': pd.Series(1, index=list(range(4)), dtype='float32'),\n", " ...: 'D': np.array([3] * 4, dtype='int32'),\n", " ...: 'E': pd.Categorical([\"test\", \"train\", \"test\", \"train\"]),\n", " ...: 'F': 'foo'})\n", "df2.loc[1,'B']=np.nan\n", "df2.loc[2,'D']=np.nan\n", "df2.loc[3,'E']=np.nan\n", "df2" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[1., 1., 3.],\n", " [1., 1., 3.],\n", " [1., 1., 3.],\n", " [1., 1., 3.]])" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "imr = SimpleImputer(missing_values=np.nan, strategy='median')\n", "# mean and median do not work with categorical data\n", "imr = imr.fit(df2.select_dtypes(include='number'))\n", "imputed_data = imr.transform(df2.select_dtypes(include='number').values)\n", "imputed_data" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3.0, 'test', 'foo'],\n", " [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3.0, 'train', 'foo'],\n", " [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3.0, 'test', 'foo'],\n", " [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3.0, 'test', 'foo']],\n", " dtype=object)" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# but most_frequent and constant are OK\n", "imr = SimpleImputer(missing_values=np.nan, strategy='most_frequent')\n", "imr = imr.fit(df2)\n", "imputed_data = imr.transform(df2.values)\n", "imputed_data" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "id 0\n", "date 0\n", "price 0\n", "bedrooms 0\n", "bathrooms 0\n", "sqft_living 0\n", "sqft_lot 0\n", "floors 0\n", "waterfront 0\n", "view 0\n", "condition 0\n", "grade 0\n", "sqft_above 0\n", "sqft_basement 0\n", "yr_built 0\n", "yr_renovated 0\n", "zipcode 0\n", "lat 0\n", "long 0\n", "sqft_living15 0\n", "sqft_lot15 0\n", "dtype: int64" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Let's check if our house dataset has any missing data\n", "house_df = pd.read_csv('kc_house_data.csv')\n", "house_df.head()\n", "house_df.isnull().sum()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Handling categorical data" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
colorsizepriceclasslabel
0greenM10.1class1
1redL13.5class2
2blueXL15.3class1
\n", "
" ], "text/plain": [ " color size price classlabel\n", "0 green M 10.1 class1\n", "1 red L 13.5 class2\n", "2 blue XL 15.3 class1" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "df = pd.DataFrame([\n", " ['green', 'M', 10.1, 'class1'], \n", " ['red', 'L', 13.5, 'class2'], \n", " ['blue', 'XL', 15.3, 'class1']])\n", "\n", "df.columns = ['color', 'size', 'price', 'classlabel']\n", "df" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Mapping ordinal features" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
colorsizepriceclasslabel
0green110.1class1
1red213.5class2
2blue315.3class1
\n", "
" ], "text/plain": [ " color size price classlabel\n", "0 green 1 10.1 class1\n", "1 red 2 13.5 class2\n", "2 blue 3 15.3 class1" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "size_mapping = {'XL': 3,\n", " 'L': 2,\n", " 'M': 1}\n", "\n", "df['size'] = df['size'].map(size_mapping)\n", "df" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 M\n", "1 L\n", "2 XL\n", "Name: size, dtype: object" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inv_size_mapping = {v: k for k, v in size_mapping.items()}\n", "df['size'].map(inv_size_mapping)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Encoding class labels" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'class1': 0, 'class2': 1}" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "class_mapping = {label:idx for idx,label in enumerate(np.unique(df['classlabel']))}\n", "class_mapping" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
colorsizepriceclasslabel
0green110.10
1red213.51
2blue315.30
\n", "
" ], "text/plain": [ " color size price classlabel\n", "0 green 1 10.1 0\n", "1 red 2 13.5 1\n", "2 blue 3 15.3 0" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['classlabel'] = df['classlabel'].map(class_mapping)\n", "df" ] }, { "cell_type": "code", "execution_count": 137, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "color object\n", "size int64\n", "price float64\n", "classlabel int64\n", "dtype: object" ] }, "execution_count": 137, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.dtypes" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
colorsizepriceclasslabel
0green110.1class1
1red213.5class2
2blue315.3class1
\n", "
" ], "text/plain": [ " color size price classlabel\n", "0 green 1 10.1 class1\n", "1 red 2 13.5 class2\n", "2 blue 3 15.3 class1" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inv_class_mapping = {v: k for k, v in class_mapping.items()}\n", "df['classlabel'] = df['classlabel'].map(inv_class_mapping)\n", "df" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([0, 1, 0])" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.preprocessing import LabelEncoder\n", "\n", "class_le = LabelEncoder()\n", "y = class_le.fit_transform(df['classlabel'].values)\n", "y" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array(['class1', 'class2', 'class1'], dtype=object)" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "class_le.inverse_transform(y)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Performing one-hot encoding on nominal features" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[['green' 1 10.1]\n", " ['red' 2 13.5]\n", " ['blue' 3 15.3]]\n" ] }, { "data": { "text/plain": [ "array([[1, 1, 10.1],\n", " [2, 2, 13.5],\n", " [0, 3, 15.3]], dtype=object)" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X = df[['color', 'size', 'price']].values\n", "print(X )\n", "color_le = LabelEncoder()\n", "X[:, 0] = color_le.fit_transform(X[:, 0])\n", "X" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\ProgramData\\Miniconda3\\envs\\py3x\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:415: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n", "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n", "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n", " warnings.warn(msg, FutureWarning)\n", "C:\\ProgramData\\Miniconda3\\envs\\py3x\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:451: DeprecationWarning: The 'categorical_features' keyword is deprecated in version 0.20 and will be removed in 0.22. You can use the ColumnTransformer instead.\n", " \"use the ColumnTransformer instead.\", DeprecationWarning)\n" ] }, { "data": { "text/plain": [ "array([[ 0. , 1. , 0. , 1. , 10.1],\n", " [ 0. , 0. , 1. , 2. , 13.5],\n", " [ 1. , 0. , 0. , 3. , 15.3]])" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.preprocessing import OneHotEncoder\n", "\n", "ohe = OneHotEncoder(categorical_features=[0])\n", "ohe.fit_transform(X).toarray()" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[0.0 1.0 0.0 1 10.1]\n", " [0.0 0.0 1.0 2 13.5]\n", " [1.0 0.0 0.0 3 15.3]]\n" ] } ], "source": [ "from sklearn.compose import ColumnTransformer\n", "\n", "X = df[['color', 'size', 'price']].values\n", "ct = ColumnTransformer(\n", " [('oh_enc', OneHotEncoder(categories='auto'), [0]),], # the column numbers I want to apply this to\n", " remainder='passthrough' # This leaves the rest of my columns in place\n", ")\n", "print(ct.fit_transform(X))" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pricesizecolor_bluecolor_greencolor_red
010.11010
113.52001
215.33100
\n", "
" ], "text/plain": [ " price size color_blue color_green color_red\n", "0 10.1 1 0 1 0\n", "1 13.5 2 0 0 1\n", "2 15.3 3 1 0 0" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.get_dummies(df[['price', 'color', 'size']])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Bringing features onto the same scale" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "A visual example:" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
inputstandardizednormalized
00-1.463850.0
11-0.878310.2
22-0.292770.4
330.292770.6
440.878310.8
551.463851.0
\n", "
" ], "text/plain": [ " input standardized normalized\n", "0 0 -1.46385 0.0\n", "1 1 -0.87831 0.2\n", "2 2 -0.29277 0.4\n", "3 3 0.29277 0.6\n", "4 4 0.87831 0.8\n", "5 5 1.46385 1.0" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ex = pd.DataFrame([0, 1, 2 ,3, 4, 5])\n", "\n", "# standardize\n", "ex[1] = (ex[0] - ex[0].mean()) / ex[0].std(ddof=0)\n", "\n", "# Please note that pandas uses ddof=1 (sample standard deviation) \n", "# by default, whereas NumPy's std method and the StandardScaler\n", "# uses ddof=0 (population standard deviation)\n", "\n", "# normalize\n", "ex[2] = (ex[0] - ex[0].min()) / (ex[0].max() - ex[0].min())\n", "ex.columns = ['input', 'standardized', 'normalized']\n", "ex" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Text(0.5, 1.0, 'normalized')" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABIAAAAIdCAYAAABFgAb9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df5Red10n8PeXJIVRCqMUgUwK7ZE6RwTWcWuBw7og1J0WOTRWhFZRwEr1aEUOdaRZXOzijy6MinKoQj2wRQRKKSFkseyACou6lG0hQGjraK1iM6m0QKeADJDEu3/cm/IknSQzyZPneeab1+uce/Lc771zn8/cZzLPd97P93tvaZomAAAAANTrAcMuAAAAAIDjSwAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQcMxKKS8qpfxNH4/39FLKrp71m0spT+/X8btjXl1K+a1+HhMAYC0opXyklPJz3eOfKqV8sM/HP62U0pRS1vfzuMCx8R8STgCllMuTPLZpmhcMu5aj0TTN9w27BgCAGjVN8/Ykbx92HcDxZwQQMFQ+GQIAODR9JaBfBEBQmVLKK0opC6WUr5RS5kspP5rkvyZ5finlq6WUT3f7vbiUcmu33+2llJ/vOcbTSym7SimXllLuKqXcWUp5cc/2h5VStpdSvlxK+X9JvvugGv6wlHJHt/0TpZQf6tl2eSnlulLKn5VSvpzkRaWUsW5K1j2llFuS/OBBx/vnUsrZ3ePF7vv4ainl37rhxad1255dSvlUt8//LaU8secYU6WUT3bf77uSPKhf5xwAYDldH+ZXSymfKaXcW0p5VynlQd22l5RSbiulfKnrV23s+bqmlPJLpZR/SPIPPW2/WEr5h64/85ullO8upXys63NdW0o5qdv3O0op7y+l3N31r95fStl0iBrvm8pfSvm1nn7WV0spe0opV3fbHlpKeXPXL1wopfxWKWVdt21dKeV3SylfKKXcnuRHj+NpBY6SAAgqUkqZTHJJkh9smubkJNNJ/i7J7yR5V9M0D26a5j90u9+V5NlJHpLkxUleV0r5gZ7DPTLJQ5NMJLkoyZWllO/otl2Z5OtJHpXkZ7ul141Jvj/JdyZ5R5J37+/sdM5Lcl2S8bRDjn8jbYj03V3NLzzU99g0zXj3fTw4yR8m+eskC13tb0ny80keluRNSbaXUh7YdYa2JXlbV9O7k/z4oZ4DAKCPnpfknCSnJ3li2g+/npHkim7bo5J8Lsk1B33d5iRPSvK4nrZzkvzHJE9O8mtJrkryU0lOTfL4JBd2+z0gyf9M8pgkj06ylOQNRyq0aZrX9vSzvjfJ3Umu7Ta/NcneJI9NMpXkvyT5uW7bS9L2K6eSnJnkuUd6LmDwBEBQl31JHpjkcaWUDU3T/HPTNP+43I5N0/x50zT/2LT+T5IPJvmhnl32JHl10zR7mqa5PslXk0x2n/T8eJJXNU3zb03TfDZth6D32H/WNM0Xm6bZ2zTN73U1Tfbs8rGmabY1TfPvTdMspe38/HbTNF9qmuaOJK8/0jdaSnl+kp9M8uNN0+xJ2/F4U9M0H2+aZl/TNG9N8o20HaQnJ9mQ5A+67+e6tCEVAMDx9vqmaXY3TfOlJP8r7YdkP5XkLU3TfLJpmm8k2ZLkKftHNXeu6PpGSz1tr2ma5stN09yc5LNJPtg0ze1N09yb5ANpA5h0/bD3NE3ztaZpvpLkt5M8baUFl1LG0n549odN01xfSnlEknOTvKzr/92V5HVJLui+5Hlp+1l3dN/nFas6Q8BACICgIk3T3JbkZUkuT3JXKeWa3uHEvUop55ZSbuiGHS8meVaSU3p2+WLTNHt71r+W5MFJHp72AvJ39Gz73EHHvrSbXnZvd+yHHnTs3q9Nko2HO94ytU+l/RTrx5qmubtrfkySS7vpX4vd857aHXtjkoWmaZqVPgcAQJ/8a8/j/f2pjenpizRN89UkX0w78nq/g/tLSfL5nsdLy6w/OElKKd9WSnlTKeVz3ZT7jyYZ3z9lawXenGS+aZrXdOuPSfth2p09/aw3Jfmubvuq+nLAcAiAoDJN07yjaZr/lPaNuknymu7f+5RSHpjkPUl+N8kjmqYZT3J9krKCp7g77fDfU3vaHt1z7B9K8oq0nwR9R3fsew869gH1JLnzUMc7WCnl4Unem+SSpml29Gy6I+0oovGe5duapnlnd/yJUkpvDYd8DgCA42x32r5akqSU8u1pp7Av9OxzcH9pNS5NO/r6SU3TPCTJf97/VEf6wlLKZd3XXtTTfEfakdWn9PSzHtJzp9YV9+WA4REAQUVKKZOllGd0Ac/X034StC/tp0OnlVL2/58/Ke20rLuT7C2lnJt2HvcRNU2zL8nWJJd3ny49Lgdes+fktAHR3UnWl1JelfY6Q4dzbZIt3QULNyX55UN8f+vTBldvb5rmXQdt/pMkv1BKeVJpfXsp5UdLKScn+VhX00tLKetLKecnOWsl3y8AwHHwjiQvLqV8f9dv+50kH2+a5p/7dPyT0/YDF0sp35n2eotH1PUJX5pkc+/Us6Zp7kx7uYDfK6U8pJTygO4C1PunlV2btp+1qbtm5GV9+j6APhIAQV0emOR/JPlC2uHG35X2DmDv7rZ/sZTyyW4u+EvTvlnfk/ZaOttX8TyXpB1i/K9Jrk57kcH95tLOQf/7tMN/v57lhzD3+u/dvv+UtnPxtkPstyntdYpedtAdKh7dNM1Naa8D9Ibue7otyYuSpGmabyY5v1u/J8nz04ZYAAAD1zTNXyb5b2k/2Loz7Y0wLjjsF63OHyQZS9snvCHJ/17h1z0/7XT/W3v6WW/stv1M2g8Rb0nbn7ou7QWsk/aDuLkkn07yyehnwUgqB14SAwAAAIDaGAEEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQufXDeNJTTjmlOe2004bx1ADAAHziE5/4QtM0Dx92HRxIHwwA6na4PthQAqDTTjstN9100zCeGgAYgFLK54ZdA/enDwYAdTtcH8wUMAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACABg9LwlyV1JPnuI7SXJ65PcluQzSX5gQHUBAGvU+mEXAADA/Vyd5A1J/vQQ289Ncka3PCnJH3f/AgAjZNuOhczOzWf34lI2jo9lZnoym6cmhlKLEUAAAKPno0m+dJjt56UNh5okNyQZT/KoAdQFAKzQth0L2bJ1ZxYWl9IkWVhcypatO7Ntx8JQ6hEAAQCsPRNJ7uhZ39W1AQAjYnZuPkt79h3QtrRnX2bn5odSjylgAABrT1mmrTnEvhd3CwAwQLsXl1bVfrwZAQQAsPbsSnJqz/qmJLsPse9VSc7sFgBgQDaOj62q/XgTAAEArD3bk/xM2pFAT05yb5I7h1oRAHCAmenJjG1Yd0Db2IZ1mZmeHEo9poABAIyedyZ5epJT0o72+Y0kG7ptb0xyfZJnpb0N/NeSvHjwJQIAh7P/bl+jchcwARAAwOi58AjbmyS/NIhCAICjt3lqYmiBz8FMAQMAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKtePAOjUJB9OcmuSm5P8Sh+OCQAAAECfrO/DMfYmuTTJJ5OcnOQTST6U5JY+HBsAAACAY9SPEUB3pg1/kuQraUcCTfThuAAAAAD0QT9GAPU6LclUko8vs+3ibgEAAABggPoZAD04yXuSvCzJl5fZflW3JEnTx+cFAAAA4DD6dRewDWnDn7cn2dqnYwIAAADQB/0IgEqSN6e99s/v9+F4AAAAAPRRPwKgpyb56STPSPKpbnlWH44LAAAAQB/04xpAf5N2FBAAAAAAI6hf1wACAAAAYEQJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKBy64ddAACwNm3bsZDZufnsXlzKxvGxzExPZvPUxLDLAgBgGQIgAGDVtu1YyJatO7O0Z1+SZGFxKVu27kwSIRDHTLgIAP1nChgAsGqzc/P3hT/7Le3Zl9m5+SFVRC32h4sLi0tp8q1wcduOhWGXBgBrmgAIAFi13YtLq2qHlRIuAsDxIQACAFZt4/jYqtphpYSLAHB8CIAAgFWbmZ7M2IZ1B7SNbViXmenJIVVELYSLAHB8CIAAgFXbPDWRK85/QibGx1KSTIyP5Yrzn+BCvRwz4SIAHB/uAgYAHJXNUxMCH/pu/8+Uu4ABQH8JgAAAGCnCRQDoP1PAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAYDSdk2Q+yW1JLltm+6OTfDjJjiSfSfKswZUGAKw1AiAAgNGzLsmVSc5N8rgkF3b/9vr1JNcmmUpyQZI/GmSBAMDaIgACABg9Z6Ud+XN7km8muSbJeQft0yR5SPf4oUl2D6w6AGDNWT/sAgAAuJ+JJHf0rO9K8qSD9rk8yQeT/HKSb09y9kAqAwDWJCOAAABGT1mmrTlo/cIkVyfZlPb6P2/L8n27i5Pc1C0AwAlKAAQAMHp2JTm1Z31T7j/F66K01wBKko8leVCSU5Y51lVJzuwWAOAEJQACABg9NyY5I8npSU5Ke5Hn7Qft8y9Jntk9/t60AdDdgyoQAFhbBEAAAKNnb5JLkswluTXtSJ+bk7w6yXO6fS5N8pIkn07yziQvyv2niQEAJHERaACAUXV9t/R6Vc/jW5I8dXDlAABrmRFAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJXrVwD0liR3Jflsn44HAAAAQJ+s79Nxrk7yhiR/2qfjAUdh246FzM7NZ/fiUjaOj2VmejKbpyaGXRYAAABD1q8A6KNJTuvTsYCjsG3HQrZs3ZmlPfuSJAuLS9mydWeSCIEAAABOcK4BBJWYnZu/L/zZb2nPvszOzQ+pIgAAAEZFv0YArcTF3QIcB7sXl1bVDgAAwIljkCOArkpyZrcAfbZxfGxV7QAAAJw4TAGDSsxMT2Zsw7oD2sY2rMvM9OSQKgIAAGBU9CsAemeSjyWZTLIryUV9Oi6wQpunJnLF+U/IxPhYSpKJ8bFccf4TXAAaAACAvl0D6MI+HQc4BpunJgQ+AAAA3I8pYAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJVbP+wCAAA4cWzbsZDZufnsXlzKxvGxzExPZvPUxLDLAoDqCYAAABiIbTsWsmXrzizt2ZckWVhcypatO5NECAQAx5kpYAAADMTs3Px94c9+S3v2ZXZufkgVAcCJQwAEAMBA7F5cWlU7ANA/AiAAAAZi4/jYqtoBgP4RAAEAMBAz05MZ27DugLaxDesyMz05pIoA4MThItAAAAzE/gs9uwsYAAyeAAgAgIHZPDUh8AGAITAFDAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAACo3PphFwCs3rYdC5mdm8/uxaVsHB/LzPRkNk9NDLssAAAARpQACNaYbTsWsmXrzizt2ZckWVhcypatO5NECAQAAMCyTAGDNWZ2bv6+8Ge/pT37Mjs3P6SKAAAAGHUCIFhjdi8uraodAAAABECwxmwcH1tVOwAAAAiAYI2ZmZ7M2IZ1B7SNbViXmenJIVUEwHFwTpL5JLcluewQ+zwvyS1Jbk7yjgHVBQCsUS4CDWvM/gs9uwsYQLXWJbkyyY8k2ZXkxiTb04Y9+52RZEuSpya5J8l3DbhGAGCNEQDBGrR5akLgA1Cvs9KO/Lm9W78myXk5MAB6SdqQ6J5u/a6BVQcArEmmgAEAjJaJJHf0rO/q2np9T7f8bZIb0k4ZO5SLk9zULQDACcoIIACA0VKWaWsOWl+fdhrY05NsSvLXSR6fZHGZr72qW5Y7DgBwgjACCABgtOxKcmrP+qYku5fZ531J9iT5p7QXjD5jINUBAGuSAAgAYLTcmDbMOT3JSUkuSHsR6F7bkvxw9/iUtNPBbg8AwCEIgAAARsveJJckmUtya5Jr097q/dVJntPtM5fki2kvDP3hJDPdOgDAslwDCABg9FzfLb1e1fO4SfLybgEAOCIjgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcv0KgM5JMp/ktiSX9emYAAAAAPTB+j4cY12SK5P8SJJdSW5Msj3JLX04NtDZtmMhs3Pz2b24lI3jY5mZnszmqYlhlwUAh+S9CwBGRz8CoLPSjvy5vVu/Jsl5EQBB32zbsZAtW3dmac++JMnC4lK2bN2ZJDrSAIwk710AMFr6MQVsIskdPeu7ujagT2bn5u/rQO+3tGdfZufmh1QRABye9y4AGC39GAFUlmlrlmm7uFuAVdq9uLSqdgAYNu9dADBa+jECaFeSU3vWNyXZvcx+VyU5s1uAVdg4PraqdgAYNu9dADBa+hEA3ZjkjCSnJzkpyQVpLwIN9MnM9GTGNqw7oG1sw7rMTE8OqSIAODzvXQAwWvoxBWxvkkuSzKW9I9hbktzch+MCnf0Xy3QnFQDWCu9dADBa+hEAJcn13QIcJ5unJnSaAVhTvHcBwOjoxxQwAAAAAEaYAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAYTeckmU9yW5LLDrPfc5M0Sc4cRFEAwNokAAIAGD3rklyZ5Nwkj0tyYffvwU5O8tIkHx9caQDAWiQAAgAYPWelHflze5JvJrkmyXnL7PebSV6b5OuDKw0AWIsEQAAAo2ciyR0967u6tl5TSU5N8v4jHOviJDd1CwBwglo/7AIAALifskxb0/P4AUlel+RFKzjWVd1y8DEAgBOIAAgqtW3HQmbn5rN7cSkbx8cyMz2ZzVMHf3gMwIjalXZ0z36bkuzuWT85yeOTfKRbf2SS7UmeEyN9AIBlCICgQtt2LGTL1p1Z2rMvSbKwuJQtW3cmiRAIYG24MckZSU5PspDkgiQ/2bP93iSn9Kx/JMmvRvgDAByCawBBhWbn5u8Lf/Zb2rMvs3PzQ6oIgFXam+SSJHNJbk1ybZKbk7w67SgfAIBVMQIIKrR7cWlV7QCMpOu7pderDrHv049vKQDAWmcEEFRo4/jYqtoBAAComwAIKjQzPZmxDesOaBvbsC4z05NDqggAAIBhMgUMKrT/Qs/uAgYAAEAiAIJqbZ6aEPgAAACQxBQwAAAAgOoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAyq0fdgEAAHA423YsZHZuPrsXl7JxfCwz05PZPDUx7LIAYE0RAAEAMLK27VjIlq07s7RnX5JkYXEpW7buTBIhEACsgilgAACMrNm5+fvCn/2W9uzL7Nz8kCoCgLVJAAQAwMjavbi0qnYAYHkCIAAARtbG8bFVtQMAyxMAAQAwsmamJzO2Yd0BbWMb1mVmenJIFQHA2uQi0AAAjKz9F3p2FzAAODYCIAAARtrmqQmBDwAcI1PAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKHWsA9BNJbk7y70nOPPZyAAAAAOi39cf49Z9Ncn6SN/WhFuAobNuxkNm5+exeXMrG8bHMTE9m89TEsMsCAABghBxrAHRrX6oAjsq2HQvZsnVnlvbsS5IsLC5ly9adSSIEAgAA4D6uAQRr2Ozc/H3hz35Le/Zldm5+SBUBAAAwilYyAugvkjxymfZXJnnfKp7r4m4B+mT34tKq2gEAADgxrSQAOrtPz3VVtyRJ06djwglt4/hYFpYJezaOjw2hGgAAAEaVKWCwhs1MT2Zsw7oD2sY2rMvM9OSQKgIAAGAUHWsA9GNJdiV5SpI/TzJ3zBUBK7Z5aiJXnP+ETIyPpSSZGB/LFec/wQWgAQAAOMCx3gXsvd0CDMnmqQmBDwAAAIdlChgAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAACj55wk80luS3LZMttfnuSWJJ9J8pdJHjO40gCAtUgABAAwWtYluTLJuUkel+TC7t9eO5KcmeSJSa5L8tpBFggArD0CIACA0XJW2pE/tyf5ZpJrkpx30D4fTvK17vENSTYNrDoAYE0SAAEAjJaJJHf0rO/q2g7loiQfOK4VAQBr3vphFwAAwAHKMm3NIfZ9QdqpYE87zPEu7hYA4AQmAAIAGC27kpzas74pye5l9js7ySvThj/fOMzxruqW5NBBEgBQOVPAAABGy41JzkhyepKTklyQZPtB+0wleVOS5yS5a6DVAQBrkgAIAGC07E1ySZK5JLcmuTbJzUlenTbwSZLZJA9O8u4kn8r9AyIAgAOYAgYAMHqu75Zer+p5fPYAawEAKmAEEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVG79sAsADm3bjoXMzs1n9+JSNo6PZWZ6MpunJoZdFgD0jfc6ABgMARCMqG07FrJl684s7dmXJFlYXMqWrTuTRMcYgCp4rwOAwTEFDEbU7Nz8fR3i/Zb27Mvs3PyQKgKA/vJeBwCDIwCCEbV7cWlV7QCw1nivA4DBEQDBiNo4PraqdgBYa7zXAcDgCIBgRM1MT2Zsw7oD2sY2rMvM9OSQKgKA/vJeBwCD4yLQMKL2X/zSnVEAqJX3OgAYHAEQjLDNUxM6wQBUzXsdAAyGKWAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVO9YAaDbJ3yX5TJL3Jhk/5ooAALxxJvkAAAkgSURBVAAA6KtjDYA+lOTxSZ6Y5O+TbDnmigAAAADoq2MNgD6YZG/3+IYkm47xeAAAAAD0WT+vAfSzST5wmO0XJ7mpWwAAAAAYkPUr2OcvkjxymfZXJnlfz+O9Sd5+mONc1S1J0qy0QAAAAACOzUoCoLOPsP2FSZ6d5JkR7AAAAACMnJUEQIdzTpJXJHlakq8dezkAAAAA9NuxXgPoDUlOTns3sE8leeMxVwQAAABAXx3rCKDH9qUKAAAAAI6bft4FDAAAAIARJAACAAAAqJwACAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKicAAgAAACgcgIgAAAAgMoJgAAAAAAqJwACAAAAqJwACAAAAKByAiAAgNF0TpL5JLcluWyZ7Q9M8q5u+8eTnDawygCANWf9sAsAAOB+1iW5MsmPJNmV5MYk25Pc0rPPRUnuSfLYJBckeU2S5w+2zG/ZtmMhs3Pz2b24lI3jY5mZnszmqYlhlQMAHMQIIACA0XNW2pE9tyf5ZpJrkpx30D7nJXlr9/i6JM9MUgZVYK9tOxayZevOLCwupUmysLiULVt3ZtuOhWGUAwAsQwAEADB6JpLc0bO+q2s71D57k9yb5GHHv7T7m52bz9KefQe0Le3Zl9m5+WGUAwAswxQwAIDRs9xInuYo9kmSi7vluNm9uLSqdgBg8IwAAgAYPbuSnNqzvinJ7sPssz7JQ5N8aZljXZXkzG45LjaOj62qHQAYPAEQAMDouTHJGUlOT3JS2os8bz9on+1JXtg9fm6Sv8ryI4COu5npyYxtWHdA29iGdZmZnhxGOQDAMkwBAwAYPXuTXJJkLu0dwd6S5OYkr05yU9rw581J3pb2YtFfShsSDcX+u325CxgAjC4BEADAaLq+W3q9qufx15P8xODKObzNUxMCHwAYYaaAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUTAAEAAABUTgAEAAAAUDkBEAAAAEDlBEAAAAAAlRMAAQAAAFROAAQAAABQOQEQAAAAQOUEQAAAAACVEwABAAAAVE4ABAAAAFA5ARAAAABA5QRAAAAAAJUrTdMM/klLuTvJ5wb+xMfBIx7xiFM+//nPf2HYdfAtXpPR5HUZPV6T0VPZa/KYpmkePuwiONAg+mCV/RyPPOd7cJzrwXGuB8e5HpwBnutD9sGGEgBV5qYkZw67CA7gNRlNXpfR4zUZPV4TauDneLCc78FxrgfHuR4c53pwhn6uTQEDAAAAqJwACAAAAKBy6y6//PJh11CDTwy7AO7HazKavC6jx2syerwm1MDP8WA534PjXA+Ocz04zvXgDPVcuwYQAAAAQOVMAQMAAAConADo2M0m+bskn0ny3iTjwy2Hzk8kuTnJv8dV7YftnCTzSW5LctmQa6H1liR3JfnssAvhPqcm+XCSW9P+7vqV4ZYDK3Kk3+8PTPKubvvHk5w2sMrqc6Rz/fIkt6Ttj/5lkscMrrTqrLTf8twkTfQzj9VKzvfz0v5835zkHQOqq0ZHOtePTtsX2ZH2d8mzBldaVY7Uzy5JXp/2dfhMkh8YUF1JBED98KEkj0/yxCR/n2TLcMuh89kk5yf56LALOcGtS3JlknOTPC7Jhd2/DNfVaTsBjI69SS5N8r1Jnpzkl+L/CqNtJb/fL0pyT5LHJnldktcMssCKrORc70gbRDwxyXVJXjvIAiuy0n7LyUlemjbY5Oit5Hyfkfbvq6cm+b4kLxtkgRVZybn+9STXJplKckGSPxpkgRW5OofvZ5+b9uf6jCQXJ/njAdR0HwHQsftg2o57ktyQZNMQa+Fbbk2bcDNcZ6VNt29P8s0k1yQ5b6gVkbTB6JeGXQQHuDPJJ7vHX0n7O2xieOXAEa3k9/t5Sd7aPb4uyTPTfvLJ6qzkXH84yde6x/qjR2+l/ZbfTBuyfX1wpVVpJef7JWmDi3u69bsGVl1dVnKumyQP6R4/NMnugVVXlyP1s89L8qdpz/cNaWcQPWoAdSURAPXbzyb5wLCLgBEykeSOnvVd8UctHMlpaT9988kyo2wlv99799mb5N4kDzv+pVVnte+lF0V/9Git5FxPpZ22+/5BFVWxlZzv7+mWv037x7IRzEdnJef68iQv6LZdn+SXB1LZiWeofx+tH9QTrXF/keSRy7S/Msn7eh7vTfL2QRXFil4Xhmu5T3rdehAO7cFJ3pN2iPuXh1wLHM5Kfr97D+iP1ZzHF6SdCva041dO1Y50rh+QdjrjiwZSTf1W8rO9Pu1UmaenHdn212kvv7F4XCurz0rO9YVppy/9XpKnJHlb2nP978e1shPPUN8bBUArc/YRtr8wybPTDm3WsRmcI70uDN+utJ+S7bcphpPCoWxIG/68PcnWIdcCR7KS3+/799mVts/50Jh+ejRW+l56dtoPwZ6W5BsDqKtGRzrXJ6f9g/gj3fojk2xP8pwkNw2gvtqs9PfIDUn2JPmntJd4OCPJjYMosCIrOdcX5VsjrD6W5EFJTolpd/021L+PTAE7duckeUXaX/xfO8K+cKK5Me2b9OlJTkp7QbntQ60IRlNJ8ua01/75/SHXAiuxkt/v29N+SJa0d0z6q/ig7Gis5FxPJXlT2v6oP9aO3pHO9b1p/yA+rVtuiPDnWKzkZ3tbkh/uHp+SdjrY7YMqsCIrOdf/knZAQ9LelOJBSe4eVIEnkO1JfiZt3+/JaX+v3DmoJxcAHbs3pP004ENJPpXkjcMth86PpU1Xn5Lkz5PMDbecE9beJJekPf+3pr2zwM1DrYgkeWfaT3Ym0/4/uWi45ZD27iY/neQZad9LPhW3X2W0Her3+6vT/kGctKHmw9JeePTlOfwttTm0lZzr2bRTSN+d9veHD1uOzkrONf2zkvM9l+SLaW8D/+EkM906q7OSc31p2otufzptX/FFEdofjeX62b/QLUl7faXb0743/kmSXxxkcaVpvKYAAAAANTMCCAAAAKByAiAAAACAygmAAAAAAConAAIAAAConAAIAAAAoHICIAAAAIDKCYAAAAAAKicAAgAAAKjc/wcokP5hrWplXwAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "dark" }, "output_type": "display_data" } ], "source": [ "# Now, what happens when you have an outlier?\n", "import matplotlib.pyplot as plt\n", "ex = pd.DataFrame([0, 1, 2 ,3, 4, 5,100])\n", "ex1=pd.DataFrame([0,3,4,1,5,2,6])\n", "\n", "# standardize\n", "ex[1] = (ex[0] - ex[0].mean()) / ex[0].std(ddof=0)\n", "ex1[1] = (ex1[0] - ex1[0].mean()) / ex1[0].std(ddof=0)\n", "# normalize\n", "ex[2] = (ex[0] - ex[0].min()) / (ex[0].max() - ex[0].min())\n", "ex1[2] = (ex1[0] - ex1[0].min()) / (ex1[0].max() - ex1[0].min())\n", "\n", "plt.figure(figsize=(20,20))\n", "\n", "plt.subplot(221)\n", "plt.scatter(ex[1],ex1[1])\n", "plt.xlim(-2.5, 2.5)\n", "plt.ylim(-2.5, 2.5)\n", "plt.tick_params(axis='both', colors='white')\n", "plt.title(\"standardized\")\n", "\n", "plt.subplot(222)\n", "plt.scatter(ex[2],ex1[2])\n", "plt.tick_params(axis='both', colors='white')\n", "plt.title(\"normalized\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "
\n", "\n", "### Note:\n", "\n", "\n", "If the link to the Wine dataset provided above does not work for you, you can find a local copy in this repository at [./../datasets/wine/wine.data](./../datasets/wine.data).\n", "\n", "Or you could fetch it via\n" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Class labelAlcoholMalic acidAshAlcalinity of ashMagnesiumTotal phenolsFlavanoidsNonflavanoid phenolsProanthocyaninsColor intensityHueOD280/OD315 of diluted winesProline
0114.231.712.4315.61272.803.060.282.295.641.043.921065
1113.201.782.1411.21002.652.760.261.284.381.053.401050
2113.162.362.6718.61012.803.240.302.815.681.033.171185
3114.371.952.5016.81133.853.490.242.187.800.863.451480
4113.242.592.8721.01182.802.690.391.824.321.042.93735
\n", "
" ], "text/plain": [ " Class label Alcohol Malic acid Ash Alcalinity of ash Magnesium \\\n", "0 1 14.23 1.71 2.43 15.6 127 \n", "1 1 13.20 1.78 2.14 11.2 100 \n", "2 1 13.16 2.36 2.67 18.6 101 \n", "3 1 14.37 1.95 2.50 16.8 113 \n", "4 1 13.24 2.59 2.87 21.0 118 \n", "\n", " Total phenols Flavanoids Nonflavanoid phenols Proanthocyanins \\\n", "0 2.80 3.06 0.28 2.29 \n", "1 2.65 2.76 0.26 1.28 \n", "2 2.80 3.24 0.30 2.81 \n", "3 3.85 3.49 0.24 2.18 \n", "4 2.80 2.69 0.39 1.82 \n", "\n", " Color intensity Hue OD280/OD315 of diluted wines Proline \n", "0 5.64 1.04 3.92 1065 \n", "1 4.38 1.05 3.40 1050 \n", "2 5.68 1.03 3.17 1185 \n", "3 7.80 0.86 3.45 1480 \n", "4 4.32 1.04 2.93 735 " ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_wine = pd.read_csv('https://raw.githubusercontent.com/rasbt/python-machine-learning-book/master/code/datasets/wine/wine.data', header=None)\n", "\n", "df_wine.columns = ['Class label', 'Alcohol', 'Malic acid', 'Ash', \n", "'Alcalinity of ash', 'Magnesium', 'Total phenols', \n", "'Flavanoids', 'Nonflavanoid phenols', 'Proanthocyanins', \n", "'Color intensity', 'Hue', 'OD280/OD315 of diluted wines', 'Proline']\n", "df_wine.head()" ] }, { "cell_type": "code", "execution_count": 146, "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Class labels [1 2 3]\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Class labelAlcoholMalic acidAshAlcalinity of ashMagnesiumTotal phenolsFlavanoidsNonflavanoid phenolsProanthocyaninsColor intensityHueOD280/OD315 of diluted winesProline
0114.231.712.4315.61272.803.060.282.295.641.043.921065
1113.201.782.1411.21002.652.760.261.284.381.053.401050
2113.162.362.6718.61012.803.240.302.815.681.033.171185
3114.371.952.5016.81133.853.490.242.187.800.863.451480
4113.242.592.8721.01182.802.690.391.824.321.042.93735
\n", "
" ], "text/plain": [ " Class label Alcohol Malic acid Ash Alcalinity of ash Magnesium \\\n", "0 1 14.23 1.71 2.43 15.6 127 \n", "1 1 13.20 1.78 2.14 11.2 100 \n", "2 1 13.16 2.36 2.67 18.6 101 \n", "3 1 14.37 1.95 2.50 16.8 113 \n", "4 1 13.24 2.59 2.87 21.0 118 \n", "\n", " Total phenols Flavanoids Nonflavanoid phenols Proanthocyanins \\\n", "0 2.80 3.06 0.28 2.29 \n", "1 2.65 2.76 0.26 1.28 \n", "2 2.80 3.24 0.30 2.81 \n", "3 3.85 3.49 0.24 2.18 \n", "4 2.80 2.69 0.39 1.82 \n", "\n", " Color intensity Hue OD280/OD315 of diluted wines Proline \n", "0 5.64 1.04 3.92 1065 \n", "1 4.38 1.05 3.40 1050 \n", "2 5.68 1.03 3.17 1185 \n", "3 7.80 0.86 3.45 1480 \n", "4 4.32 1.04 2.93 735 " ] }, "execution_count": 146, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data', header=None)\n", "\n", "df_wine.columns = ['Class label', 'Alcohol', 'Malic acid', 'Ash', \n", "'Alcalinity of ash', 'Magnesium', 'Total phenols', \n", "'Flavanoids', 'Nonflavanoid phenols', 'Proanthocyanins', \n", "'Color intensity', 'Hue', 'OD280/OD315 of diluted wines', 'Proline']\n", "\n", "print('Class labels', np.unique(df_wine['Class label']))\n", "df_wine.head()" ] }, { "cell_type": "code", "execution_count": 184, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Types of grapes [1 2 3]\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Class labelAlcoholMalic acidAshAlcalinity of ashMagnesiumTotal phenolsFlavanoidsNonflavanoid phenolsProanthocyaninsColor intensityHueOD280/OD315 of diluted winesProline
0114.231.712.4315.61272.803.060.282.295.641.043.921065
1113.201.782.1411.21002.652.760.261.284.381.053.401050
2113.162.362.6718.61012.803.240.302.815.681.033.171185
3114.371.952.5016.81133.853.490.242.187.800.863.451480
4113.242.592.8721.01182.802.690.391.824.321.042.93735
\n", "
" ], "text/plain": [ " Class label Alcohol Malic acid Ash Alcalinity of ash Magnesium \\\n", "0 1 14.23 1.71 2.43 15.6 127 \n", "1 1 13.20 1.78 2.14 11.2 100 \n", "2 1 13.16 2.36 2.67 18.6 101 \n", "3 1 14.37 1.95 2.50 16.8 113 \n", "4 1 13.24 2.59 2.87 21.0 118 \n", "\n", " Total phenols Flavanoids Nonflavanoid phenols Proanthocyanins \\\n", "0 2.80 3.06 0.28 2.29 \n", "1 2.65 2.76 0.26 1.28 \n", "2 2.80 3.24 0.30 2.81 \n", "3 3.85 3.49 0.24 2.18 \n", "4 2.80 2.69 0.39 1.82 \n", "\n", " Color intensity Hue OD280/OD315 of diluted wines Proline \n", "0 5.64 1.04 3.92 1065 \n", "1 4.38 1.05 3.40 1050 \n", "2 5.68 1.03 3.17 1185 \n", "3 7.80 0.86 3.45 1480 \n", "4 4.32 1.04 2.93 735 " ] }, "execution_count": 184, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print('Types of grapes', np.unique(df_wine['Class label']))\n", "df_wine.head()" ] }, { "cell_type": "code", "execution_count": 186, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Class label 0\n", "Alcohol 0\n", "Malic acid 0\n", "Ash 0\n", "Alcalinity of ash 0\n", "Magnesium 0\n", "Total phenols 0\n", "Flavanoids 0\n", "Nonflavanoid phenols 0\n", "Proanthocyanins 0\n", "Color intensity 0\n", "Hue 0\n", "OD280/OD315 of diluted wines 0\n", "Proline 0\n", "dtype: int64" ] }, "execution_count": 186, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_wine.isnull().sum()\n", "# Yay! we got lucky, no NaNs." ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", "\n", "X, y = df_wine.iloc[:, 1:].values, df_wine.iloc[:, 0].values\n", "\n", "X_train, X_test, y_train, y_test = \\\n", " train_test_split(X, y, test_size=0.3, random_state=0)" ] }, { "cell_type": "code", "execution_count": 222, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.30337078651685395" ] }, "execution_count": 222, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(X_test)/(len(X_test)+len(X_train))" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[0.72043011, 0.20378151, 0.53763441, ..., 0.48717949, 1. ,\n", " 0.5854251 ],\n", " [0.31989247, 0.08403361, 0.31182796, ..., 0.27350427, 0.64102564,\n", " 0. ],\n", " [0.60215054, 0.71218487, 0.48387097, ..., 0.04273504, 0.10622711,\n", " 0.42348178],\n", " ...,\n", " [0.37365591, 0.1512605 , 0.44623656, ..., 0.44444444, 0.61904762,\n", " 0.02672065],\n", " [0.77150538, 0.16596639, 0.40860215, ..., 0.31623932, 0.75457875,\n", " 0.54493927],\n", " [0.84139785, 0.34033613, 0.60215054, ..., 0.06837607, 0.16117216,\n", " 0.28178138]])" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.preprocessing import MinMaxScaler\n", "mms = MinMaxScaler()\n", "X_train_norm = mms.fit_transform(X_train)\n", "X_test_norm = mms.transform(X_test)\n", "X_train_norm" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 0.91083058, -0.46259897, -0.01142613, ..., 0.65706596,\n", " 1.94354495, 0.93700997],\n", " [-0.95609928, -0.96608672, -1.53725357, ..., -0.40859506,\n", " 0.58118003, -1.41336684],\n", " [ 0.35952243, 1.67501572, -0.37471838, ..., -1.55950896,\n", " -1.44846566, 0.28683658],\n", " ...,\n", " [-0.70550467, -0.68342693, -0.62902295, ..., 0.44393375,\n", " 0.49776993, -1.30608823],\n", " [ 1.14889546, -0.6215951 , -0.88332752, ..., -0.19546286,\n", " 1.0121322 , 0.77446662],\n", " [ 1.47466845, 0.11155374, 0.42452457, ..., -1.43162964,\n", " -1.23994042, -0.28206514]])" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.preprocessing import StandardScaler\n", "stdsc = StandardScaler()\n", "X_train_std = stdsc.fit_transform(X_train)\n", "X_test_std = stdsc.transform(X_test)\n", "X_train_std" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# What changes if you standardize/normalize the whole data set?" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 1 }