DAT4/code/07_model_evaluation_procedures.py at master · justmarkham/DAT4 · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

'''

CLASS: Model evaluation procedures

'''

import numpy as np

import matplotlib.pyplot as plt

# read in the iris data

from sklearn.datasets import load_iris

iris = load_iris()

X, y = iris.data, iris.target

## TRAIN AND TEST ON THE SAME DATA (OVERFITTING)

from sklearn.neighbors import KNeighborsClassifier

knn = KNeighborsClassifier(n_neighbors=1)

knn.fit(X, y)

knn.score(X, y)

## TEST SET APPROACH

# understanding train_test_split

from sklearn.cross_validation import train_test_split

features = np.array([range(10), range(10, 20)]).T

response = ['even', 'odd'] * 5

features_train, features_test = train_test_split(features)

features_train

features_test

features_train, features_test, response_train, response_test = train_test_split(features, response, random_state=1)

features_train

features_test

response_train

response_test

# step 1: split data into training set and test set

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=4)

X_train.shape

X_test.shape

y_train.shape

y_test.shape

# steps 2 and 3: calculate test set error for K=1

knn = KNeighborsClassifier(n_neighbors=1)

knn.fit(X_train, y_train)

knn.score(X_test, y_test)

# step 4 (parameter tuning): calculate test set error for K=5

knn = KNeighborsClassifier(n_neighbors=5)

knn.fit(X_train, y_train)

knn.score(X_test, y_test)

# steps 5 and 6: choose best model (K=5) and train on all data

knn = KNeighborsClassifier(n_neighbors=5)

knn.fit(X, y)

# step 7: make predictions on new ("out of sample") data

out_of_sample = [[5, 4, 3, 2], [4, 3, 2, 1]]

knn.predict(out_of_sample)

# verify that a different train/test split can result in a different test set error

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)

knn = KNeighborsClassifier(n_neighbors=5)

knn.fit(X_train, y_train)

knn.score(X_test, y_test)

## CROSS-VALIDATION

# check CV score for K=1

from sklearn.cross_validation import cross_val_score

knn = KNeighborsClassifier(n_neighbors=1)

scores = cross_val_score(knn, X, y, cv=5, scoring='accuracy')

scores

np.mean(scores)

# check CV score for K=5

knn = KNeighborsClassifier(n_neighbors=5)

scores = cross_val_score(knn, X, y, cv=5, scoring='accuracy')

scores

np.mean(scores)

# search for an optimal value of K

k_range = range(1, 30, 2)

scores = []

for k in k_range:

knn = KNeighborsClassifier(n_neighbors=k)

scores.append(np.mean(cross_val_score(knn, X, y, cv=5, scoring='accuracy')))

scores

# plot the K values (x-axis) versus the 5-fold CV score (y-axis)

plt.figure()

plt.plot(k_range, scores)

# automatic grid search for an optimal value of K

from sklearn.grid_search import GridSearchCV

knn = KNeighborsClassifier()

k_range = range(1, 30, 2)

param_grid = dict(n_neighbors=k_range)

grid = GridSearchCV(knn, param_grid, cv=5, scoring='accuracy')

grid.fit(X, y)

# check the results of the grid search

grid.grid_scores_

grid_mean_scores = [result[1] for result in grid.grid_scores_]

plt.figure()

plt.plot(k_range, grid_mean_scores)

grid.best_score_

grid.best_params_

grid.best_estimator_

DAT4/code/07_model_evaluation_procedures.py at master · justmarkham/DAT4 · GitHub

Repository navigation

Expand file tree

Breadcrumbs

07_model_evaluation_procedures.py

Latest commit

History

Breadcrumbs

07_model_evaluation_procedures.py

File metadata and controls