AISE
/
iui-group-l-name-zensiert
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Too many changes to properly fit in a commit msg, changes will be discussed on discord or simply ask me

master
Tuan-Dat Tran 2021-05-17 01:34:11 +00:00
parent 04d9431cae
commit 62fae7c77b
7 changed files with 1976 additions and 633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

522
0-pilot-project/MNIST-kNN-best-pipeline.ipynb Normal file
View File

@ -0,0 +1,522 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e4d89124",
"metadata": {},
"source": [
"### Load MNIST dataset"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "5789ec72",
"metadata": {},
"outputs": [],
"source": [
"# Python ≥3.5 is required\n",
"import sys\n",
"assert sys.version_info >= (3, 5)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "f491d383",
"metadata": {},
"outputs": [],
"source": [
"# scikit-learn ≥0.20 is required\n",
"import sklearn\n",
"assert sklearn.__version__ >= \"0.20\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "575a6a42",
"metadata": {},
"outputs": [],
"source": [
"# common imports\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "921dc114",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"sklearn.utils.Bunch"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# import function to scikit-learn datasets\n",
"from sklearn.datasets import fetch_openml\n",
"\n",
"# load specified dataset (MNIST)\n",
"mnist = fetch_openml('mnist_784', version=1, as_frame=False)\n",
"\n",
"# print type of dataset\n",
"type(mnist)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "6271045c",
"metadata": {},
"outputs": [],
"source": [
"X, y = mnist[\"data\"], mnist[\"target\"]"
]
},
{
"cell_type": "markdown",
"id": "37777133",
"metadata": {},
"source": [
"### Fix labels"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "30a441d3",
"metadata": {},
"outputs": [],
"source": [
"# import plotting libraries\n",
"import matplotlib as mpl\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "2d9693b1",
"metadata": {},
"outputs": [],
"source": [
"# convert string labels to int\n",
"y = y.astype(np.uint8)"
]
},
{
"cell_type": "markdown",
"id": "182f4b1b",
"metadata": {},
"source": [
"### Prepare data for machine learning"
]
},
{
"cell_type": "markdown",
"id": "77ff6bd1",
"metadata": {},
"source": [
"### Identify Train Set and Test Set"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "f8247d13",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"X_train: 56000, (56000, 784)\n",
"X_test: 14000, (14000, 784)\n",
"y_train: 56000, (56000,)\n",
"y_test: 14000, (14000,)\n"
]
}
],
"source": [
"from sklearn.model_selection import train_test_split\n",
"\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1337)\n",
"\n",
"print(f\"X_train: {len(X_train)}, {X_train.shape}\")\n",
"print(f\"X_test: {len(X_test)}, {X_test.shape}\")\n",
"print(f\"y_train: {len(y_train)}, {y_train.shape}\")\n",
"print(f\"y_test: {len(y_test)}, {y_test.shape}\")"
]
},
{
"cell_type": "markdown",
"id": "c4062436",
"metadata": {},
"source": [
"## Pipeline Declaration"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "67645a47",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.pipeline import Pipeline\n",
"from sklearn.decomposition import PCA\n",
"from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
"from sklearn.neighbors import KNeighborsClassifier\n",
"from sklearn.model_selection import cross_val_predict\n",
"from sklearn.metrics import classification_report, precision_score\n",
"\n",
"n_neighbors = 3\n",
"n95_components = 0.95\n",
"n99_components = 0.99"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "82b1e834",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"6"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"names = ['knn', \n",
" 'scalar+knn', \n",
" 'standard+pca95+knn', \n",
" 'minmax+pca95+knn', \n",
" 'standard+pca99+knn', \n",
" 'minmax+pca99+knn'\n",
" ]\n",
"\n",
"classifiers = [\n",
" Pipeline([('knn', KNeighborsClassifier(n_neighbors=n_neighbors))]),\n",
" Pipeline([\n",
" ('scaler', StandardScaler()),\n",
" ('knn', KNeighborsClassifier(n_neighbors=n_neighbors))\n",
" ]),\n",
" Pipeline([\n",
" ('standard', StandardScaler()),\n",
" ('pca', PCA(n_components=n95_components)),\n",
" ('knn', KNeighborsClassifier(n_neighbors=n_neighbors))\n",
" ]),\n",
" Pipeline([\n",
" ('minmax', MinMaxScaler()),\n",
" ('pca', PCA(n_components=n95_components)),\n",
" ('knn', KNeighborsClassifier(n_neighbors=n_neighbors))\n",
" ]),\n",
" Pipeline([\n",
" ('standard', StandardScaler()),\n",
" ('pca', PCA(n_components=n99_components)),\n",
" ('knn', KNeighborsClassifier(n_neighbors=n_neighbors))\n",
" ]),\n",
" Pipeline([\n",
" ('minmax', MinMaxScaler()),\n",
" ('pca', PCA(n_components=n99_components)),\n",
" ('knn', KNeighborsClassifier(n_neighbors=n_neighbors))\n",
" ])\n",
"]\n",
"\n",
"len(names)"
]
},
{
"cell_type": "markdown",
"id": "156dbf2c",
"metadata": {},
"source": [
"# Crossvalidation"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "8e5168e4",
"metadata": {},
"outputs": [],
"source": [
"# for name, clf in zip(names, classifiers):\n",
"# y_train_pred = cross_val_predict(clf, X_train, y_train, cv=3)\n",
"# print(f\"Pipeline: {name} ({precision_score(y_train, y_train_pred, average='weighted'):.4f})\")\n",
"# print(classification_report(y_train, y_train_pred))\n",
"precs = []"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "db14a027",
"metadata": {},
"outputs": [],
"source": [
"def cv(num):\n",
" name = names[num]\n",
" clf = classifiers[num]\n",
" y_train_pred = cross_val_predict(clf, X_train, y_train, cv=3)\n",
" precision = precision_score(y_train, y_train_pred, average='weighted')\n",
" print(f\"Pipeline: {name} ({precision:.4f})\")\n",
" print(classification_report(y_train, y_train_pred))\n",
" return precision"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "b8983bf8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline: knn (0.9691)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.98 0.99 0.99 5499\n",
" 1 0.95 0.99 0.97 6287\n",
" 2 0.98 0.96 0.97 5595\n",
" 3 0.96 0.97 0.96 5679\n",
" 4 0.98 0.96 0.97 5450\n",
" 5 0.96 0.96 0.96 5068\n",
" 6 0.98 0.99 0.98 5542\n",
" 7 0.96 0.97 0.97 5846\n",
" 8 0.99 0.93 0.96 5504\n",
" 9 0.95 0.96 0.96 5530\n",
"\n",
" accuracy 0.97 56000\n",
" macro avg 0.97 0.97 0.97 56000\n",
"weighted avg 0.97 0.97 0.97 56000\n",
"\n"
]
}
],
"source": [
"cv(0)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "62ff42f4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline: scalar+knn (0.9420)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.95 0.99 0.97 5499\n",
" 1 0.95 0.99 0.97 6287\n",
" 2 0.95 0.93 0.94 5595\n",
" 3 0.92 0.94 0.93 5679\n",
" 4 0.94 0.93 0.94 5450\n",
" 5 0.93 0.92 0.93 5068\n",
" 6 0.96 0.97 0.97 5542\n",
" 7 0.94 0.93 0.94 5846\n",
" 8 0.97 0.89 0.93 5504\n",
" 9 0.91 0.92 0.91 5530\n",
"\n",
" accuracy 0.94 56000\n",
" macro avg 0.94 0.94 0.94 56000\n",
"weighted avg 0.94 0.94 0.94 56000\n",
"\n"
]
}
],
"source": [
"cv(1)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "9b553141",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline: standard+pca95+knn (0.9457)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.96 0.99 0.97 5499\n",
" 1 0.95 0.99 0.97 6287\n",
" 2 0.95 0.94 0.95 5595\n",
" 3 0.93 0.94 0.94 5679\n",
" 4 0.95 0.93 0.94 5450\n",
" 5 0.93 0.92 0.93 5068\n",
" 6 0.96 0.97 0.97 5542\n",
" 7 0.95 0.94 0.94 5846\n",
" 8 0.97 0.89 0.93 5504\n",
" 9 0.92 0.92 0.92 5530\n",
"\n",
" accuracy 0.95 56000\n",
" macro avg 0.95 0.94 0.94 56000\n",
"weighted avg 0.95 0.95 0.95 56000\n",
"\n"
]
}
],
"source": [
"cv(2)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "2a1d6c65",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline: minmax+pca95+knn (0.9706)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.98 0.99 0.99 5499\n",
" 1 0.96 0.99 0.98 6287\n",
" 2 0.98 0.97 0.97 5595\n",
" 3 0.96 0.96 0.96 5679\n",
" 4 0.98 0.97 0.97 5450\n",
" 5 0.96 0.96 0.96 5068\n",
" 6 0.98 0.99 0.98 5542\n",
" 7 0.97 0.98 0.97 5846\n",
" 8 0.99 0.93 0.96 5504\n",
" 9 0.95 0.96 0.96 5530\n",
"\n",
" accuracy 0.97 56000\n",
" macro avg 0.97 0.97 0.97 56000\n",
"weighted avg 0.97 0.97 0.97 56000\n",
"\n"
]
}
],
"source": [
"cv(3)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "de40f817",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline: standard+pca99+knn (0.9424)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.95 0.99 0.97 5499\n",
" 1 0.95 0.99 0.97 6287\n",
" 2 0.95 0.93 0.94 5595\n",
" 3 0.92 0.94 0.93 5679\n",
" 4 0.94 0.93 0.94 5450\n",
" 5 0.93 0.92 0.93 5068\n",
" 6 0.96 0.97 0.97 5542\n",
" 7 0.94 0.93 0.94 5846\n",
" 8 0.97 0.89 0.93 5504\n",
" 9 0.91 0.92 0.92 5530\n",
"\n",
" accuracy 0.94 56000\n",
" macro avg 0.94 0.94 0.94 56000\n",
"weighted avg 0.94 0.94 0.94 56000\n",
"\n"
]
}
],
"source": [
"cv(4)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "2c1c26b0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline: minmax+pca99+knn (0.9695)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.98 0.99 0.99 5499\n",
" 1 0.95 0.99 0.97 6287\n",
" 2 0.98 0.96 0.97 5595\n",
" 3 0.96 0.97 0.96 5679\n",
" 4 0.98 0.96 0.97 5450\n",
" 5 0.96 0.96 0.96 5068\n",
" 6 0.98 0.99 0.98 5542\n",
" 7 0.96 0.97 0.97 5846\n",
" 8 0.99 0.93 0.96 5504\n",
" 9 0.95 0.96 0.96 5530\n",
"\n",
" accuracy 0.97 56000\n",
" macro avg 0.97 0.97 0.97 56000\n",
"weighted avg 0.97 0.97 0.97 56000\n",
"\n"
]
}
],
"source": [
"cv(5)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb6b6d69",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

933
0-pilot-project/MNIST-kNN-best.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

782
0-pilot-project/MNIST-kNN.ipynb
View File

File diff suppressed because one or more lines are too long

163
0-pilot-project/MNIST.ipynb
View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "9507bfb9", "id": "7a0c752a",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Load MNIST dataset" "### Load MNIST dataset"
@ -11,7 +11,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 1,
"id": "1ed54820", "id": "e07d82fe",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -23,7 +23,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 2,
"id": "532fc961", "id": "1f97dcb1",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -35,7 +35,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 3,
"id": "37391208", "id": "01f83832",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -46,7 +46,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 4,
"id": "aaf9d74d", "id": "affa0e2b",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -73,7 +73,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "02444b3c", "id": "4d51fd43",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Bunch objects are sometimes used as an output for functions and methods. They extend dictionaries by enabling values to be accessed by key, bunch[\"value_key\"], or by an attribute, bunch.value_key.\\\n", "Bunch objects are sometimes used as an output for functions and methods. They extend dictionaries by enabling values to be accessed by key, bunch[\"value_key\"], or by an attribute, bunch.value_key.\\\n",
@ -83,7 +83,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 5,
"id": "60abd5bd", "id": "78be57ab",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -106,7 +106,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 6,
"id": "f9eb1a3e", "id": "d0450c41",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -127,7 +127,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "6ffbf39a", "id": "e61e2adb",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Datasets loaded by Scikit-Learn generally have a similar dictionary structure, including the following:\\\n", "Datasets loaded by Scikit-Learn generally have a similar dictionary structure, including the following:\\\n",
@ -139,13 +139,13 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 7,
"id": "2b3ee5f6", "id": "fe285433",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"\"The MNIST database of handwritten digits with 784 features. It can be split in a training set of the first 60,000 examples, and a test set of 10,000 examples \\n\\nIt is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. The original black and white (bilevel) images from NIST were size normalized to fit in a 20x20 pixel box while preserving their aspect ratio. The resulting images contain grey levels as a result of the anti-aliasing technique used by the normalization algorithm. the images were centered in a 28x28 image by computing the center of mass of the pixels, and translating the image so as to position this point at the center of the 28x28 field. \\n\\nWith some classification methods (particularly template-based methods, such as SVM and K-nearest neighbors), the error rate improves when the digits are centered by bounding box rather than center of mass. If you do this kind of pre-processing, you should report it in your publications. The MNIST database was constructed from NIST's NIST originally designated SD-3 as their training set and SD-1 as their test set. However, SD-3 is much cleaner and easier to recognize than SD-1. The reason for this can be found on the fact that SD-3 was collected among Census Bureau employees, while SD-1 was collected among high-school students. Drawing sensible conclusions from learning experiments requires that the result be independent of the choice of training set and test among the complete set of samples. Therefore it was necessary to build a new database by mixing NIST's datasets. \\n\\nThe MNIST training set is composed of 30,000 patterns from SD-3 and 30,000 patterns from SD-1. Our test set was composed of 5,000 patterns from SD-3 and 5,000 patterns from SD-1. The 60,000 pattern training set contained examples from approximately 250 writers. We made sure that the sets of writers of the training set and test set were disjoint. SD-1 contains 58,527 digit images written by 500 different writers. In contrast to SD-3, where blocks of data from each writer appeared in sequence, the data in SD-1 is scrambled. Writer identities for SD-1 is available and we used this information to unscramble the writers. We then split SD-1 in two: characters written by the first 250 writers went into our new training set. The remaining 250 writers were placed in our test set. Thus we had two sets with nearly 30,000 examples each. The new training set was completed with enough examples from SD-3, starting at pattern # 0, to make a full set of 60,000 training patterns. Similarly, the new test set was completed with SD-3 examples starting at pattern # 35,000 to make a full set with 60,000 test patterns. Only a subset of 10,000 test images (5,000 from SD-1 and 5,000 from SD-3) is available on this site. The full 60,000 sample training set is available.\\n\\nDownloaded from openml.org.\"" "\"**Author**: Yann LeCun, Corinna Cortes, Christopher J.C. Burges \\n**Source**: [MNIST Website](http://yann.lecun.com/exdb/mnist/) - Date unknown \\n**Please cite**: \\n\\nThe MNIST database of handwritten digits with 784 features, raw data available at: http://yann.lecun.com/exdb/mnist/. It can be split in a training set of the first 60,000 examples, and a test set of 10,000 examples \\n\\nIt is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. The original black and white (bilevel) images from NIST were size normalized to fit in a 20x20 pixel box while preserving their aspect ratio. The resulting images contain grey levels as a result of the anti-aliasing technique used by the normalization algorithm. the images were centered in a 28x28 image by computing the center of mass of the pixels, and translating the image so as to position this point at the center of the 28x28 field. \\n\\nWith some classification methods (particularly template-based methods, such as SVM and K-nearest neighbors), the error rate improves when the digits are centered by bounding box rather than center of mass. If you do this kind of pre-processing, you should report it in your publications. The MNIST database was constructed from NIST's NIST originally designated SD-3 as their training set and SD-1 as their test set. However, SD-3 is much cleaner and easier to recognize than SD-1. The reason for this can be found on the fact that SD-3 was collected among Census Bureau employees, while SD-1 was collected among high-school students. Drawing sensible conclusions from learning experiments requires that the result be independent of the choice of training set and test among the complete set of samples. Therefore it was necessary to build a new database by mixing NIST's datasets. \\n\\nThe MNIST training set is composed of 30,000 patterns from SD-3 and 30,000 patterns from SD-1. Our test set was composed of 5,000 patterns from SD-3 and 5,000 patterns from SD-1. The 60,000 pattern training set contained examples from approximately 250 writers. We made sure that the sets of writers of the training set and test set were disjoint. SD-1 contains 58,527 digit images written by 500 different writers. In contrast to SD-3, where blocks of data from each writer appeared in sequence, the data in SD-1 is scrambled. Writer identities for SD-1 is available and we used this information to unscramble the writers. We then split SD-1 in two: characters written by the first 250 writers went into our new training set. The remaining 250 writers were placed in our test set. Thus we had two sets with nearly 30,000 examples each. The new training set was completed with enough examples from SD-3, starting at pattern # 0, to make a full set of 60,000 training patterns. Similarly, the new test set was completed with SD-3 examples starting at pattern # 35,000 to make a full set with 60,000 test patterns. Only a subset of 10,000 test images (5,000 from SD-1 and 5,000 from SD-3) is available on this site. The full 60,000 sample training set is available.\\n\\nDownloaded from openml.org.\""
] ]
}, },
"execution_count": 7, "execution_count": 7,
@ -159,7 +159,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "04b6042e", "id": "5a70a746",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Prepare the MNIST dataset" "### Prepare the MNIST dataset"
@ -167,7 +167,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "0c6a4dcf", "id": "a9b7a120",
"metadata": {}, "metadata": {},
"source": [ "source": [
"$f(X) = y$\n", "$f(X) = y$\n",
@ -181,7 +181,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 8,
"id": "53f4723d", "id": "4e02cf2a",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -191,7 +191,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 9,
"id": "0c9ba2bb", "id": "001d736f",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -212,7 +212,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 10,
"id": "cb7b421b", "id": "b344be1d",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -233,7 +233,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 11,
"id": "82e2f7ba", "id": "cef23e9f",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -253,7 +253,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "65185c61", "id": "fe3b1259",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Plot data" "### Plot data"
@ -262,7 +262,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 12,
"id": "8370fd0e", "id": "953d9415",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -274,7 +274,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 13,
"id": "448321e3", "id": "b68f6cee",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -297,7 +297,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 14,
"id": "7f66718a", "id": "8779b1a2",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -317,7 +317,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 15,
"id": "ba95d655", "id": "dcc605cf",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -343,7 +343,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 16, "execution_count": 16,
"id": "7d665e31", "id": "6d41d752",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -364,7 +364,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 17, "execution_count": 17,
"id": "e12786e6", "id": "230cfd35",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -375,7 +375,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 18, "execution_count": 18,
"id": "b114c10d", "id": "25a3a2e7",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -389,7 +389,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 19, "execution_count": 19,
"id": "5f4d40c4", "id": "f1552762",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -413,7 +413,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 20, "execution_count": 20,
"id": "5a18f4f9", "id": "74b3a063",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -429,7 +429,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 21, "execution_count": 21,
"id": "fdd1cc4d", "id": "949b3914",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -454,7 +454,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "88c50f5b", "id": "ec8a9d34",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Prepare data for machine learning" "### Prepare data for machine learning"
@ -463,7 +463,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 22, "execution_count": 22,
"id": "0ebec6fa", "id": "febbd286",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -485,7 +485,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 23, "execution_count": 23,
"id": "6e0cb6ee", "id": "fff839b6",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -495,7 +495,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "6edf6046", "id": "2bdbeb4e",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Train classifier" "### Train classifier"
@ -504,7 +504,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 24, "execution_count": 24,
"id": "549db50c", "id": "4c32ae9f",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -515,7 +515,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 25, "execution_count": 25,
"id": "92297e2c", "id": "fe06ae55",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -540,7 +540,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 26, "execution_count": 26,
"id": "c08adc09", "id": "e6209258",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -565,7 +565,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 27, "execution_count": 27,
"id": "e9622d3b", "id": "62773b1b",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -584,7 +584,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 28, "execution_count": 28,
"id": "afd7df0b", "id": "0ce21474",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -603,7 +603,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": 29,
"id": "3591ea2c", "id": "78a8e8a7",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -626,7 +626,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 30, "execution_count": 30,
"id": "21c1ce01", "id": "45d93a99",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -647,7 +647,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "62982f84", "id": "fc739051",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Evaluation" "### Evaluation"
@ -656,7 +656,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 31, "execution_count": 31,
"id": "e257d6fa", "id": "990a5b7c",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -677,7 +677,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 32, "execution_count": 32,
"id": "160c355e", "id": "f125a37d",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -697,7 +697,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "5ca9eb13", "id": "bdcb6e6e",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Accuracy is strongly influenced by the distribution of the classes in the test data." "Accuracy is strongly influenced by the distribution of the classes in the test data."
@ -705,7 +705,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "7e1d98d0", "id": "be858cd5",
"metadata": {}, "metadata": {},
"source": [ "source": [
"#### Cross Validation\n", "#### Cross Validation\n",
@ -715,7 +715,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 33, "execution_count": 33,
"id": "bf3fe38d", "id": "7adb1ea7",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -736,7 +736,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 34, "execution_count": 34,
"id": "ef410d43", "id": "11d22c5e",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -759,7 +759,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "b5d581d2", "id": "b54e83a5",
"metadata": {}, "metadata": {},
"source": [ "source": [
"#### Precision" "#### Precision"
@ -768,7 +768,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 35, "execution_count": 35,
"id": "6c6c13b2", "id": "ef7a9e7e",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -790,7 +790,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "ad953977", "id": "da723740",
"metadata": {}, "metadata": {},
"source": [ "source": [
"#### Recall" "#### Recall"
@ -799,7 +799,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 36, "execution_count": 36,
"id": "6f9f5706", "id": "cb77bf58",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -821,7 +821,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "c5d15006", "id": "28867d1b",
"metadata": {}, "metadata": {},
"source": [ "source": [
"#### F1 Score" "#### F1 Score"
@ -830,7 +830,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 37, "execution_count": 37,
"id": "4c6a0676", "id": "0674e0de",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -852,7 +852,7 @@
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "29f5d5e2", "id": "da59da11",
"metadata": {}, "metadata": {},
"source": [ "source": [
"#### Confusion Matrix" "#### Confusion Matrix"
@ -861,7 +861,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 38, "execution_count": 38,
"id": "db5092b9", "id": "adbdeece",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -891,7 +891,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 39, "execution_count": 39,
"id": "9b9b93f7", "id": "fb50c5a4",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -929,7 +929,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 40, "execution_count": 40,
"id": "d0c229f1", "id": "2f0d536a",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -940,7 +940,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 41, "execution_count": 41,
"id": "634396e4", "id": "dddf5fe8",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -969,33 +969,44 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 42,
"id": "1ef938c5", "id": "44537aae",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
"source": []
},
{ {
"cell_type": "code", "name": "stdout",
"execution_count": null, "output_type": "stream",
"id": "f55ba3ed", "text": [
"metadata": {}, " precision recall f1-score support\n",
"outputs": [], "\n",
"source": [] " 0 1.00 0.92 0.96 13\n",
}, " 1 0.39 1.00 0.56 14\n",
{ " 2 1.00 0.33 0.50 6\n",
"cell_type": "markdown", " 3 1.00 0.64 0.78 11\n",
"id": "9238e672", " 4 0.86 0.55 0.67 11\n",
"metadata": {}, " 5 1.00 0.20 0.33 5\n",
" 6 0.82 0.82 0.82 11\n",
" 7 0.89 0.80 0.84 10\n",
" 8 1.00 0.62 0.77 8\n",
" 9 0.80 0.73 0.76 11\n",
"\n",
" accuracy 0.72 100\n",
" macro avg 0.88 0.66 0.70 100\n",
"weighted avg 0.85 0.72 0.73 100\n",
"\n"
]
}
],
"source": [ "source": [
"## Train kNN Classifer\n", "from sklearn.metrics import classification_report\n",
"__TODO__" "\n",
"print(classification_report(y_train, y_train_pred))"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "a8fbef47", "id": "57d96f56",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": []

48
0-pilot-project/Process.md Normal file
View File

@ -0,0 +1,48 @@
# Using kNN
## Finding optimal k-Value
Through testing on the original dataset (split 80:20) we found, that the optimal k-value is 3.
Running the kNN on the dataset without any preprocessing results in:
> weighted avg 0.97 0.97 0.97 56000
# Dataset optimization
## Standardization
### Standard
It seemed like StandardScalar on the MNIST dataset wouldn't change the outcome, so we ommitted standardization.
Reason for that is probably, that the MNIST Dataset was already optimized for processing.
### MinMax
Needs to be updated.
## Feature selection
To be tested
## Feature reduction
### PCA
Testing with PCA and plotting component vs. variance we found that a 98.64% variance could be archived with only 300 components [^1].
Testing further the a variance of 99.99999999999992% was archived at 709 components, which was also the same for 784 components (the original amount of components), which means, that no/minimal variance/information is lost when using 709 components in comparison to 784 components[^2].
For now we will simply go with n_components of 709.
### LDA
To be tested
# TODO
- [ ] Look up point of Covariance Matrix and how it works
- https://www.youtube.com/watch?v=152tSYtiQbw
- Probably part of PCA
- [ ] Reference for standardization not changing results of classifier
- [ ] Reference for MNIST already been standardized
- [ ] Test standardization method other than `StandardScalar`
- [ ] Test feature reduction method other than `PCA` (i.e. LDA(Linear Discriminant Analysis))
- https://en.wikipedia.org/wiki/Dimensionality_reduction
- https://towardsdatascience.com/is-lda-a-dimensionality-reduction-technique-or-a-classifier-algorithm-eeed4de9953a
- https://medium.com/machine-learning-researcher/dimensionality-reduction-pca-and-lda-6be91734f567
- https://towardsdatascience.com/dimensionality-reduction-does-pca-really-improve-classification-outcome-6e9ba21f0a32
- [ ] Add feature selection process
- https://scikit-learn.org/stable/modules/feature_selection.html
[^1]: https://medium.com/@miat1015/mnist-using-pca-for-dimension-reduction-and-also-t-sne-and-also-3d-visualization-55084e0320b5
[^2]: Could be due to rounding in python

9
0-pilot-project/notes.md
View File

@ -5,9 +5,10 @@
- Wo ist der Unterschied zwischen sklearn.model_selection.train_test_split und manuellem pick mit list arguments - Wo ist der Unterschied zwischen sklearn.model_selection.train_test_split und manuellem pick mit list arguments
## Todos ## Todos
- Unterschied zwischen accuracy und precision score (steht vlt in Folien) - ~~Unterschied zwischen accuracy und precision score (steht vlt in Folien)~~
- Classifier grafisch anzeigen lassen (https://scikit-learn.org/stable/auto_examples/neighbors/plot_classification.html#sphx-glr-auto-examples-neighbors-plot-classification-py) - Classifier grafisch anzeigen lassen (https://scikit-learn.org/stable/auto_examples/neighbors/plot_classification.html#sphx-glr-auto-examples-neighbors-plot-classification-py)
- Schauen wie wir mit weniger Features arbeiten können - ~~Schauen wie wir mit weniger Features arbeiten können~~ PCA/LDA
- Zeitmessung von einzelnen classifier test loops und mitprinten bei jedem Durchlauf - ~~Zeitmessung von einzelnen classifier test loops und mitprinten bei jedem Durchlauf~~ %%time
- Unterschiedliche Validierungsmethoden testen - Unterschiedliche Validierungsmethoden testen
- Code anpassen, dass er nicht stirbt - ~~Code anpassen, dass er nicht stirbt~~
- Accuracy anhand Testdatensatz

146
0-pilot-project/results.md Normal file
View File

@ -0,0 +1,146 @@
## StandardScalar & LDA=9
```
precision recall f1-score support
0 0.94 0.97 0.96 5499
1 0.93 0.98 0.95 6287
2 0.90 0.91 0.90 5595
3 0.89 0.87 0.88 5679
4 0.91 0.93 0.92 5450
5 0.88 0.86 0.87 5068
6 0.95 0.95 0.95 5542
7 0.95 0.93 0.94 5846
8 0.90 0.84 0.87 5504
9 0.90 0.89 0.89 5530
accuracy 0.91 56000
macro avg 0.91 0.91 0.91 56000
weighted avg 0.91 0.91 0.91 56000
```
## StandardScalar & PCA=70
```
precision recall f1-score support
0 0.97 0.99 0.98 5499
1 0.97 0.99 0.98 6287
2 0.96 0.96 0.96 5595
3 0.94 0.94 0.94 5679
4 0.95 0.95 0.95 5450
5 0.95 0.94 0.94 5068
6 0.97 0.98 0.97 5542
7 0.96 0.95 0.95 5846
8 0.95 0.93 0.94 5504
9 0.93 0.93 0.93 5530
accuracy 0.96 56000
macro avg 0.95 0.95 0.95 56000
weighted avg 0.96 0.96 0.96 56000
```
## StandardScalar & PCA=400
```
precision recall f1-score support
0 0.95 0.99 0.97 5499
1 0.95 0.99 0.97 6287
2 0.95 0.94 0.94 5595
3 0.93 0.94 0.93 5679
4 0.94 0.93 0.94 5450
5 0.93 0.92 0.93 5068
6 0.96 0.97 0.97 5542
7 0.94 0.94 0.94 5846
8 0.97 0.89 0.93 5504
9 0.91 0.92 0.92 5530
accuracy 0.94 56000
macro avg 0.94 0.94 0.94 56000
weighted avg 0.94 0.94 0.94 56000
```
## PCA=300
```
precision recall f1-score support
0 0.96 0.99 0.97 5499
1 0.96 0.99 0.97 6287
2 0.95 0.94 0.95 5595
3 0.93 0.94 0.94 5679
4 0.95 0.94 0.94 5450
5 0.93 0.92 0.93 5068
6 0.96 0.97 0.97 5542
7 0.95 0.94 0.94 5846
8 0.96 0.90 0.93 5504
9 0.92 0.92 0.92 5530
accuracy 0.95 56000
macro avg 0.95 0.95 0.95 56000
weighted avg 0.95 0.95 0.95 56000
```
## Nothing
```
precision recall f1-score support
0 0.98 0.99 0.99 5499
1 0.95 0.99 0.97 6287
2 0.98 0.96 0.97 5595
3 0.96 0.97 0.96 5679
4 0.98 0.96 0.97 5450
5 0.96 0.96 0.96 5068
6 0.98 0.99 0.98 5542
7 0.96 0.97 0.97 5846
8 0.99 0.93 0.96 5504
9 0.95 0.96 0.96 5530
accuracy 0.97 56000
macro avg 0.97 0.97 0.97 56000
weighted avg 0.97 0.97 0.97 56000
```
## PCA=400 & StandardScalar (same for non StandardScalar)
```
precision recall f1-score support
0 0.85 0.95 0.90 5499
1 0.68 0.99 0.80 6287
2 0.88 0.75 0.81 5595
3 0.82 0.84 0.83 5679
4 0.90 0.79 0.84 5450
5 0.88 0.76 0.82 5068
6 0.91 0.93 0.92 5542
7 0.87 0.87 0.87 5846
8 0.96 0.66 0.78 5504
9 0.83 0.85 0.84 5530
accuracy 0.84 56000
macro avg 0.86 0.84 0.84 56000
weighted avg 0.86 0.84 0.84 56000
```
## PCA=708 & StandardScalar (same for non StandardScalar)
```
precision recall f1-score support
0 0.95 0.99 0.97 5499
1 0.95 0.99 0.97 6287
2 0.95 0.93 0.94 5595
3 0.92 0.94 0.93 5679
4 0.94 0.93 0.94 5450
5 0.93 0.92 0.93 5068
6 0.96 0.97 0.97 5542
7 0.94 0.93 0.94 5846
8 0.97 0.89 0.93 5504
9 0.91 0.92 0.91 5530
accuracy 0.94 56000
macro avg 0.94 0.94 0.94 56000
weighted avg 0.94 0.94 0.94 56000
```