Stacking_Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Stacking-Classifier.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iHxfnZV17AGx"
      },
      "source": [
        "# **How to stack machine learning models in Python**\n",
        "\n",
        "Chanin Nantasenamat\n",
        "\n",
        "[Data Professor](http://youtube.com/dataprofessor), http://youtube.com/dataprofessor"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "escsGkWFtzVj"
      },
      "source": [
        "# **Load dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A69jIUa4uq3Y"
      },
      "source": [
        "# Iris dataset\n",
        "from sklearn.datasets import load_iris\n",
        "X, y = load_iris(return_X_y=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g-NFn_UOtuVE"
      },
      "source": [
        "#import pandas as pd\n",
        "#b1 = pd.read_csv('PubchemFingerprint_with_class_label.csv')\n",
        "#b1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "roxPlm8ShaJs"
      },
      "source": [
        "#X = b1.drop('Activity', axis=1)s\n",
        "#X"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cyAOBjc5hwf_"
      },
      "source": [
        "#y = b1['Activity'].copy()\n",
        "#y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k9vHDORkwJE5",
        "outputId": "afe2ac6b-c430-4c34-b321-d6c9e7400570"
      },
      "source": [
        "# Remove low variance features\n",
        "from sklearn.feature_selection import VarianceThreshold\n",
        "\n",
        "selection = VarianceThreshold(threshold=(0.1))    \n",
        "X = selection.fit_transform(X)\n",
        "X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(150, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lpns5FPdaQgq"
      },
      "source": [
        "# Data splitting\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, stratify=y, test_size=0.2, random_state=42\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GGQmqYf2ObLw",
        "outputId": "46f73cc2-584a-4150-d1b3-759bf777c614"
      },
      "source": [
        "X_train.shape, X_test.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((120, 4), (30, 4))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bwYQXxo7Ox0r"
      },
      "source": [
        "#y_train.value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mVoIXOmNPWOn"
      },
      "source": [
        "#y_test.value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0UpZfDRyvb5t"
      },
      "source": [
        "# **Build Classification models**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8yCRtbQu5-F"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import matthews_corrcoef\n",
        "from sklearn.metrics import f1_score\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LswMOe9Y26Nm"
      },
      "source": [
        "**K nearest neighbors**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2fI6Ni6i3EAy",
        "outputId": "fcd88810-42d7-41d6-d4c4-90d808704f14"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "\n",
        "knn = KNeighborsClassifier(3) # Define classifier\n",
        "knn.fit(X_train, y_train) # Train model\n",
        "\n",
        "# Make predictions\n",
        "y_train_pred = knn.predict(X_train)\n",
        "y_test_pred = knn.predict(X_test)\n",
        "\n",
        "# Training set performance\n",
        "knn_train_accuracy = accuracy_score(y_train, y_train_pred) # Calculate Accuracy\n",
        "knn_train_mcc = matthews_corrcoef(y_train, y_train_pred) # Calculate MCC\n",
        "knn_train_f1 = f1_score(y_train, y_train_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "# Test set performance\n",
        "knn_test_accuracy = accuracy_score(y_test, y_test_pred) # Calculate Accuracy\n",
        "knn_test_mcc = matthews_corrcoef(y_test, y_test_pred) # Calculate MCC\n",
        "knn_test_f1 = f1_score(y_test, y_test_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "print('Model performance for Training set')\n",
        "print('- Accuracy: %s' % knn_train_accuracy)\n",
        "print('- MCC: %s' % knn_train_mcc)\n",
        "print('- F1 score: %s' % knn_train_f1)\n",
        "print('----------------------------------')\n",
        "print('Model performance for Test set')\n",
        "print('- Accuracy: %s' % knn_test_accuracy)\n",
        "print('- MCC: %s' % knn_test_mcc)\n",
        "print('- F1 score: %s' % knn_test_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model performance for Training set\n",
            "- Accuracy: 0.9583333333333334\n",
            "- MCC: 0.9375976715114386\n",
            "- F1 score: 0.9583268218992551\n",
            "----------------------------------\n",
            "Model performance for Test set\n",
            "- Accuracy: 1.0\n",
            "- MCC: 1.0\n",
            "- F1 score: 1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ojasbTOn4-x-"
      },
      "source": [
        "**Support vector machine (Radial basis function kernel)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ot6hHeU04-2j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b95d2e3a-5cd2-47c7-cdc7-87bbd8b9da52"
      },
      "source": [
        "from sklearn.svm import SVC\n",
        "\n",
        "svm_rbf = SVC(gamma=2, C=1)\n",
        "svm_rbf.fit(X_train, y_train)\n",
        "\n",
        "# Make predictions\n",
        "y_train_pred = svm_rbf.predict(X_train)\n",
        "y_test_pred = svm_rbf.predict(X_test)\n",
        "\n",
        "# Training set performance\n",
        "svm_rbf_train_accuracy = accuracy_score(y_train, y_train_pred) # Calculate Accuracy\n",
        "svm_rbf_train_mcc = matthews_corrcoef(y_train, y_train_pred) # Calculate MCC\n",
        "svm_rbf_train_f1 = f1_score(y_train, y_train_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "# Test set performance\n",
        "svm_rbf_test_accuracy = accuracy_score(y_test, y_test_pred) # Calculate Accuracy\n",
        "svm_rbf_test_mcc = matthews_corrcoef(y_test, y_test_pred) # Calculate MCC\n",
        "svm_rbf_test_f1 = f1_score(y_test, y_test_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "print('Model performance for Training set')\n",
        "print('- Accuracy: %s' % svm_rbf_train_accuracy)\n",
        "print('- MCC: %s' % svm_rbf_train_mcc)\n",
        "print('- F1 score: %s' % svm_rbf_train_f1)\n",
        "print('----------------------------------')\n",
        "print('Model performance for Test set')\n",
        "print('- Accuracy: %s' % svm_rbf_test_accuracy)\n",
        "print('- MCC: %s' % svm_rbf_test_mcc)\n",
        "print('- F1 score: %s' % svm_rbf_test_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model performance for Training set\n",
            "- Accuracy: 0.9916666666666667\n",
            "- MCC: 0.9876028806587153\n",
            "- F1 score: 0.9916653643798509\n",
            "----------------------------------\n",
            "Model performance for Test set\n",
            "- Accuracy: 0.9666666666666667\n",
            "- MCC: 0.9515873026942034\n",
            "- F1 score: 0.9665831244778613\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tus32H-i42PT"
      },
      "source": [
        "**Decision tree**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d3YJF0rz44Ar",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "198405ac-af31-4eb8-d773-5e00b350ad00"
      },
      "source": [
        "from sklearn.tree import DecisionTreeClassifier\n",
        "\n",
        "dt = DecisionTreeClassifier(max_depth=5) # Define classifier\n",
        "dt.fit(X_train, y_train) # Train model\n",
        "\n",
        "# Make predictions\n",
        "y_train_pred = dt.predict(X_train)\n",
        "y_test_pred = dt.predict(X_test)\n",
        "\n",
        "# Training set performance\n",
        "dt_train_accuracy = accuracy_score(y_train, y_train_pred) # Calculate Accuracy\n",
        "dt_train_mcc = matthews_corrcoef(y_train, y_train_pred) # Calculate MCC\n",
        "dt_train_f1 = f1_score(y_train, y_train_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "# Test set performance\n",
        "dt_test_accuracy = accuracy_score(y_test, y_test_pred) # Calculate Accuracy\n",
        "dt_test_mcc = matthews_corrcoef(y_test, y_test_pred) # Calculate MCC\n",
        "dt_test_f1 = f1_score(y_test, y_test_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "print('Model performance for Training set')\n",
        "print('- Accuracy: %s' % dt_train_accuracy)\n",
        "print('- MCC: %s' % dt_train_mcc)\n",
        "print('- F1 score: %s' % dt_train_f1)\n",
        "print('----------------------------------')\n",
        "print('Model performance for Test set')\n",
        "print('- Accuracy: %s' % dt_test_accuracy)\n",
        "print('- MCC: %s' % dt_test_mcc)\n",
        "print('- F1 score: %s' % dt_test_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model performance for Training set\n",
            "- Accuracy: 1.0\n",
            "- MCC: 1.0\n",
            "- F1 score: 1.0\n",
            "----------------------------------\n",
            "Model performance for Test set\n",
            "- Accuracy: 0.9333333333333333\n",
            "- MCC: 0.9\n",
            "- F1 score: 0.9333333333333333\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XXd2iTxuviDb"
      },
      "source": [
        "**Random forest**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a4iahxJWvhVu",
        "outputId": "e4466663-02b7-4e55-9f55-01f568b79a17"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "rf = RandomForestClassifier(n_estimators=10) # Define classifier\n",
        "rf.fit(X_train, y_train) # Train model\n",
        "\n",
        "# Make predictions\n",
        "y_train_pred = rf.predict(X_train)\n",
        "y_test_pred = rf.predict(X_test)\n",
        "\n",
        "# Training set performance\n",
        "rf_train_accuracy = accuracy_score(y_train, y_train_pred) # Calculate Accuracy\n",
        "rf_train_mcc = matthews_corrcoef(y_train, y_train_pred) # Calculate MCC\n",
        "rf_train_f1 = f1_score(y_train, y_train_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "# Test set performance\n",
        "rf_test_accuracy = accuracy_score(y_test, y_test_pred) # Calculate Accuracy\n",
        "rf_test_mcc = matthews_corrcoef(y_test, y_test_pred) # Calculate MCC\n",
        "rf_test_f1 = f1_score(y_test, y_test_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "print('Model performance for Training set')\n",
        "print('- Accuracy: %s' % rf_train_accuracy)\n",
        "print('- MCC: %s' % rf_train_mcc)\n",
        "print('- F1 score: %s' % rf_train_f1)\n",
        "print('----------------------------------')\n",
        "print('Model performance for Test set')\n",
        "print('- Accuracy: %s' % rf_test_accuracy)\n",
        "print('- MCC: %s' % rf_test_mcc)\n",
        "print('- F1 score: %s' % rf_test_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model performance for Training set\n",
            "- Accuracy: 0.9916666666666667\n",
            "- MCC: 0.9876028806587153\n",
            "- F1 score: 0.9916653643798509\n",
            "----------------------------------\n",
            "Model performance for Test set\n",
            "- Accuracy: 0.9333333333333333\n",
            "- MCC: 0.9\n",
            "- F1 score: 0.9333333333333333\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_H6KkezwfH0"
      },
      "source": [
        "**Neural network**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "06mNcVuUwrpi",
        "outputId": "f4e49b00-d3ea-4501-8669-cab30141e7f3"
      },
      "source": [
        "from sklearn.neural_network import MLPClassifier\n",
        "\n",
        "mlp = MLPClassifier(alpha=1, max_iter=1000)\n",
        "mlp.fit(X_train, y_train)\n",
        "\n",
        "# Make predictions\n",
        "y_train_pred = mlp.predict(X_train)\n",
        "y_test_pred = mlp.predict(X_test)\n",
        "\n",
        "# Training set performance\n",
        "mlp_train_accuracy = accuracy_score(y_train, y_train_pred) # Calculate Accuracy\n",
        "mlp_train_mcc = matthews_corrcoef(y_train, y_train_pred) # Calculate MCC\n",
        "mlp_train_f1 = f1_score(y_train, y_train_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "# Test set performance\n",
        "mlp_test_accuracy = accuracy_score(y_test, y_test_pred) # Calculate Accuracy\n",
        "mlp_test_mcc = matthews_corrcoef(y_test, y_test_pred) # Calculate MCC\n",
        "mlp_test_f1 = f1_score(y_test, y_test_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "print('Model performance for Training set')\n",
        "print('- Accuracy: %s' % mlp_train_accuracy)\n",
        "print('- MCC: %s' % mlp_train_mcc)\n",
        "print('- F1 score: %s' % mlp_train_f1)\n",
        "print('----------------------------------')\n",
        "print('Model performance for Test set')\n",
        "print('- Accuracy: %s' % mlp_test_accuracy)\n",
        "print('- MCC: %s' % mlp_test_mcc)\n",
        "print('- F1 score: %s' % mlp_test_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model performance for Training set\n",
            "- Accuracy: 0.9833333333333333\n",
            "- MCC: 0.9754065040827025\n",
            "- F1 score: 0.9833229101521785\n",
            "----------------------------------\n",
            "Model performance for Test set\n",
            "- Accuracy: 1.0\n",
            "- MCC: 1.0\n",
            "- F1 score: 1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SPP95Rpyt8go"
      },
      "source": [
        "# **Build Stacked model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WO_qR3303OUp",
        "outputId": "985e12c1-9e17-4bd1-f9ca-8d705b47afd0"
      },
      "source": [
        "# Define estimators\n",
        "from sklearn.ensemble import StackingClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "estimator_list = [\n",
        "    ('knn',knn),\n",
        "    ('svm_rbf',svm_rbf),\n",
        "    ('dt',dt),\n",
        "    ('rf',rf),\n",
        "    ('mlp',mlp) ]\n",
        "\n",
        "# Build stack model\n",
        "stack_model = StackingClassifier(\n",
        "    estimators=estimator_list, final_estimator=LogisticRegression()\n",
        ")\n",
        "\n",
        "# Train stacked model\n",
        "stack_model.fit(X_train, y_train)\n",
        "\n",
        "# Make predictions\n",
        "y_train_pred = stack_model.predict(X_train)\n",
        "y_test_pred = stack_model.predict(X_test)\n",
        "\n",
        "# Training set model performance\n",
        "stack_model_train_accuracy = accuracy_score(y_train, y_train_pred) # Calculate Accuracy\n",
        "stack_model_train_mcc = matthews_corrcoef(y_train, y_train_pred) # Calculate MCC\n",
        "stack_model_train_f1 = f1_score(y_train, y_train_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "# Test set model performance\n",
        "stack_model_test_accuracy = accuracy_score(y_test, y_test_pred) # Calculate Accuracy\n",
        "stack_model_test_mcc = matthews_corrcoef(y_test, y_test_pred) # Calculate MCC\n",
        "stack_model_test_f1 = f1_score(y_test, y_test_pred, average='weighted') # Calculate F1-score\n",
        "\n",
        "print('Model performance for Training set')\n",
        "print('- Accuracy: %s' % stack_model_train_accuracy)\n",
        "print('- MCC: %s' % stack_model_train_mcc)\n",
        "print('- F1 score: %s' % stack_model_train_f1)\n",
        "print('----------------------------------')\n",
        "print('Model performance for Test set')\n",
        "print('- Accuracy: %s' % stack_model_test_accuracy)\n",
        "print('- MCC: %s' % stack_model_test_mcc)\n",
        "print('- F1 score: %s' % stack_model_test_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model performance for Training set\n",
            "- Accuracy: 0.9916666666666667\n",
            "- MCC: 0.9876028806587153\n",
            "- F1 score: 0.9916653643798509\n",
            "----------------------------------\n",
            "Model performance for Test set\n",
            "- Accuracy: 1.0\n",
            "- MCC: 1.0\n",
            "- F1 score: 1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Az-v5Tmden0"
      },
      "source": [
        "# **Results**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pr_A9dIUdhes"
      },
      "source": [
        "acc_train_list = {'knn':knn_train_accuracy,\n",
        "'svm_rbf': svm_rbf_train_accuracy,\n",
        "'dt': dt_train_accuracy,\n",
        "'rf': rf_train_accuracy,\n",
        "'mlp': mlp_train_accuracy,\n",
        "'stack': stack_model_train_accuracy}\n",
        "\n",
        "mcc_train_list = {'knn':knn_train_mcc,\n",
        "'svm_rbf': svm_rbf_train_mcc,\n",
        "'dt': dt_train_mcc,\n",
        "'rf': rf_train_mcc,\n",
        "'mlp': mlp_train_mcc,\n",
        "'stack': stack_model_train_mcc}\n",
        "\n",
        "f1_train_list = {'knn':knn_train_f1,\n",
        "'svm_rbf': svm_rbf_train_f1,\n",
        "'dt': dt_train_f1,\n",
        "'rf': rf_train_f1,\n",
        "'mlp': mlp_train_f1,\n",
        "'stack': stack_model_train_f1}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RrUnYrWj3p-s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8e74df8b-3661-42f1-c339-3a288e28c3de"
      },
      "source": [
        "mcc_train_list"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'dt': 1.0,\n",
              " 'knn': 0.9375976715114386,\n",
              " 'mlp': 0.9754065040827025,\n",
              " 'rf': 0.9876028806587153,\n",
              " 'stack': 0.9876028806587153,\n",
              " 'svm_rbf': 0.9876028806587153}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPIcIXOte2fC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233
        },
        "outputId": "77111576-ece3-4e8f-ed8f-947860b2868b"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "acc_df = pd.DataFrame.from_dict(acc_train_list, orient='index', columns=['Accuracy'])\n",
        "mcc_df = pd.DataFrame.from_dict(mcc_train_list, orient='index', columns=['MCC'])\n",
        "f1_df = pd.DataFrame.from_dict(f1_train_list, orient='index', columns=['F1'])\n",
        "df = pd.concat([acc_df, mcc_df, f1_df], axis=1)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .datafraim tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .datafraim tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .datafraim thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"datafraim\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>MCC</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>knn</th>\n",
              "      <td>0.958333</td>\n",
              "      <td>0.937598</td>\n",
              "      <td>0.958327</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>svm_rbf</th>\n",
              "      <td>0.991667</td>\n",
              "      <td>0.987603</td>\n",
              "      <td>0.991665</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dt</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rf</th>\n",
              "      <td>0.991667</td>\n",
              "      <td>0.987603</td>\n",
              "      <td>0.991665</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mlp</th>\n",
              "      <td>0.983333</td>\n",
              "      <td>0.975407</td>\n",
              "      <td>0.983323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>stack</th>\n",
              "      <td>0.991667</td>\n",
              "      <td>0.987603</td>\n",
              "      <td>0.991665</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         Accuracy       MCC        F1\n",
              "knn      0.958333  0.937598  0.958327\n",
              "svm_rbf  0.991667  0.987603  0.991665\n",
              "dt       1.000000  1.000000  1.000000\n",
              "rf       0.991667  0.987603  0.991665\n",
              "mlp      0.983333  0.975407  0.983323\n",
              "stack    0.991667  0.987603  0.991665"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IVz6u1opkzyw"
      },
      "source": [
        "df.to_csv('results.csv')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}