{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "コード4.1■Step 1のコード"
      ],
      "metadata": {
        "id": "SBhHZ5Cut6em"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 983
        },
        "id": "ZEJZrO88qbv7",
        "outputId": "13b1e949-9bfe-4a12-e4f8-a150d53f67b3"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 18940 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 19804 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.datasets import make_moons\n",
        "import sklearn.model_selection\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import classification_report\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "# データをプロットする関数を定義\n",
        "def pt_plot(x1_x2, label):\n",
        "    # 図のサイズを設定\n",
        "    plt.figure(figsize=(12, 8))\n",
        "    # x軸の名前\n",
        "    plt.xlabel('x1', size=\"large\")\n",
        "    # y軸の名前\n",
        "    plt.ylabel('x2', size=\"large\")\n",
        "    # データの個数\n",
        "    datalen = len(x1_x2)\n",
        "    # データごとに繰り返す\n",
        "    for cnt in range(datalen):\n",
        "        # もしラベルが0ならば，四角形でグレーのマーカーで点を打つ\n",
        "        if label[cnt] == 0:\n",
        "            plt.scatter(x1_x2[:,0][cnt],x1_x2[:,1][cnt], \\\n",
        "                        marker='s', s=50, c=\"grey\")\n",
        "        # もしラベルが1ならば，三角形で黒のマーカーで点を打つ\n",
        "        elif label[cnt] == 1:\n",
        "            plt.scatter(x1_x2[:,0][cnt],x1_x2[:,1][cnt], \\\n",
        "                        marker='^', s=50, c=\"black\")\n",
        "\n",
        "# 偽陽性率を算出する関数\n",
        "def false_positive_rate(y_true, y_pred):\n",
        "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).flatten()\n",
        "    return fp / (tn + fp)\n",
        "\n",
        "# Step 1. データ準備\n",
        "# Step 1①データの読み込み\n",
        "features_org, true_labels_org = make_moons(\n",
        "    n_samples=300,\n",
        "    noise=0.15,\n",
        "    random_state=0\n",
        ")\n",
        "# Step 1② トレーニングデータとテストデータに分割\n",
        "features, test_features, true_labels, test_true_labels = \\\n",
        "    train_test_split(features_org, true_labels_org, \\\n",
        "                     test_size=0.2, random_state=0)\n",
        "pt_plot(features, true_labels)\n",
        "# Step 1⑤ 数値データのスケーリング\n",
        "scaler = StandardScaler()\n",
        "scaled_features = scaler.fit_transform(features)\n",
        "pt_plot(scaled_features, true_labels)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "コード4.2■Step 2のコード"
      ],
      "metadata": {
        "id": "c2E8-TVEudN5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Step 2. トレーニング用データを用いた分類モデルの学習\n",
        "# Step 2① 初期モデルの作成\n",
        "# Step 2② ハイパーパラメータの設定\n",
        "# カーネルはガウスカーネル使用\n",
        "knl = 'rbf'\n",
        "# カーネルに多項式カーネルを採用（#を外すと有効）\n",
        "# knl = 'poly'\n",
        "# カーネルにシグモイドカーネルを採用（#を外すと有効）\n",
        "# knl = 'sigmoid'\n",
        "# γを5に設定\n",
        "gma = 5\n",
        "# 多項式カーネル採用時に次数を5に設定\n",
        "dgr = 2\n",
        "# コストパラメータC のリスト\n",
        "c_values = [0.01, 0.1, 1.0, 10, 100]\n",
        "# Step 2③ 交差検証用にトレーニング用データを分割\n",
        "k = 5\n",
        "skf = sklearn.model_selection.StratifiedKFold(n_splits=k,\n",
        "                                              random_state=0, shuffle=True)\n",
        "\n",
        "# Step 2④ 交差検証\n",
        "# 評価指標のリスト\n",
        "scoring = ['f1_macro', 'precision_macro', 'recall_macro']\n",
        "scores = None\n",
        "# C の値を変化させながら学習，テスト\n",
        "for c in c_values:\n",
        "    if knl=='rbf':\n",
        "        clf = SVC(kernel=knl, C=c, gamma=gma)\n",
        "    elif knl=='poly':\n",
        "        clf = SVC(kernel=knl, C=c, gamma=gma, degree=dgr)\n",
        "    else:\n",
        "        clf = SVC(kernel=knl, C=c)\n",
        "\n",
        "    score = sklearn.model_selection.cross_validate(\n",
        "            clf,\n",
        "            scaled_features,\n",
        "            true_labels,\n",
        "            scoring=scoring,\n",
        "            cv=skf,\n",
        "            )\n",
        "    x = pd.concat([pd.Series(np.full(k, c), name='C'), \\\n",
        "                  pd.DataFrame(score)], axis=1)\n",
        "    scores = pd.concat([scores, x])\n",
        "    # Step 2⑤ 終了条件判定\n",
        "print(scores)\n",
        "\n",
        "# Step 2⑥ 最良パラメータで全トレーニング用データ使いトレーニング\n",
        "best_c = scores.loc[scores.loc[:, 'test_f1_macro'] \\\n",
        "                  == max(scores.loc[:, 'test_f1_macro']), \\\n",
        "                  'C'].values[0]\n",
        "\n",
        "if knl == 'rbf':\n",
        "    clf = SVC(kernel=knl, C=best_c, gamma=gma)\n",
        "    print('kernel=', knl)\n",
        "    print('bestC=', best_c)\n",
        "    print('gamma=', gma)\n",
        "elif knl == 'poly':\n",
        "    clf = SVC(kernel=knl, C=best_c, degree=dgr)\n",
        "    print('kernel=', knl)\n",
        "    print('bestC=', best_c)\n",
        "    print('degree=', dgr)\n",
        "    print('gamma=', gma)\n",
        "elif knl == 'sigmoid':\n",
        "    clf = SVC(kernel=knl, C=best_c)\n",
        "    print('kernel=', knl)\n",
        "    print('bestC=', best_c)\n",
        "\n",
        "clf.fit(scaled_features, true_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hq64d8TzudGF",
        "outputId": "edd38794-e9a2-44a6-cc8b-a88714ae3252"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "        C  fit_time  score_time  test_f1_macro  test_precision_macro  \\\n",
            "0    0.01  0.002464    0.005577       0.351351              0.270833   \n",
            "1    0.01  0.002287    0.003788       0.351351              0.270833   \n",
            "2    0.01  0.002333    0.003847       0.342466              0.260417   \n",
            "3    0.01  0.002395    0.003736       0.342466              0.260417   \n",
            "4    0.01  0.002280    0.004164       0.342466              0.260417   \n",
            "0    0.10  0.002146    0.002703       0.979085              0.978261   \n",
            "1    0.10  0.002227    0.002999       1.000000              1.000000   \n",
            "2    0.10  0.002243    0.002695       0.979158              0.979167   \n",
            "3    0.10  0.002222    0.002694       1.000000              1.000000   \n",
            "4    0.10  0.002090    0.002663       0.958261              0.958261   \n",
            "0    1.00  0.001519    0.002398       0.958261              0.958333   \n",
            "1    1.00  0.001491    0.002350       1.000000              1.000000   \n",
            "2    1.00  0.001436    0.002300       0.979158              0.979167   \n",
            "3    1.00  0.001539    0.002387       1.000000              1.000000   \n",
            "4    1.00  0.001421    0.002311       0.958261              0.958261   \n",
            "0   10.00  0.001425    0.002352       0.979085              0.978261   \n",
            "1   10.00  0.001453    0.002299       1.000000              1.000000   \n",
            "2   10.00  0.001300    0.002258       0.958261              0.958261   \n",
            "3   10.00  0.001359    0.002273       1.000000              1.000000   \n",
            "4   10.00  0.001347    0.002269       0.958261              0.958261   \n",
            "0  100.00  0.001413    0.002338       0.958261              0.958333   \n",
            "1  100.00  0.001394    0.002266       0.978938              0.981481   \n",
            "2  100.00  0.001276    0.002271       0.958261              0.958261   \n",
            "3  100.00  0.001246    0.002248       1.000000              1.000000   \n",
            "4  100.00  0.001203    0.002247       0.958261              0.958261   \n",
            "\n",
            "   test_recall_macro  \n",
            "0           0.500000  \n",
            "1           0.500000  \n",
            "2           0.500000  \n",
            "3           0.500000  \n",
            "4           0.500000  \n",
            "0           0.980769  \n",
            "1           1.000000  \n",
            "2           0.980000  \n",
            "3           1.000000  \n",
            "4           0.958261  \n",
            "0           0.961538  \n",
            "1           1.000000  \n",
            "2           0.980000  \n",
            "3           1.000000  \n",
            "4           0.958261  \n",
            "0           0.980769  \n",
            "1           1.000000  \n",
            "2           0.958261  \n",
            "3           1.000000  \n",
            "4           0.958261  \n",
            "0           0.961538  \n",
            "1           0.977273  \n",
            "2           0.958261  \n",
            "3           1.000000  \n",
            "4           0.958261  \n",
            "kernel= rbf\n",
            "bestC= 0.1\n",
            "gamma= 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SVC(C=0.1, gamma=5)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "コード4.3■Step 3のコード"
      ],
      "metadata": {
        "id": "cYEkNBSIuc_Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Step 3. テスト用データを用いた分類モデルパフォーマンスの推定\n",
        "# テスト用データのスケーリング\n",
        "scaled_test_features = scaler.fit_transform(test_features)\n",
        "# Step 2で作ったモデルでテスト用データを分類\n",
        "pred_labels = clf.predict(scaled_test_features)\n",
        "# 評価指標の値を算出\n",
        "print(f'Accuracy : \\\n",
        "    {accuracy_score(y_true=test_true_labels, y_pred=pred_labels)}')\n",
        "print(f'Precision : \\\n",
        "    {precision_score(y_true=test_true_labels, y_pred=pred_labels)}')\n",
        "print(f'Recall : \\\n",
        "    {recall_score(y_true=test_true_labels, y_pred=pred_labels)}')\n",
        "print(f'False_positive_rate : \\\n",
        "    {false_positive_rate(y_true=test_true_labels, y_pred=pred_labels)}')\n",
        "\n",
        "# 混同行列を作成\n",
        "cm = confusion_matrix(test_true_labels, pred_labels)\n",
        "# 混同行列をヒートマップに変換\n",
        "sns.heatmap(cm, annot=True, cmap='Blues')\n",
        "# 混同行列のx 軸ｙ軸にラベルをつける\n",
        "plt.xlabel(\"Predicted\", fontsize=13, rotation=0)\n",
        "plt.ylabel(\"Actual\", fontsize=13)\n",
        "# 混同行列のヒートマップを表示\n",
        "plt.show()\n",
        "\n",
        "# Step 2で作ったモデルの分類境界を図示\n",
        "plt.figure(figsize=(12, 8))\n",
        "_x0 = np.linspace(-2.5, 2.5, 500)\n",
        "_x1 = np.linspace(-2.5, 2.5, 500)\n",
        "x0, x1 = np.meshgrid(_x0,_x1)\n",
        "X = np.hstack((x0.ravel().reshape(-1, 1),\n",
        "               x1.ravel().reshape(-1, 1)))\n",
        "y_decision = clf.predict(X).reshape(x0.shape)\n",
        "# 分類境界の図上にテスト用データをプロット\n",
        "pt_plot(scaled_test_features, test_true_labels)\n",
        "plt.contourf(x0, x1, y_decision,\n",
        "             levels=[y_decision.min(), 0.1,\n",
        "                     y_decision.max()], alpha=0.3)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 856
        },
        "id": "pv8vlvn7uc5I",
        "outputId": "6081ebb3-2659-4ad3-b6df-6b8a2f92bb3a"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy :     0.9666666666666667\n",
            "Precision :     1.0\n",
            "Recall :     0.9459459459459459\n",
            "False_positive_rate :     0.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 8260 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 13772 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "コード4.4■カーネル化SVMによるアヤメ分類モデル"
      ],
      "metadata": {
        "id": "o12q3kEWucxx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ライブラリの読み込み\n",
        "from sklearn import svm\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import classification_report\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "# スケーリングを標準化スケーラーに設定\n",
        "sc = StandardScaler()\n",
        "\n",
        "# seabornのスタイルをセット\n",
        "sns.set(style=\"ticks\")\n",
        "\n",
        "# データを読み込む\n",
        "df = sns.load_dataset('iris')\n",
        "df = df.query('species != \"setosa\"')\n",
        "\n",
        "# 欠損値を除去\n",
        "df = df.copy().dropna()\n",
        "\n",
        "# pair plotでデータを可視化\n",
        "sns.pairplot(df, hue='species', markers=[\"s\", \"^\"] )\n",
        "plt.show()\n",
        "\n",
        "# versicolorを1，virginicaを0としてspecies_num列新設し格納\n",
        "df.loc[df[\"species\"] == 'versicolor',[\"species_num\"]] = 1\n",
        "df.loc[df[\"species\"] == 'virginica',[\"species_num\"]] = 0\n",
        "\n",
        "# 分類に使う特徴量を選ぶ\n",
        "x1label = \"sepal_length\"\n",
        "x2label = \"petal_length\"\n",
        "y1label = \"species\"\n",
        "y2label = \"species_num\"\n",
        "\n",
        "X = df[[x1label, x2label]].values\n",
        "y = df[[y1label, y2label]].values\n",
        "\n",
        "# 学習用データとテスト用データに分割\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, \\\n",
        "    test_size=0.3, random_state=42, stratify=y)\n",
        "\n",
        "# 標準化\n",
        "X_train = sc.fit_transform(X_train)\n",
        "X_test = sc.transform(X_test)\n",
        "\n",
        "# 整数型に変換\n",
        "y_train_num = y_train[:,1].astype('int8')\n",
        "y_train = y_train[:,0]\n",
        "y_test_num = y_test[:,1].astype('int8')\n",
        "\n",
        "# 描画キャンパスを作成\n",
        "plt.figure(figsize=(8, 8))\n",
        "\n",
        "# カウンタを初期化\n",
        "cnt = 0\n",
        "cnt1 = 0\n",
        "cnt2 = 0\n",
        "datalen = len(y_train)\n",
        "for cnt in range(datalen):\n",
        "    if y_train[cnt] == 'versicolor':\n",
        "        if cnt1 == 0:\n",
        "            plt.scatter(X_train[:,0][cnt],X_train[:,1][cnt],marker='s',\\\n",
        "                        s=50, c=\"grey\", label='versicolor')\n",
        "            cnt1 = 1\n",
        "        else:\n",
        "            plt.scatter(X_train[:,0][cnt],X_train[:,1][cnt],marker='s',\\\n",
        "                        s=50, c=\"grey\")\n",
        "    elif y_train[cnt] == 'virginica':\n",
        "        if cnt2 == 0:\n",
        "            plt.scatter(X_train[:,0][cnt],X_train[:,1][cnt],marker='^',\\\n",
        "                        s=50, c=\"k\", label='virginica')\n",
        "            cnt2 = 1\n",
        "        else:\n",
        "            plt.scatter(X_train[:,0][cnt],X_train[:,1][cnt],marker='^',\\\n",
        "                        s=50, c=\"k\")\n",
        "\n",
        "# 罫線を表示\n",
        "plt.grid()\n",
        "# 点群が表すアヤメの種類を凡例表示\n",
        "plt.legend()\n",
        "# x 軸のデータの最小値・最大値を取得\n",
        "xlim = plt.xlim()\n",
        "# y 軸のデータの最小値・最大値を取得\n",
        "ylim = plt.ylim()\n",
        "# 座標の最小値・最大値間を30等分する\n",
        "xx = np.linspace(xlim[0], xlim[1], 30)\n",
        "yy = np.linspace(ylim[0], ylim[1], 30)\n",
        "XX, YY = np.meshgrid(xx, yy)\n",
        "xy = np.vstack([XX.ravel(), YY.ravel()]).T\n",
        "\n",
        "# 機械学習モデルを設定（コメントアウトを外して選択）\n",
        "clf = SVC(kernel='poly', degree=3, C=10)\n",
        "# clf = SVC(kernel='rbf', gamma=2, C=1000)\n",
        "# clf = SVC(kernel='linear', C=10)\n",
        "\n",
        "# 学習\n",
        "clf.fit(X_train,y_train_num)\n",
        "\n",
        "# 分類の決定関数\n",
        "Z = clf.decision_function(xy).reshape(XX.shape)\n",
        "plt.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1],\n",
        "            alpha=0.5, linestyles=['--', '-', '--'])\n",
        "\n",
        "# サポートベクトルを○で囲む\n",
        "plt.scatter(clf.support_vectors_[:, 0],\n",
        "            clf.support_vectors_[:, 1],\n",
        "            s=140, linewidth=1,\n",
        "            facecolors='none', edgecolors='k')\n",
        "cnt = 0\n",
        "datalen = len(y_test)\n",
        "for cnt in range(datalen):\n",
        "    if y_test[cnt][0] == 'versicolor':\n",
        "        plt.scatter(X_test[:,0][cnt],X_test[:,1][cnt], \\\n",
        "                    marker='+', s=50, color =\"grey\")\n",
        "    elif y_test[cnt][0] == 'virginica':\n",
        "        plt.scatter(X_test[:,0][cnt],X_test[:,1][cnt], \\\n",
        "                    marker='+', s=50, color =\"k\")\n",
        "\n",
        "# テスト用データで分類できるかどうか確認する\n",
        "y_test_pred = clf.predict(X_test)\n",
        "# 予測値\n",
        "print(y_test_pred)\n",
        "# 正解\n",
        "print(y_test_num)\n",
        "\n",
        "plt.xlabel(x1label, fontsize=15)\n",
        "plt.ylabel(x2label, fontsize=15)\n",
        "plt.figsize=(8,8)\n",
        "plt.show()\n",
        "\n",
        "# 混同行列を作成\n",
        "cm = confusion_matrix(y_test_num, y_test_pred)\n",
        "cm_data = confusion_matrix(y_test_num, y_test_pred)\n",
        "cm = pd.DataFrame(data=cm_data, \\\n",
        "                  index=[\"virginica\", \"versicolor\"], \\\n",
        "                  columns=[\"virginica\", 'versicolor'])\n",
        "# 混同行列をヒートマップに変換\n",
        "sns.heatmap(cm, annot=True, cmap='Blues')\n",
        "# 軸にラベル設定\n",
        "plt.yticks(rotation=0)\n",
        "plt.xlabel(\"Predicted\", fontsize=13, rotation=0)\n",
        "plt.ylabel(\"Actual\", fontsize=13)\n",
        "plt.show()\n",
        "\n",
        "# テスト用データの分類結果一覧\n",
        "print(classification_report(y_test_num, y_test_pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "y5fhuHPjucou",
        "outputId": "1d2e0feb-9245-4123-bbe1-14b15f81aeb9"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 813.85x720 with 20 Axes>"
            ],
            "image/png": "[encoded data, 147444 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 0 1 1 0 0 0 0 0 0 0 1 1 0 0 1 1 0 1 1 1 0 0 1 0 1 0 0 1]\n",
            "[1 1 0 1 1 0 0 0 0 1 0 0 1 1 0 0 1 1 0 1 1 1 0 0 1 1 0 0 0 1]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 47892 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 13272 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.93      0.90        15\n",
            "           1       0.93      0.87      0.90        15\n",
            "\n",
            "    accuracy                           0.90        30\n",
            "   macro avg       0.90      0.90      0.90        30\n",
            "weighted avg       0.90      0.90      0.90        30\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "図4.2の出力"
      ],
      "metadata": {
        "id": "LOZ45vScyAWm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "sc = StandardScaler()\n",
        "X_train_std = sc.fit_transform(X_train)\n",
        "X_test_std = sc.transform(X_test)\n",
        "\n",
        "# COMBINING FOR FUTURE PLOTTING\n",
        "\n",
        "X_combined_standard = np.vstack((X_train_std, X_test_std))\n",
        "Y_combined = np.hstack((y_train_num, y_test_num))\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(8, 8))\n",
        "\n",
        "# Printing decision regions\n",
        "# plt.subplot(3,2,j)\n",
        "\n",
        "datalen = len(y_train)\n",
        "for cnt in range(datalen):\n",
        "    if y_train[cnt] == 'versicolor':\n",
        "        p1 = ax.scatter(X_train_std[:, 0][cnt],X_train_std[:, 1][cnt], marker='s', s=50, c=\"grey\")\n",
        "    elif y_train[cnt] == 'virginica':\n",
        "        p2 = ax.scatter(X_train_std[:, 0][cnt],X_train_std[:, 1][cnt], marker='^', s=50, c=\"black\")  \n",
        "\n",
        "ax.grid()\n",
        "ax.legend([p1, p2], ['versicolor', 'virginica'])\n",
        "\n",
        "plt.xlabel(x1label, fontsize=20)\n",
        "plt.ylabel(x2label, fontsize=20)\n",
        "plt.title('Iris Species', fontsize=20)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        },
        "id": "87jUZg4dyCRt",
        "outputId": "c0f1eed5-a65d-44a4-849b-c3fc65515816"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Iris Species')"
            ]
          },
          "metadata": {},
          "execution_count": 35
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 23776 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "図4.3などの出力（Cの値は0.01，0.1，1，10（図4.3），100）"
      ],
      "metadata": {
        "id": "DomJwsFIyCBy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "C_param_range = [0.01, 0.1, 1, 10, 100]\n",
        "\n",
        "acc_table = pd.DataFrame(columns=['C_parameter', 'Accuracy'])\n",
        "acc_table['C_parameter'] = C_param_range\n",
        "j = 0\n",
        "for i in C_param_range:\n",
        "    fig, ax = plt.subplots(figsize=(8, 8))\n",
        "    # Apply SVM model to training data\n",
        "    clf = SVC(kernel='linear', C=i, random_state=0)\n",
        "    clf.fit(X_train_std, y_train_num)\n",
        "    \n",
        "    # Predict using model\n",
        "    y_pred_num = clf.predict(X_test_std)\n",
        "    \n",
        "    # Saving accuracy score in table\n",
        "    acc_table.iloc[j, 1] = accuracy_score(y_test_num, y_pred_num)\n",
        "    j += 1\n",
        "    # Printing decision regions\n",
        "    # plt.subplot(3,2,j)\n",
        "    cnt = 0\n",
        "    datalen = len(y_train)\n",
        "    for cnt in range(datalen):\n",
        "        if y_train[cnt] == 'versicolor':\n",
        "            p1 = ax.scatter(X_train_std[:, 0][cnt],X_train_std[:, 1][cnt], marker='s', s=50, c=\"grey\")\n",
        "        elif y_train[cnt] == 'virginica':\n",
        "            p2 = ax.scatter(X_train_std[:, 0][cnt],X_train_std[:, 1][cnt], marker='^', s=50, c=\"black\")  \n",
        "\n",
        "    ax.grid()\n",
        "    ax.legend([p1, p2], ['versicolor', 'virginica'])\n",
        "    xlim = ax.get_xlim()\n",
        "    ylim = ax.get_ylim()\n",
        "    # create grid to evaluate model\n",
        "    xx = np.linspace(xlim[0]-1, xlim[1]+1, 30)\n",
        "    yy = np.linspace(ylim[0]-1, ylim[1]+1, 30)\n",
        "    XX, YY = np.meshgrid(xx, yy)\n",
        "    xy = np.vstack([XX.ravel(), YY.ravel()]).T\n",
        "    Z = clf.decision_function(xy).reshape(XX.shape)\n",
        "    plt.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5,\n",
        "                linestyles=['--', '-', '--'])\n",
        "    plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], \n",
        "                s=140, linewidth=1, \n",
        "                facecolors='none', edgecolors='k')\n",
        "    cnt = 0\n",
        "    datalen = len(y_test)\n",
        "    for cnt in range(datalen):\n",
        "        if y_test[cnt][0] == 'versicolor':\n",
        "            p1 = plt.scatter(X_test_std[:, 0][cnt], X_test_std[:,1][cnt], marker='+', s=50, c=\"grey\")\n",
        "        elif y_test[cnt][0] == 'virginica':\n",
        "            p2 = plt.scatter(X_test_std[:, 0][cnt], X_test_std[:,1][cnt], marker='+', s=50, c=\"black\")  \n",
        "\n",
        "    plt.xlabel(x1label, fontsize=20)\n",
        "    plt.ylabel(x2label, fontsize=20)\n",
        "    plt.title('Linear Kernel using C = %s'%i, fontsize=20)\n",
        "\n",
        "    plt.show()\n",
        "print(acc_table)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "3nP_1HlryBqB",
        "outputId": "f6b877f4-f142-4202-e0e4-0e9ef84bd115"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 73708 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 51956 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 45944 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 44348 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 43148 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   C_parameter  Accuracy\n",
            "0         0.01       0.8\n",
            "1         0.10  0.933333\n",
            "2         1.00  0.933333\n",
            "3        10.00       0.9\n",
            "4       100.00       0.9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "図4.4の出力"
      ],
      "metadata": {
        "id": "E1MDBZUNyDIZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "polynomial_degree_range = [1, 2, 3, 4, 5, 6] \n",
        "polynomial_degree_range = [3] \n",
        "\n",
        "acc_table = pd.DataFrame(columns=['degree', 'Accuracy'])\n",
        "acc_table['degree'] = polynomial_degree_range\n",
        "j = 0\n",
        "for i in polynomial_degree_range:\n",
        "    fig, ax = plt.subplots(figsize=(8, 8))\n",
        "    # Apply SVM model to training data\n",
        "    clf = SVC(kernel='poly', degree=i, C=1, random_state=0)\n",
        "    clf.fit(X_train_std, y_train_num)\n",
        "    \n",
        "    # Predict using model\n",
        "    y_pred_num = clf.predict(X_test_std)\n",
        "    \n",
        "    # Saving accuracy score in table\n",
        "    acc_table.iloc[j,1] = accuracy_score(y_test_num, y_pred_num)\n",
        "    j += 1\n",
        "    # Printing decision regions\n",
        "    cnt = 0\n",
        "    datalen = len(y_train)\n",
        "    for cnt in range(datalen):\n",
        "        if y_train[cnt] == 'versicolor':\n",
        "            p1 = ax.scatter(X_train_std[:, 0][cnt], X_train_std[:, 1][cnt], marker='s', s=50, c=\"grey\")\n",
        "        elif y_train[cnt] == 'virginica':\n",
        "            p2 = ax.scatter(X_train_std[:, 0][cnt], X_train_std[:, 1][cnt], marker='^', s=50, c=\"black\")  \n",
        "\n",
        "    ax.grid()\n",
        "    ax.legend([p1, p2], ['versicolor', 'virginica'])\n",
        "    xlim = ax.get_xlim()\n",
        "    ylim = ax.get_ylim()\n",
        "    # create grid to evaluate model\n",
        "    xx = np.linspace(xlim[0]-1, xlim[1]+1, 30)\n",
        "    yy = np.linspace(ylim[0]-1, ylim[1]+1, 30)\n",
        "    XX, YY = np.meshgrid(xx, yy)\n",
        "    xy = np.vstack([XX.ravel(), YY.ravel()]).T\n",
        "    Z = clf.decision_function(xy).reshape(XX.shape)\n",
        "    # plt.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5,\n",
        "    #             linestyles=['--', '-', '--'])\n",
        "    plt.contour(XX, YY, Z, colors='k', levels=[0], alpha=0.5,\n",
        "                linestyles=['-'])\n",
        "    # plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],\n",
        "    #                 s=140, linewidth=1\n",
        "    #                 facecolors='none', edgecolors='k')\n",
        "    cnt = 0\n",
        "    datalen = len(y_test)\n",
        "    for cnt in range(datalen):\n",
        "        if y_test[cnt][0] == 'versicolor':\n",
        "            p1 = plt.scatter(X_test_std[:, 0][cnt], X_test_std[:, 1][cnt], marker='+', s=50, c=\"grey\")\n",
        "        elif y_test[cnt][0] == 'virginica':\n",
        "            p2 = plt.scatter(X_test_std[:, 0][cnt], X_test_std[:, 1][cnt], marker='+', s=50, c=\"black\")  \n",
        "\n",
        "    plt.xlabel(x1label, fontsize=20)\n",
        "    plt.ylabel(x2label, fontsize=20)\n",
        "    plt.title('Polynomial Kernel using degree = %s'%i, fontsize=20)\n",
        "\n",
        "    plt.show()\n",
        "print(acc_table)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 565
        },
        "id": "gU5TaNpIyAO7",
        "outputId": "c8b50a9d-3bdb-4341-96ed-a50a1a056319"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "[encoded data, 34736 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   degree Accuracy\n",
            "0       3      0.8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "図4.5の出力"
      ],
      "metadata": {
        "id": "S3StGASnyAH5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import make_blobs\n",
        "from sklearn.svm import LinearSVC\n",
        "\n",
        "X, y = make_blobs(centers=4, random_state=8)\n",
        "y = y % 2\n",
        " \n",
        "linsvm = LinearSVC(max_iter=5500).fit(X, y)\n",
        " \n",
        "y_min, y_max = -15, 15\n",
        "b = linsvm.intercept_\n",
        "w0 = linsvm.coef_[0][0]\n",
        "w1 = linsvm.coef_[0][1]\n",
        "x_lower = -(b + w1 * y_min) / w0\n",
        "x_upper = -(b + w1 * y_max) / w0\n",
        " \n",
        "fig, ax = plt.subplots()\n",
        "X0 = X[y==0]\n",
        "X1 = X[y==1]\n",
        "ax.scatter(X0[:, 0], X0[:, 1], marker='o', s=60, ec='k')\n",
        "ax.scatter(X1[:, 0], X1[:, 1], marker='^', s=60, ec='k')\n",
        "#ax.plot([x_lower, x_upper], [y_min, y_max], linewidth=2, c='tab:green')\n",
        "ax.set_ylim(y_min, y_max)\n",
        "ax.set_xlabel(\"$x_{1}$\")\n",
        "ax.set_ylabel(\"$x_{2}$\")\n",
        "plt.show()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "Ca7EPgNDx_-r",
        "outputId": "54878546-5b1d-4c3b-837a-a27ca292b397"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 18096 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "図4.6の出力"
      ],
      "metadata": {
        "id": "wEr037Lux_3M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "X, y = make_blobs(centers=4, random_state=8)\n",
        "y = y % 2\n",
        "X_new = np.hstack((X, X[:, 1].reshape(-1, 1)**2))\n",
        "X0, X1 = X_new[y==0], X_new[y==1]\n",
        " \n",
        "linsvc = LinearSVC(max_iter=3700).fit(X_new, y)\n",
        "intercept = linsvc.intercept_\n",
        "coef = linsvc.coef_.ravel()\n",
        " \n",
        "fig, ax = plt.subplots(subplot_kw=dict(projection='3d'))\n",
        "ax.view_init(elev=-152, azim=-23)\n",
        " \n",
        "u = np.linspace(X_new[:, 0].min() - 2, X_new[:, 1].max() + 2)\n",
        "v = np.linspace(X_new[:, 0].min() - 2, X_new[:, 1].max() + 2)\n",
        "u, v = np.meshgrid(u, v)\n",
        "w = -(coef[0] * u + coef[1] * v + intercept) / coef[2]\n",
        " \n",
        "ax.scatter(X0[:, 0], X0[:, 1], X0[:, 2], marker='o', s=40, ec='k')\n",
        "ax.scatter(X1[:, 0], X1[:, 1], X1[:, 2], marker='^', s=40, ec='k')\n",
        "ax.plot_wireframe(u, v, w, rstride=8, cstride=8, color='tab:green', alpha=0.5)\n",
        " \n",
        "ax.set_xlabel(\"$x_{1}$\")\n",
        "ax.set_ylabel(\"$x_{2}$\")\n",
        "ax.set_zlabel(\"$x_{3}$\")\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "id": "oK3WqsE7x_wd",
        "outputId": "173c607c-d5a2-4328-f49f-e274338ab6de"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 47512 characters]\n"
          },
          "metadata": {}
        }
      ]
    }
  ]
}