1-Expected_Loss_Minimization-Weighting-Sampling.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyML/1Ps1jzQMQClOF959iwH"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Imports"],"metadata":{"id":"aP2kNOvOC77r"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"ViogCRDW3kjW"},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","\n","from sklearn.model_selection import train_test_split\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import confusion_matrix, classification_report\n","from sklearn.calibration import CalibratedClassifierCV\n","from sklearn.svm import SVC \n","from sklearn.ensemble import RandomForestClassifier\n","from sklearn.naive_bayes import GaussianNB\n","from imblearn.under_sampling import RandomUnderSampler\n","from imblearn.over_sampling import RandomOverSampler"]},{"cell_type":"markdown","source":["# Read and Preprocess data"],"metadata":{"id":"7QMo5VNFC_bl"}},{"cell_type":"code","source":["data_df = pd.read_fwf(\"/content/german.data-numeric\", header=None)\n","data_df = data_df.add_prefix(\"Attr_\")\n","data_df.rename(columns={\"Attr_24\": \"Target\"}, inplace=True)\n","data_df"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":488},"id":"SU0I_lhb5RbS","executionInfo":{"status":"ok","timestamp":1680563762180,"user_tz":-180,"elapsed":13,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"650f1cff-7fdd-4e2e-fa52-5ebc615f3fdb"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["     Attr_0  Attr_1  Attr_2  Attr_3  Attr_4  Attr_5  Attr_6  Attr_7  Attr_8  \\\n","0         1       6       4      12       5       5       3       4       1   \n","1         2      48       2      60       1       3       2       2       1   \n","2         4      12       4      21       1       4       3       3       1   \n","3         1      42       2      79       1       4       3       4       2   \n","4         1      24       3      49       1       3       3       4       4   \n","..      ...     ...     ...     ...     ...     ...     ...     ...     ...   \n","995       4      12       2      17       1       4       2       4       1   \n","996       1      30       2      39       1       3       1       4       2   \n","997       4      12       2       8       1       5       3       4       3   \n","998       1      45       2      18       1       3       3       4       4   \n","999       2      45       4      46       2       1       3       4       3   \n","\n","     Attr_9  ...  Attr_15  Attr_16  Attr_17  Attr_18  Attr_19  Attr_20  \\\n","0        67  ...        0        0        1        0        0        1   \n","1        22  ...        0        0        1        0        0        1   \n","2        49  ...        0        0        1        0        0        1   \n","3        45  ...        0        0        0        0        0        0   \n","4        53  ...        1        0        1        0        0        0   \n","..      ...  ...      ...      ...      ...      ...      ...      ...   \n","995      31  ...        0        0        1        0        0        1   \n","996      40  ...        0        1        1        0        0        1   \n","997      38  ...        0        0        1        0        0        1   \n","998      23  ...        0        0        1        0        0        0   \n","999      27  ...        0        1        1        0        0        1   \n","\n","     Attr_21  Attr_22  Attr_23  Target  \n","0          0        0        1       1  \n","1          0        0        1       2  \n","2          0        1        0       1  \n","3          0        0        1       1  \n","4          0        0        1       2  \n","..       ...      ...      ...     ...  \n","995        0        1        0       1  \n","996        0        0        0       1  \n","997        0        0        1       1  \n","998        0        0        1       2  \n","999        0        0        1       1  \n","\n","[1000 rows x 25 columns]"],"text/html":["\n","  <div id=\"df-560bf2c6-67f6-4b5e-ba50-05d9387bbd23\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Attr_0</th>\n","      <th>Attr_1</th>\n","      <th>Attr_2</th>\n","      <th>Attr_3</th>\n","      <th>Attr_4</th>\n","      <th>Attr_5</th>\n","      <th>Attr_6</th>\n","      <th>Attr_7</th>\n","      <th>Attr_8</th>\n","      <th>Attr_9</th>\n","      <th>...</th>\n","      <th>Attr_15</th>\n","      <th>Attr_16</th>\n","      <th>Attr_17</th>\n","      <th>Attr_18</th>\n","      <th>Attr_19</th>\n","      <th>Attr_20</th>\n","      <th>Attr_21</th>\n","      <th>Attr_22</th>\n","      <th>Attr_23</th>\n","      <th>Target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>6</td>\n","      <td>4</td>\n","      <td>12</td>\n","      <td>5</td>\n","      <td>5</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>1</td>\n","      <td>67</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>48</td>\n","      <td>2</td>\n","      <td>60</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>22</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>4</td>\n","      <td>12</td>\n","      <td>4</td>\n","      <td>21</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>49</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>1</td>\n","      <td>42</td>\n","      <td>2</td>\n","      <td>79</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>2</td>\n","      <td>45</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>1</td>\n","      <td>24</td>\n","      <td>3</td>\n","      <td>49</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>4</td>\n","      <td>53</td>\n","      <td>...</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>995</th>\n","      <td>4</td>\n","      <td>12</td>\n","      <td>2</td>\n","      <td>17</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>2</td>\n","      <td>4</td>\n","      <td>1</td>\n","      <td>31</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>996</th>\n","      <td>1</td>\n","      <td>30</td>\n","      <td>2</td>\n","      <td>39</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>2</td>\n","      <td>40</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>997</th>\n","      <td>4</td>\n","      <td>12</td>\n","      <td>2</td>\n","      <td>8</td>\n","      <td>1</td>\n","      <td>5</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>38</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>998</th>\n","      <td>1</td>\n","      <td>45</td>\n","      <td>2</td>\n","      <td>18</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>4</td>\n","      <td>23</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>999</th>\n","      <td>2</td>\n","      <td>45</td>\n","      <td>4</td>\n","      <td>46</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>27</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000 rows × 25 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-560bf2c6-67f6-4b5e-ba50-05d9387bbd23')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-560bf2c6-67f6-4b5e-ba50-05d9387bbd23 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-560bf2c6-67f6-4b5e-ba50-05d9387bbd23');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":76}]},{"cell_type":"code","source":["# PARAMETERS\n","N_TEST_SAMPLES = 200\n","TARGET = 'Target'\n","\n","# Split to Train-Validation and Test sets\n","train_val_df = data_df.iloc[:-N_TEST_SAMPLES, :]\n","test_df = data_df.iloc[-N_TEST_SAMPLES:, :]\n","\n","# Scale data\n","scaler = MinMaxScaler()\n","train_val_sc = scaler.fit_transform(train_val_df)\n","train_val_df_sc = pd.DataFrame(train_val_sc, columns=train_val_df.columns, index=train_val_df.index)\n","test_sc = scaler.transform(test_df)\n","test_df_sc = pd.DataFrame(test_sc, columns=test_df.columns, index=test_df.index)\n","\n","\n","# split to features and targets\n","X_train_val_df, y_train_val_df = train_val_df_sc.drop(TARGET, axis=1), train_val_df_sc[TARGET]\n","X_test_df, y_test_df = test_df_sc.drop(TARGET, axis=1), test_df_sc[TARGET]\n","\n","# Split to train and validation sets \n","X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(X_train_val_df,\n","                                                              y_train_val_df, \n","                                                              test_size=0.4,\n","                                                              shuffle=True,\n","                                                              random_state=0)\n","\n","print(X_train_df.shape, X_val_df.shape, X_test_df.shape)\n","print(y_train_df.shape, y_val_df.shape, y_test_df.shape)\n","\n","# Change 0's to -1's, because SVM requires that labels are +1 and -1\n","y_train_df[y_train_df == 0.] = -1\n","y_val_df[y_val_df == 0.] = -1\n","y_test_df[y_test_df == 0.] = -1"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0_fElbcHA3OZ","executionInfo":{"status":"ok","timestamp":1680563762181,"user_tz":-180,"elapsed":12,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"7b60671d-e6b7-4d7d-ea8c-0c4ebcb02c70"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(480, 24) (320, 24) (200, 24)\n","(480,) (320,) (200,)\n"]}]},{"cell_type":"markdown","source":["# 1. Minimizing the expected cost"],"metadata":{"id":"1w3keHSCIGO4"}},{"cell_type":"code","source":["# create the cost matrix\n","cost_matrix = [[0, 1],\n","               [5, 0]]\n","\n","classifiers = {\n","    'random forest': RandomForestClassifier(n_estimators=100, random_state=0),\n","    'linear SVM': SVC(kernel='linear', probability=True),\n","    'naive bayes': GaussianNB(),\n","}\n","\n","y_test = y_test_df.to_numpy(dtype='int64')\n","\n","for name in classifiers.keys():\n","  base_clf = classifiers[name]\n","  print(f'======================== {name} ======================== \\n')\n","\n","  print(\" ***** Cost minimization WITHOUT probability calibration *****\")\n","  base_clf.fit(pd.concat([X_train_df, X_val_df]), \n","               pd.concat([y_train_df, y_val_df]))\n","  y_pred_probs = base_clf.predict_proba(X_test_df)\n","  y_preds = np.argmin(np.matmul(y_pred_probs, np.array(cost_matrix).T), axis=1)\n","  y_preds[y_preds == 0] = -1\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")\n","\n","  print(\" ***** Cost minimization with SIGMOID calibration *****\")\n","  base_clf.fit(X_train_df, y_train_df)\n","  calibrated_clf = CalibratedClassifierCV(base_clf, method=\"sigmoid\", cv='prefit')\n","  calibrated_clf.fit(X_val_df, y_val_df)\n","  y_pred_probs = calibrated_clf.predict_proba(X_test_df)\n","  y_preds = np.argmin(np.matmul(y_pred_probs, np.array(cost_matrix).T), axis=1)\n","  y_preds[y_preds == 0] = -1\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")\n","\n","  print(\" ***** Cost minimization with ISOTONIC calibration *****\")\n","  base_clf.fit(X_train_df, y_train_df)\n","  calibrated_clf = CalibratedClassifierCV(base_clf, method=\"isotonic\", cv='prefit')\n","  calibrated_clf.fit(X_val_df, y_val_df)\n","  y_pred_probs = calibrated_clf.predict_proba(X_test_df)\n","  y_preds = np.argmin(np.matmul(y_pred_probs, np.array(cost_matrix).T), axis=1)\n","  y_preds[y_preds == 0] = -1\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"Total Cost = {total_cost}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"n4CXyhppIrfI","executionInfo":{"status":"ok","timestamp":1680563916388,"user_tz":-180,"elapsed":1298,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"448afc1f-2911-4faf-93a9-5d4db54d6d50"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["======================== random forest ======================== \n","\n"," ***** Cost minimization WITHOUT probability calibration *****\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n"]},{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","          -1       0.69      1.00      0.82       139\n","           1       0.00      0.00      0.00        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.35      0.50      0.41       200\n","weighted avg       0.48      0.69      0.57       200\n","\n","[[139  61]\n"," [  0   0]]\n","\n","Total Cost = 61\n"," ***** Cost minimization with SIGMOID calibration *****\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n"]},{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","          -1       0.69      1.00      0.82       139\n","           1       0.00      0.00      0.00        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.35      0.50      0.41       200\n","weighted avg       0.48      0.69      0.57       200\n","\n","[[139  61]\n"," [  0   0]]\n","\n","Total Cost = 61\n"," ***** Cost minimization with ISOTONIC calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.72      1.00      0.84       139\n","           1       1.00      0.11      0.21        61\n","\n","    accuracy                           0.73       200\n","   macro avg       0.86      0.56      0.52       200\n","weighted avg       0.81      0.73      0.64       200\n","\n","[[139  54]\n"," [  0   7]]\n","Total Cost = 54\n","======================== linear SVM ======================== \n","\n"," ***** Cost minimization WITHOUT probability calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.71      0.99      0.82       139\n","           1       0.67      0.07      0.12        61\n","\n","    accuracy                           0.70       200\n","   macro avg       0.69      0.53      0.47       200\n","weighted avg       0.69      0.70      0.61       200\n","\n","[[137  57]\n"," [  2   4]]\n","\n","Total Cost = 67\n"," ***** Cost minimization with SIGMOID calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.70      1.00      0.82       139\n","           1       1.00      0.02      0.03        61\n","\n","    accuracy                           0.70       200\n","   macro avg       0.85      0.51      0.43       200\n","weighted avg       0.79      0.70      0.58       200\n","\n","[[139  60]\n"," [  0   1]]\n","\n","Total Cost = 60\n"," ***** Cost minimization with ISOTONIC calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.69      1.00      0.82       139\n","           1       0.00      0.00      0.00        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.35      0.50      0.41       200\n","weighted avg       0.48      0.69      0.57       200\n","\n","[[139  61]\n"," [  0   0]]\n","Total Cost = 61\n","======================== naive bayes ======================== \n","\n"," ***** Cost minimization WITHOUT probability calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.78      0.88      0.82       139\n","           1       0.60      0.43      0.50        61\n","\n","    accuracy                           0.74       200\n","   macro avg       0.69      0.65      0.66       200\n","weighted avg       0.72      0.74      0.73       200\n","\n","[[122  35]\n"," [ 17  26]]\n","\n","Total Cost = 120\n"," ***** Cost minimization with SIGMOID calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.69      1.00      0.82       139\n","           1       0.00      0.00      0.00        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.35      0.50      0.41       200\n","weighted avg       0.48      0.69      0.57       200\n","\n","[[139  61]\n"," [  0   0]]\n","\n","Total Cost = 61\n"," ***** Cost minimization with ISOTONIC calibration *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.69      1.00      0.82       139\n","           1       0.00      0.00      0.00        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.35      0.50      0.41       200\n","weighted avg       0.48      0.69      0.57       200\n","\n","[[139  61]\n"," [  0   0]]\n","Total Cost = 61\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n"]}]},{"cell_type":"markdown","source":["Από τα παραπάνω αποτελέσματα παρατηρώ τα παρακάτω:\n","1. Για το **Random Forest**:\n","  * *χωρίς probability calibration* έχουμε τα χειρότερα αποτελέσματα με **total cost = 61**.\n","  * με *sigmoid calibration* έχουμε τα ίδια αποτελέσματα σε σχέση με την προηγούμενη περίπτωση.\n","  * με *isotonic calibration* πετυχαίνουμε τα καλύτερα αποτελέσματα για τον συγκεκριμένο αλγόριθμο με **total cost = 54**.\n","\n","2. Για το **Linear SVM**:\n","  * *χωρίς probability calibration* έχουμε τα χειρότερα αποτελέσματα με **total cost = 67**.\n","  * με *sigmoid calibration* πετυχαίνουμε τα καλύτερα αποτελέσματα για τον συγκεκριμένο αλγόριθμο με **total cost = 60**.\n","  * με *isotonic calibration* πετυχαίνουμε παραπλήσια αποτελέσματα, σε σχέση με την περίπτωση του sigmoid calibration, με **total cost = 60**.\n","\n","3. Για το **Naive Bayes**:\n","  * *χωρίς probability calibration* έχουμε τα χειρότερα αποτελέσματα με **total cost = 120**.\n","  * με *sigmoid calibration* πετυχαίνουμε τα καλύτερα αποτελέσματα για τον συγκεκριμένο αλγόριθμο με **total cost = 61**.\n","  * με *isotonic calibration* πετυχαίνουμε τα ίδια αποτελέσματα σε σχέση με την περίπτωση του sigmoid calibration με **total cost = 61**.\n","\n","Συνολικά, παρατηρούμε ότι:\n","* στην προκειμένη περίπτωση **η μέθοδος probability calibration (τόσο η sigmoid όοο και η isotonic) *βελτιώνουν* την επίδοση των παραπάνω αλγορίθμων**\n","* στο συγκεκειμένο task και dataset, τα **καλύτερα αποτελέσματα** πετυχαίνει το μοντέλο **Random Forest**.\n","* στο συγκεκειμένο task και dataset, τη **μεγαλύτερη βελτίωση όταν εφαρμόζεται calibration** πετυχαίνει το μοντέλο **Naive Bayes**."],"metadata":{"id":"nmEw4Dxd4uUs"}},{"cell_type":"markdown","source":["# 2. Sampling"],"metadata":{"id":"ziBp2mbMjsAa"}},{"cell_type":"code","source":["X_train_val_df = pd.concat([X_train_df, X_val_df])\n","y_train_val_df = pd.concat([y_train_df, y_val_df])"],"metadata":{"id":"a1KKMSYFp77V"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["y_train_val_df.value_counts()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"jLk5Fs-Uj152","executionInfo":{"status":"ok","timestamp":1680567007118,"user_tz":-180,"elapsed":258,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"81d0bd8b-f543-45ce-880f-cfbaf5b12b50"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["-1.0    561\n"," 1.0    239\n","Name: Target, dtype: int64"]},"metadata":{},"execution_count":86}]},{"cell_type":"code","source":["# create the cost matrix\n","cost_matrix = [[0, 1],\n","               [5, 0]]\n","\n","classifiers = {\n","    'random forest': RandomForestClassifier(n_estimators=100, random_state=0),\n","    'linear SVM': SVC(kernel='linear', probability=True),\n","    'naive bayes': GaussianNB(),\n","}\n","\n","y_test = y_test_df.to_numpy(dtype='int64')\n","\n","for name in classifiers.keys():\n","  base_clf = classifiers[name]\n","  print(f'======================== {name} ========================')\n","\n","  print(\"\\n***** WITHOUT sampling *****\")\n","  base_clf.fit(X_train_val_df, y_train_val_df)\n","  y_preds = base_clf.predict(X_test_df)\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")\n","\n","  print(\"\\n***** with UNDER-SAMPLING *****\")\n","  sampler = RandomUnderSampler(sampling_strategy={-1: 239, 1: 239}, random_state=0)\n","  X_rs_df, y_rs_df = sampler.fit_resample(X_train_val_df, y_train_val_df)\n","  print(y_rs_df.value_counts())\n","  base_clf.fit(X_rs_df, y_rs_df)\n","  y_preds = base_clf.predict(X_test_df)\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")\n","\n","  print(\"\\n***** with OVER-SAMPLING *****\")\n","  sampler = RandomOverSampler(sampling_strategy={-1: 561, 1: 561}, random_state=0)\n","  X_rs_df, y_rs_df = sampler.fit_resample(X_train_val_df, y_train_val_df)\n","  print(y_rs_df.value_counts())\n","  base_clf.fit(X_rs_df, y_rs_df)\n","  y_preds = base_clf.predict(X_test_df)\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cnorL7WVkLNn","executionInfo":{"status":"ok","timestamp":1680567651185,"user_tz":-180,"elapsed":3732,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"4a71d036-a983-4ccb-e28c-864ade2d109b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["======================== random forest ========================\n","\n","***** WITHOUT sampling *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.78      0.92      0.84       139\n","           1       0.69      0.39      0.50        61\n","\n","    accuracy                           0.76       200\n","   macro avg       0.73      0.66      0.67       200\n","weighted avg       0.75      0.76      0.74       200\n","\n","[[128  37]\n"," [ 11  24]]\n","\n","Total Cost = 92\n","\n","***** with UNDER-SAMPLING *****\n","-1.0    239\n"," 1.0    239\n","Name: Target, dtype: int64\n","              precision    recall  f1-score   support\n","\n","          -1       0.85      0.76      0.81       139\n","           1       0.57      0.70      0.63        61\n","\n","    accuracy                           0.74       200\n","   macro avg       0.71      0.73      0.72       200\n","weighted avg       0.77      0.74      0.75       200\n","\n","[[106  18]\n"," [ 33  43]]\n","\n","Total Cost = 183\n","\n","***** with OVER-SAMPLING *****\n"," 1.0    561\n","-1.0    561\n","Name: Target, dtype: int64\n","              precision    recall  f1-score   support\n","\n","          -1       0.82      0.87      0.84       139\n","           1       0.65      0.56      0.60        61\n","\n","    accuracy                           0.78       200\n","   macro avg       0.74      0.71      0.72       200\n","weighted avg       0.77      0.78      0.77       200\n","\n","[[121  27]\n"," [ 18  34]]\n","\n","Total Cost = 117\n","======================== linear SVM ========================\n","\n","***** WITHOUT sampling *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.82      0.89      0.86       139\n","           1       0.69      0.56      0.62        61\n","\n","    accuracy                           0.79       200\n","   macro avg       0.76      0.72      0.74       200\n","weighted avg       0.78      0.79      0.78       200\n","\n","[[124  27]\n"," [ 15  34]]\n","\n","Total Cost = 102\n","\n","***** with UNDER-SAMPLING *****\n","-1.0    239\n"," 1.0    239\n","Name: Target, dtype: int64\n","              precision    recall  f1-score   support\n","\n","          -1       0.86      0.73      0.79       139\n","           1       0.55      0.74      0.63        61\n","\n","    accuracy                           0.73       200\n","   macro avg       0.71      0.74      0.71       200\n","weighted avg       0.77      0.73      0.74       200\n","\n","[[102  16]\n"," [ 37  45]]\n","\n","Total Cost = 201\n","\n","***** with OVER-SAMPLING *****\n"," 1.0    561\n","-1.0    561\n","Name: Target, dtype: int64\n","              precision    recall  f1-score   support\n","\n","          -1       0.87      0.72      0.79       139\n","           1       0.54      0.75      0.63        61\n","\n","    accuracy                           0.73       200\n","   macro avg       0.71      0.74      0.71       200\n","weighted avg       0.77      0.73      0.74       200\n","\n","[[100  15]\n"," [ 39  46]]\n","\n","Total Cost = 210\n","======================== naive bayes ========================\n","\n","***** WITHOUT sampling *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.84      0.76      0.80       139\n","           1       0.55      0.67      0.60        61\n","\n","    accuracy                           0.73       200\n","   macro avg       0.69      0.71      0.70       200\n","weighted avg       0.75      0.73      0.74       200\n","\n","[[105  20]\n"," [ 34  41]]\n","\n","Total Cost = 190\n","\n","***** with UNDER-SAMPLING *****\n","-1.0    239\n"," 1.0    239\n","Name: Target, dtype: int64\n","              precision    recall  f1-score   support\n","\n","          -1       0.86      0.67      0.75       139\n","           1       0.50      0.75      0.60        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.68      0.71      0.68       200\n","weighted avg       0.75      0.69      0.71       200\n","\n","[[93 15]\n"," [46 46]]\n","\n","Total Cost = 245\n","\n","***** with OVER-SAMPLING *****\n"," 1.0    561\n","-1.0    561\n","Name: Target, dtype: int64\n","              precision    recall  f1-score   support\n","\n","          -1       0.86      0.64      0.73       139\n","           1       0.48      0.75      0.59        61\n","\n","    accuracy                           0.68       200\n","   macro avg       0.67      0.70      0.66       200\n","weighted avg       0.74      0.68      0.69       200\n","\n","[[89 15]\n"," [50 46]]\n","\n","Total Cost = 265\n"]}]},{"cell_type":"markdown","source":["Από τα παραπάνω αποτελέσματα παρατηρώ τα παρακάτω:\n","1. Για το **Random Forest**:\n","  * *χωρίς sampling* έχουμε τα καλύτερα αποτελέσματα με **total cost = 92**.\n","  * με *undersampling* έχουμε τα χειρότερα αποτελέσματα για τον συγκεκριμένο αλγόριρθμο.\n","  * με *οversampling* επίσης πετυχαίνουμε χειρότερα αποτελέσματα σε σχέση με την περίπτωση που δεν εφαρμόζουμε καθόλου την τεχνική του sampling.\n","\n","2. Για το **Linear SVM**:\n","  * *χωρίς sampling* έχουμε τα καλύτερα αποτελέσματα με **total cost = 102**.\n","  * με *undersampling* πετυχαίνουμε χειρότερα αποτελέσματα σε σχέση με την περίπτωση που δεν εφαρμόζουμε καθόλου την τεχνική του sampling.\n","  * με *οversampling* έχουμε τα χειρότερα αποτελέσματα για τον συγκεκριμένο αλγόριρθμο \n","\n","3. Για το **Naive Bayes**:\n","  * *χωρίς sampling* έχουμε τα καλύτερα αποτελέσματα με **total cost = 190**.\n","  * με *undersampling* πετυχαίνουμε χειρότερα αποτελέσματα σε σχέση με την περίπτωση που δεν εφαρμόζουμε καθόλου την τεχνική του sampling.\n","  * με *οversampling* έχουμε τα χειρότερα αποτελέσματα για τον συγκεκριμένο αλγόριρθμο.\n","\n","Συνολικά, παρατηρούμε ότι:\n","* στην προκειμένη περίπτωση **η μέθοδος sampling (τόσο το oversampling όοο και το undersampling) *χειροτερεύουν* την επίδοση των παραπάνω αλγορίθμων**\n","* η σειρά με την οποία παρουσίασα τους παραπάνω αλγορίθμους είναι και η σειρά κατάταξής τους στο συγκεκειμένο task και dataset."],"metadata":{"id":"7NIRUcuh02bL"}},{"cell_type":"markdown","source":["# 3. Weighting"],"metadata":{"id":"qxPxv0xcpeOX"}},{"cell_type":"code","source":["X_train_val_df = pd.concat([X_train_df, X_val_df])\n","y_train_val_df = pd.concat([y_train_df, y_val_df])"],"metadata":{"id":"ErCEa178qXhF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["y_train_val_df.value_counts()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1680568296803,"user_tz":-180,"elapsed":5,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"54827885-ad13-4456-b1fe-8e8d10b0894c","id":"JkGhGtAFqXhF"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["-1.0    561\n"," 1.0    239\n","Name: Target, dtype: int64"]},"metadata":{},"execution_count":93}]},{"cell_type":"code","source":["# create the cost matrix\n","cost_matrix = [[0, 1],\n","               [5, 0]]\n","\n","classifiers = {\n","    'random forest': RandomForestClassifier(n_estimators=100, random_state=0),\n","    'linear SVM': SVC(kernel='linear', probability=True),\n","    'naive bayes': GaussianNB(),\n","}\n","\n","y_test = y_test_df.to_numpy(dtype='int64')\n","\n","for name in classifiers.keys():\n","  base_clf = classifiers[name]\n","  print(f'======================== {name} ========================')\n","\n","  print(\"\\n***** WITHOUT weights *****\")\n","  base_clf.fit(X_train_val_df, y_train_val_df)\n","  y_preds = base_clf.predict(X_test_df)\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")\n","\n","  print(\"\\n***** WITH weights *****\")\n","  weights = np.zeros(y_train_val_df.shape[0])\n","  weights[y_train_val_df == -1] = 5\n","  weights[y_train_val_df == 1] = 1\n","  base_clf.fit(X_train_val_df, y_train_val_df, weights)\n","  y_preds = base_clf.predict(X_test_df)\n","  print(classification_report(y_test, y_preds))\n","  conf_m = confusion_matrix(y_test, y_preds).T\n","  print(conf_m)\n","  total_cost = np.sum(conf_m * cost_matrix)\n","  print(f\"\\nTotal Cost = {total_cost}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1680568297493,"user_tz":-180,"elapsed":693,"user":{"displayName":"Achil Pal","userId":"04608072963754583562"}},"outputId":"6fcc9898-8075-469c-ff83-69408a8110b1","id":"NA3fjrGkpeOX"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["======================== random forest ========================\n","\n","***** WITHOUT weights *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.78      0.92      0.84       139\n","           1       0.69      0.39      0.50        61\n","\n","    accuracy                           0.76       200\n","   macro avg       0.73      0.66      0.67       200\n","weighted avg       0.75      0.76      0.74       200\n","\n","[[128  37]\n"," [ 11  24]]\n","\n","Total Cost = 92\n","\n","***** WITH weights *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.78      0.90      0.83       139\n","           1       0.64      0.41      0.50        61\n","\n","    accuracy                           0.75       200\n","   macro avg       0.71      0.65      0.67       200\n","weighted avg       0.74      0.75      0.73       200\n","\n","[[125  36]\n"," [ 14  25]]\n","\n","Total Cost = 106\n","======================== linear SVM ========================\n","\n","***** WITHOUT weights *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.82      0.89      0.86       139\n","           1       0.69      0.56      0.62        61\n","\n","    accuracy                           0.79       200\n","   macro avg       0.76      0.72      0.74       200\n","weighted avg       0.78      0.79      0.78       200\n","\n","[[124  27]\n"," [ 15  34]]\n","\n","Total Cost = 102\n","\n","***** WITH weights *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.69      1.00      0.82       139\n","           1       0.00      0.00      0.00        61\n","\n","    accuracy                           0.69       200\n","   macro avg       0.35      0.50      0.41       200\n","weighted avg       0.48      0.69      0.57       200\n","\n","[[139  61]\n"," [  0   0]]\n","\n","Total Cost = 61\n","======================== naive bayes ========================\n","\n","***** WITHOUT weights *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.84      0.76      0.80       139\n","           1       0.55      0.67      0.60        61\n","\n","    accuracy                           0.73       200\n","   macro avg       0.69      0.71      0.70       200\n","weighted avg       0.75      0.73      0.74       200\n","\n","[[105  20]\n"," [ 34  41]]\n","\n","Total Cost = 190\n","\n","***** WITH weights *****\n","              precision    recall  f1-score   support\n","\n","          -1       0.78      0.88      0.82       139\n","           1       0.60      0.43      0.50        61\n","\n","    accuracy                           0.74       200\n","   macro avg       0.69      0.65      0.66       200\n","weighted avg       0.72      0.74      0.73       200\n","\n","[[122  35]\n"," [ 17  26]]\n","\n","Total Cost = 120\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.9/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n"]}]},{"cell_type":"markdown","source":["Από τα παραπάνω αποτελέσματα παρατηρώ τα παρακάτω:\n","1. Για το **Random Forest**:\n","  * *χωρίς weights* έχουμε καλύτερα αποτελέσματα (**total cost = 92**) σε σχέση με την περίπτωση όπου *'εχουμε weights* (**total cost = 102**).\n","\n","2. Για το **Linear SVM**:\n","  * *χωρίς weights* έχουμε χειρότερα αποτελέσματα (**total cost = 102**) σε σχέση με την περίπτωση όπου *'εχουμε weights* (**total cost = 61**).\n","\n","3. Για το **Naive Bayes**:\n","  * *χωρίς weights* έχουμε χειρότερα αποτελέσματα (**total cost = 190**) σε σχέση με την περίπτωση όπου *'εχουμε weights* (**total cost = 120**).\n","\n","Συνολικά, παρατηρούμε ότι:\n","* στην προκειμένη περίπτωση **η χρήση των weights *βελτιώνουν* την επίδοση των αλγορίθμων Linear SVM και Naive Bayes**, ενώ ** *χειροτερεύουν* την επίδοση του αλγορίθμου Random Forest**\n","* στο συγκεκειμένο task και dataset, τα **καλύτερα αποτελέσματα** πετυχαίνει το μοντέλο **Linear SVM**, όταν εφαρμόζονται τα βάρη.\n","* στο συγκεκειμένο task και dataset, τη **μεγαλύτερη βελτίωση όταν εφαρμόζεται η χρήση των weights** πετυχαίνει το μοντέλο **Naive Bayes**."],"metadata":{"id":"357aGg457fxl"}},{"cell_type":"code","source":[],"metadata":{"id":"LjuHOXqK9rl2"},"execution_count":null,"outputs":[]}]}