examples/autoai/demo_autoai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aware-sessions",
   "metadata": {},
   "source": [
    "## In this demo, we will show how to use uq360 metrics for hyperparameters optimization using Lale and sklearn gridsearchCV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "processed-scheduling",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "organic-federation",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "pharmaceutical-journalist",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lale in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (0.4.14)\n",
      "Requirement already satisfied: black==19.10b0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (19.10b0)\n",
      "Requirement already satisfied: astunparse in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (1.6.3)\n",
      "Requirement already satisfied: h5py in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (2.10.0)\n",
      "Requirement already satisfied: jsonsubschema in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (0.0.1)\n",
      "Requirement already satisfied: jsonschema in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (3.2.0)\n",
      "Requirement already satisfied: numpy in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (1.18.5)\n",
      "Requirement already satisfied: pandas in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (1.2.3)\n",
      "Requirement already satisfied: hyperopt in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (0.2.5)\n",
      "Requirement already satisfied: scipy in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (1.5.3)\n",
      "Requirement already satisfied: scikit-learn>=0.20.3 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (0.24.1)\n",
      "Requirement already satisfied: graphviz in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (0.16)\n",
      "Requirement already satisfied: decorator in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from lale) (4.4.2)\n",
      "Requirement already satisfied: regex in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (2020.11.13)\n",
      "Requirement already satisfied: attrs>=18.1.0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (20.3.0)\n",
      "Requirement already satisfied: pathspec<1,>=0.6 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (0.8.1)\n",
      "Requirement already satisfied: typed-ast>=1.4.0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (1.4.2)\n",
      "Requirement already satisfied: toml>=0.9.4 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (0.10.2)\n",
      "Requirement already satisfied: click>=6.5 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (7.1.2)\n",
      "Requirement already satisfied: appdirs in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from black==19.10b0->lale) (1.4.4)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from scikit-learn>=0.20.3->lale) (1.0.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from scikit-learn>=0.20.3->lale) (2.1.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from astunparse->lale) (0.36.2)\n",
      "Requirement already satisfied: six<2.0,>=1.6.1 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from astunparse->lale) (1.15.0)\n",
      "Requirement already satisfied: networkx>=2.2 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from hyperopt->lale) (2.5)\n",
      "Requirement already satisfied: tqdm in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from hyperopt->lale) (4.59.0)\n",
      "Requirement already satisfied: future in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from hyperopt->lale) (0.18.2)\n",
      "Requirement already satisfied: cloudpickle in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from hyperopt->lale) (1.6.0)\n",
      "Requirement already satisfied: importlib-metadata in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from jsonschema->lale) (3.7.0)\n",
      "Requirement already satisfied: setuptools in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from jsonschema->lale) (54.0.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from jsonschema->lale) (0.17.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from importlib-metadata->jsonschema->lale) (3.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from importlib-metadata->jsonschema->lale) (3.7.4.3)\n",
      "Requirement already satisfied: greenery in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from jsonsubschema->lale) (3.3.1)\n",
      "Requirement already satisfied: python-intervals in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from jsonsubschema->lale) (1.10.0.post1)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from pandas->lale) (2021.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (from pandas->lale) (2.8.1)\n",
      "Requirement already satisfied: liac-arff>=2.4.0 in /Users/psattig/anaconda3/envs/uq360/lib/python3.7/site-packages (2.5.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install lale\n",
    "!pip install 'liac-arff>=2.4.0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fossil-lottery",
   "metadata": {},
   "outputs": [],
   "source": [
    "from uq360.utils.misc import make_sklearn_compatible_scorer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exact-yahoo",
   "metadata": {},
   "source": [
    "### The uq360 metrics can be converted to a sklearn scorer usign `make_sklearn_compatible_scorer` utility function. \n",
    "\n",
    "#### We will now show to how to convert ECE and AURRRC metric in uq360 into scorer that can be used with LALE framework for hyperparameter optimization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dependent-philosophy",
   "metadata": {},
   "outputs": [],
   "source": [
    "sklearn_aurrrc = make_sklearn_compatible_scorer(task_type=\"classification\", metric=\"aurrrc\", greater_is_better=False)\n",
    "sklearn_ece = make_sklearn_compatible_scorer(task_type=\"classification\", metric=\"ece\", greater_is_better=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "vanilla-dryer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lale\n",
    "from lale.lib.lale import Hyperopt\n",
    "lale.wrap_imported_operators()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "false-childhood",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "X, y = datasets.load_breast_cancer(return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "checked-waterproof",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "rotary-trunk",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cubic-startup",
   "metadata": {},
   "outputs": [],
   "source": [
    "from lale.lib.sklearn import AdaBoostClassifier as Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "modified-armenia",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_ece = Hyperopt(estimator=Model, cv=3, max_evals=20, scoring=sklearn_ece, verbose=True)\n",
    "clf_aurrrc = Hyperopt(estimator=Model, cv=3, max_evals=20, scoring=sklearn_aurrrc, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "blank-distance",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:29<00:00,  1.46s/trial, best loss: 0.259819517887197] \n"
     ]
    }
   ],
   "source": [
    "trained_with_ece = clf_ece.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "banner-wholesale",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:30<00:00,  1.51s/trial, best loss: 0.004077380952380953]\n"
     ]
    }
   ],
   "source": [
    "trained_with_aurrrc = clf_aurrrc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ambient-anderson",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:29<00:00,  1.50s/trial, best loss: -0.9606299212598426]\n"
     ]
    }
   ],
   "source": [
    "clf_accuracy = Hyperopt(estimator=Model, cv=3, max_evals=20, verbose=True)\n",
    "trained_with_accuracy = clf_accuracy.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "immune-transcription",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "beginning-stomach",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98        67\n",
      "           1       0.99      0.98      0.99       121\n",
      "\n",
      "    accuracy                           0.98       188\n",
      "   macro avg       0.98      0.98      0.98       188\n",
      "weighted avg       0.98      0.98      0.98       188\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, trained_with_accuracy.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "under-mexico",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97        67\n",
      "           1       0.99      0.98      0.98       121\n",
      "\n",
      "    accuracy                           0.98       188\n",
      "   macro avg       0.97      0.98      0.98       188\n",
      "weighted avg       0.98      0.98      0.98       188\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, trained_with_ece.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "delayed-davis",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.97      0.96        67\n",
      "           1       0.98      0.98      0.98       121\n",
      "\n",
      "    accuracy                           0.97       188\n",
      "   macro avg       0.97      0.97      0.97       188\n",
      "weighted avg       0.97      0.97      0.97       188\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, trained_with_aurrrc.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "controlling-coaching",
   "metadata": {},
   "source": [
    "## Lets us now see how to use `uq360` models with sklearn's `GridsearchCV`\n",
    "\n",
    "### We use the boston housing dataset with `QuantileRegression` model while optimizing the `picp` metric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "diagnostic-antigua",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bearing-paradise",
   "metadata": {},
   "outputs": [],
   "source": [
    "house_prices_dataset = datasets.load_boston()\n",
    "house_prices_df = pd.DataFrame(house_prices_dataset['data'])\n",
    "house_prices_df.columns = house_prices_dataset['feature_names']\n",
    "\n",
    "all_features = ['RM','CRIM','PTRATIO', 'DIS']\n",
    "X = house_prices_df[all_features].values\n",
    "y = house_prices_dataset['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "composed-dairy",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "attractive-replacement",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_config = {\n",
    "    \"alpha\":0.95,\n",
    "    \"n_estimators\":20,\n",
    "    \"max_depth\":3,\n",
    "    \"learning_rate\":0.1,\n",
    "    \"min_samples_leaf\":20,\n",
    "    \"min_samples_split\":20\n",
    "}\n",
    "\n",
    "configs  = {\"config\": []}\n",
    "for num_estimators in [1, 2, 5, 10, 20, 30, 40, 50]:\n",
    "    config = base_config.copy()\n",
    "    config[\"n_estimators\"] = num_estimators\n",
    "    configs[\"config\"].append(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "electrical-sight",
   "metadata": {},
   "outputs": [],
   "source": [
    "from uq360.algorithms.quantile_regression import QuantileRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "packed-services",
   "metadata": {},
   "outputs": [],
   "source": [
    "sklearn_picp = make_sklearn_compatible_scorer(task_type=\"regression\", metric=\"picp\", greater_is_better=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "streaming-religion",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = GridSearchCV(QuantileRegression(config=base_config), configs, scoring=sklearn_picp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "public-bedroom",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=<uq360.algorithms.quantile_regression.quantile_regression.QuantileRegression object at 0x7ff1c83a9650>,\n",
       "             param_grid={'config': [{'alpha': 0.95, 'learning_rate': 0.1,\n",
       "                                     'max_depth': 3, 'min_samples_leaf': 20,\n",
       "                                     'min_samples_split': 20,\n",
       "                                     'n_estimators': 1},\n",
       "                                    {'alpha': 0.95, 'learning_rate': 0.1,\n",
       "                                     'max_depth': 3, 'min_samples_leaf': 20,\n",
       "                                     'min_samples_split': 20,\n",
       "                                     '...\n",
       "                                     'min_samples_split': 20,\n",
       "                                     'n_estimators': 30},\n",
       "                                    {'alpha': 0.95, 'learning_rate': 0.1,\n",
       "                                     'max_depth': 3, 'min_samples_leaf': 20,\n",
       "                                     'min_samples_split': 20,\n",
       "                                     'n_estimators': 40},\n",
       "                                    {'alpha': 0.95, 'learning_rate': 0.1,\n",
       "                                     'max_depth': 3, 'min_samples_leaf': 20,\n",
       "                                     'min_samples_split': 20,\n",
       "                                     'n_estimators': 50}]},\n",
       "             scoring=<function make_sklearn_compatible_scorer.<locals>.sklearn_compatible_score at 0x7ff1a09f75f0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dress-experience",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(clf.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "saving-buddy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_config</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.054366</td>\n",
       "      <td>0.003077</td>\n",
       "      <td>0.000536</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 1, 'max_depth'...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 1, ...</td>\n",
       "      <td>0.929577</td>\n",
       "      <td>0.873239</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.842857</td>\n",
       "      <td>0.884064</td>\n",
       "      <td>0.033151</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.051386</td>\n",
       "      <td>0.000514</td>\n",
       "      <td>0.000507</td>\n",
       "      <td>0.000066</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 2, 'max_depth'...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 2, ...</td>\n",
       "      <td>0.957746</td>\n",
       "      <td>0.887324</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.895372</td>\n",
       "      <td>0.037771</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.052438</td>\n",
       "      <td>0.001631</td>\n",
       "      <td>0.000601</td>\n",
       "      <td>0.000234</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 5, 'max_depth'...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 5, ...</td>\n",
       "      <td>0.943662</td>\n",
       "      <td>0.901408</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.845070</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.892555</td>\n",
       "      <td>0.036675</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.056137</td>\n",
       "      <td>0.005973</td>\n",
       "      <td>0.000513</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 10, 'max_depth...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 10,...</td>\n",
       "      <td>0.943662</td>\n",
       "      <td>0.887324</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.828571</td>\n",
       "      <td>0.886841</td>\n",
       "      <td>0.040526</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.063329</td>\n",
       "      <td>0.007458</td>\n",
       "      <td>0.000620</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 20, 'max_depth...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 20,...</td>\n",
       "      <td>0.957746</td>\n",
       "      <td>0.887324</td>\n",
       "      <td>0.929577</td>\n",
       "      <td>0.845070</td>\n",
       "      <td>0.842857</td>\n",
       "      <td>0.892515</td>\n",
       "      <td>0.045547</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.062398</td>\n",
       "      <td>0.006398</td>\n",
       "      <td>0.000588</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 30, 'max_depth...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 30,...</td>\n",
       "      <td>0.957746</td>\n",
       "      <td>0.873239</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.845070</td>\n",
       "      <td>0.842857</td>\n",
       "      <td>0.886881</td>\n",
       "      <td>0.044075</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.064843</td>\n",
       "      <td>0.006143</td>\n",
       "      <td>0.000870</td>\n",
       "      <td>0.000315</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 40, 'max_depth...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 40,...</td>\n",
       "      <td>0.943662</td>\n",
       "      <td>0.887324</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.842857</td>\n",
       "      <td>0.889698</td>\n",
       "      <td>0.036634</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.062831</td>\n",
       "      <td>0.003828</td>\n",
       "      <td>0.000769</td>\n",
       "      <td>0.000323</td>\n",
       "      <td>{'alpha': 0.95, 'n_estimators': 50, 'max_depth...</td>\n",
       "      <td>{'config': {'alpha': 0.95, 'n_estimators': 50,...</td>\n",
       "      <td>0.943662</td>\n",
       "      <td>0.873239</td>\n",
       "      <td>0.915493</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.842857</td>\n",
       "      <td>0.886881</td>\n",
       "      <td>0.037244</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0       0.054366      0.003077         0.000536        0.000057   \n",
       "1       0.051386      0.000514         0.000507        0.000066   \n",
       "2       0.052438      0.001631         0.000601        0.000234   \n",
       "3       0.056137      0.005973         0.000513        0.000041   \n",
       "4       0.063329      0.007458         0.000620        0.000073   \n",
       "5       0.062398      0.006398         0.000588        0.000048   \n",
       "6       0.064843      0.006143         0.000870        0.000315   \n",
       "7       0.062831      0.003828         0.000769        0.000323   \n",
       "\n",
       "                                        param_config  \\\n",
       "0  {'alpha': 0.95, 'n_estimators': 1, 'max_depth'...   \n",
       "1  {'alpha': 0.95, 'n_estimators': 2, 'max_depth'...   \n",
       "2  {'alpha': 0.95, 'n_estimators': 5, 'max_depth'...   \n",
       "3  {'alpha': 0.95, 'n_estimators': 10, 'max_depth...   \n",
       "4  {'alpha': 0.95, 'n_estimators': 20, 'max_depth...   \n",
       "5  {'alpha': 0.95, 'n_estimators': 30, 'max_depth...   \n",
       "6  {'alpha': 0.95, 'n_estimators': 40, 'max_depth...   \n",
       "7  {'alpha': 0.95, 'n_estimators': 50, 'max_depth...   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "0  {'config': {'alpha': 0.95, 'n_estimators': 1, ...           0.929577   \n",
       "1  {'config': {'alpha': 0.95, 'n_estimators': 2, ...           0.957746   \n",
       "2  {'config': {'alpha': 0.95, 'n_estimators': 5, ...           0.943662   \n",
       "3  {'config': {'alpha': 0.95, 'n_estimators': 10,...           0.943662   \n",
       "4  {'config': {'alpha': 0.95, 'n_estimators': 20,...           0.957746   \n",
       "5  {'config': {'alpha': 0.95, 'n_estimators': 30,...           0.957746   \n",
       "6  {'config': {'alpha': 0.95, 'n_estimators': 40,...           0.943662   \n",
       "7  {'config': {'alpha': 0.95, 'n_estimators': 50,...           0.943662   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "0           0.873239           0.915493           0.859155           0.842857   \n",
       "1           0.887324           0.915493           0.859155           0.857143   \n",
       "2           0.901408           0.915493           0.845070           0.857143   \n",
       "3           0.887324           0.915493           0.859155           0.828571   \n",
       "4           0.887324           0.929577           0.845070           0.842857   \n",
       "5           0.873239           0.915493           0.845070           0.842857   \n",
       "6           0.887324           0.915493           0.859155           0.842857   \n",
       "7           0.873239           0.915493           0.859155           0.842857   \n",
       "\n",
       "   mean_test_score  std_test_score  rank_test_score  \n",
       "0         0.884064        0.033151                8  \n",
       "1         0.895372        0.037771                1  \n",
       "2         0.892555        0.036675                2  \n",
       "3         0.886841        0.040526                7  \n",
       "4         0.892515        0.045547                3  \n",
       "5         0.886881        0.044075                5  \n",
       "6         0.889698        0.036634                4  \n",
       "7         0.886881        0.037244                5  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "latter-insurance",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}