{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# %pip install awswrangler\n", "%pip install pycaret[full]\n", "%pip install xgboost\n", "# %pip install mlflow" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# import awswrangler as wr\n", "import pandas as pd\n", "import numpy as np\n", "# import boto3\n", "# from sagemaker import get_execution_role\n", "import datetime\n", "import string\n", "import random\n", "from pycaret.classification import *" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# data = pd.read_csv('data/data_feature_eng.csv')\n", "data = pd.read_csv('data/feature_ori.csv')\n", "# print(data)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
 DescriptionValue
0session_id4505
1Targetlabel
2Target TypeBinary
3Label Encoded0.0: 0, 1.0: 1
4Original Data(64800, 11)
5Missing ValuesFalse
6Numeric Features8
7Categorical Features2
8Ordinal FeaturesFalse
9High Cardinality FeaturesFalse
10High Cardinality MethodNone
11Transformed Train Set(45359, 19)
12Transformed Test Set(19441, 19)
13Shuffle Train-TestTrue
14Stratify Train-TestFalse
15Fold GeneratorStratifiedKFold
16Fold Number10
17CPU Jobs-1
18Use GPUTrue
19Log ExperimentFalse
20Experiment Nameclf-default-name
21USIc1be
22Imputation Typesimple
23Iterative Imputation IterationNone
24Numeric Imputermean
25Iterative Imputation Numeric ModelNone
26Categorical Imputerconstant
27Iterative Imputation Categorical ModelNone
28Unknown Categoricals Handlingleast_frequent
29NormalizeFalse
30Normalize MethodNone
31TransformationFalse
32Transformation MethodNone
33PCAFalse
34PCA MethodNone
35PCA ComponentsNone
36Ignore Low VarianceFalse
37Combine Rare LevelsFalse
38Rare Level ThresholdNone
39Numeric BinningFalse
40Remove OutliersFalse
41Outliers ThresholdNone
42Remove MulticollinearityFalse
43Multicollinearity ThresholdNone
44Remove Perfect CollinearityTrue
45ClusteringFalse
46Clustering IterationNone
47Polynomial FeaturesFalse
48Polynomial DegreeNone
49Trignometry FeaturesFalse
50Polynomial ThresholdNone
51Group FeaturesFalse
52Feature SelectionFalse
53Feature Selection Methodclassic
54Features Selection ThresholdNone
55Feature InteractionFalse
56Feature RatioFalse
57Interaction ThresholdNone
58Fix ImbalanceFalse
59Fix Imbalance MethodSMOTE
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "m_setup = setup(data=data, target='label', normalize=False, \n", " feature_interaction=False, \n", " feature_ratio=False,\n", " trigonometry_features=False,\n", " use_gpu=True)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
 AccuracyAUCRecallPrec.F1KappaMCC
Fold       
00.97770.99530.92580.92710.92640.91330.9133
10.97570.99460.92590.91520.92050.90620.9062
20.97970.99560.95200.91740.93440.92240.9226
30.98080.99630.94040.93360.93700.92570.9257
40.97730.99470.93310.91850.92570.91230.9124
50.97860.99480.93900.92150.93020.91750.9176
60.97690.99490.94330.90770.92520.91150.9117
70.98020.99390.95060.92110.93560.92390.9241
80.97640.99330.93170.91440.92300.90900.9091
90.97840.99480.94610.91420.92990.91710.9173
Mean0.97820.99480.93880.91910.92880.91590.9160
Std0.00160.00080.00900.00690.00530.00620.0062
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# max_depth = 트리 최대 깊이\n", "# max_leaves = 트리 최대 리프\n", "# subsample = row sampling\n", "# colsample_bytree = column sampling 각 이터레이션에 사용되는 칼럼의 비율\n", "# learning_rate = 러닝레이트\n", "# 일반적으로 row sampling 보다는 column sampling이 모형성능과 학습시간에 더 큰 영향을 준다\n", "# xgboost = create_model('xgboost', max_depth=16, max_leaves=255)\n", "xgboost = create_model('xgboost')\n", "# xgboost = create_model('xgboost', max_depth=8, max_leaves=256)\n", "# best_model = compare_models(n_select=6)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
 AccuracyAUCRecallPrec.F1KappaMCC
Fold       
00.97930.99550.93160.93160.93160.91940.9194
10.97620.99550.91130.93030.92070.90670.9068
20.98280.99580.95060.93700.94370.93360.9336
30.98130.99630.91720.95750.93690.92590.9262
40.97770.99610.91720.93480.92590.91280.9129
50.97880.99340.92010.93920.92950.91710.9171
60.98080.99630.94480.92990.93730.92600.9260
70.98240.99440.94040.94310.94180.93140.9314
80.97820.99450.92150.93370.92760.91470.9148
90.97860.99490.92720.93130.92920.91660.9167
Mean0.97960.99530.92820.93680.93240.92040.9205
Std0.00200.00090.01260.00800.00690.00810.0081
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# max leaves -> 2**(n-1), n = max_depth\n", "# params = {'max_depth': [128, 64, 32],\n", "# 'max_leaves': [256, 1024, 4096], \n", "# 'colsample_bytree':[0.2, 0.4, 0.6, 0.8, 1.0],\n", "# 'learning_rate':[0.05, 0.005]\n", "# }#range(14,17)}\n", "params = {'max_depth': [128],\n", " 'max_leaves': [1024], \n", " 'colsample_bytree':[0.6, 0.8],\n", " 'learning_rate':[0.005]\n", " }#range(14,17)}\n", "# tuned_xgboost = tune_model(xgboost, optimize='F1', custom_grid=params)\n", "tuned_xgboost = tune_model(xgboost, optimize='Accuracy', custom_grid=params, tuner_verbose=3)\n", "# tuned_xgboost = tune_model(xgboost, optimize='Kappa')" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n", " colsample_bynode=1, colsample_bytree=0.8,\n", " enable_categorical=False, gamma=0, gpu_id=0, importance_type=None,\n", " interaction_constraints='', learning_rate=0.005, max_delta_step=0,\n", " max_depth=128, max_leaves=1024, min_child_weight=1, missing=nan,\n", " monotone_constraints='()', n_estimators=100, n_jobs=-1,\n", " num_parallel_tree=1, objective='binary:logistic',\n", " predictor='auto', random_state=4505, reg_alpha=0, reg_lambda=1,\n", " scale_pos_weight=1, subsample=1, tree_method='gpu_hist',\n", " use_label_encoder=True, validate_parameters=1, ...)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tuned_xgboost" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "873563af42834355a9d06a1960767093", "version_major": 2, "version_minor": 0 }, "text/plain": [ "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# plot_model(tuned_dt, plot='auc')\n", "evaluate_model(tuned_xgboost)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Transformation Pipeline and Model Successfully Saved\n" ] }, { "data": { "text/plain": [ "(Pipeline(memory=None,\n", " steps=[('dtypes',\n", " DataTypes_Auto_infer(categorical_features=[],\n", " display_types=True, features_todrop=[],\n", " id_columns=[],\n", " ml_usecase='classification',\n", " numerical_features=[], target='label',\n", " time_features=[])),\n", " ('imputer',\n", " Simple_Imputer(categorical_strategy='not_available',\n", " fill_value_categorical=None,\n", " fill_value_numerical=None,\n", " numeric_strate...\n", " max_delta_step=0, max_depth=128, max_leaves=1024,\n", " min_child_weight=1, missing=nan,\n", " monotone_constraints='()', n_estimators=100,\n", " n_jobs=-1, num_parallel_tree=1,\n", " objective='binary:logistic', predictor='auto',\n", " random_state=4505, reg_alpha=0, reg_lambda=1,\n", " scale_pos_weight=1, subsample=1,\n", " tree_method='gpu_hist', use_label_encoder=True,\n", " validate_parameters=1, ...)]],\n", " verbose=False),\n", " 'tuned_xgboost_239_0408.pkl')" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mdl = finalize_model(tuned_xgboost)\n", "save_model(mdl, 'tuned_xgboost_239_0408')\n", "# save_model(tuned_xgboost, 'tuned_xgboost_0207')" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Transformation Pipeline and Model Successfully Loaded\n" ] }, { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
 ModelAccuracyAUCRecallPrec.F1KappaMCC
0Extreme Gradient Boosting0.99130.99850.96430.97780.97100.96590.9659
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 64800 entries, 0 to 64799\n", "Data columns (total 13 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 label 64800 non-null float64\n", " 1 hum 64800 non-null float64\n", " 2 temp 64800 non-null float64\n", " 3 door 64800 non-null float64\n", " 4 motion 64800 non-null float64\n", " 5 illum 64800 non-null float64\n", " 6 dayofweek 64800 non-null int64 \n", " 7 month 64800 non-null int64 \n", " 8 day 64800 non-null int64 \n", " 9 hour 64800 non-null int64 \n", " 10 minute 64800 non-null int64 \n", " 11 Label 64800 non-null float64\n", " 12 Score 64800 non-null float32\n", "dtypes: float32(1), float64(7), int64(5)\n", "memory usage: 6.2 MB\n" ] }, { "data": { "text/plain": [ "0.971" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "load_mdl = load_model('tuned_xgboost_239_0408')\n", "prediction = predict_model(load_mdl, data=data.iloc[:])\n", "prediction = prediction.astype({'Label':'float64'})\n", "prediction.info()\n", "from pycaret.utils import check_metric\n", "check_metric(prediction['Label'], prediction['label'], metric = 'F1')" ] } ], "metadata": { "interpreter": { "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" }, "kernelspec": { "display_name": "Python 3.8.10 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.12" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }