fine_tuning_lgbm.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"fine_tuning_lgbm.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1qZgEcAAFSJwGs6cPhHr3p7DzCDJEJiAs","authorship_tag":"ABX9TyOfZRAV60DOKEK1NePQtoBV"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"bKAYNT9WkvY7"},"source":["import numpy as np\n","import pandas as pd\n","import joblib\n","\n","from sklearn.model_selection import KFold, StratifiedKFold, RandomizedSearchCV\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score\n","from lightgbm.sklearn import LGBMClassifier"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"vCeT7ks4DDtu"},"source":["## Load Clean Dataset"]},{"cell_type":"code","metadata":{"id":"qOY5jBJwk5u9"},"source":["PROJECT_DIR = '/content/drive/MyDrive/Colab Projects/product-pair-matching/'\n","DATA_DIR = PROJECT_DIR+'data/'\n","OUTPUTS_DIR = PROJECT_DIR+'outputs/'\n","\n","# load image feature\n","train_image_df = pd.read_csv(DATA_DIR+'clean/train_image_df.csv')\n","test_image_df = pd.read_csv(DATA_DIR+'clean/test_image_df.csv')\n","# load text feature\n","train_text_df = pd.read_csv(DATA_DIR+'clean/train_text_df.csv')\n","test_text_df = pd.read_csv(DATA_DIR+'clean/test_text_df.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Q4f1I6RfrzH3"},"source":["train_df = pd.concat([train_image_df.drop('Label', axis=1), train_text_df], axis=1, join='outer')\n","test_df = pd.concat([test_image_df, test_text_df], axis=1, join='outer')\n","\n","feat = []\n","for f in train_df.columns:\n","    if f != 'Label':\n","        feat.append(f)\n","\n","DEBUG = False\n","if DEBUG:\n","    train_df = train_df.head(100)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"R9b-THuLDNfm"},"source":["## Hyperparameter search using RandomizedSearchCV"]},{"cell_type":"code","metadata":{"id":"qgS9mIYj2Cwc"},"source":["def params_search(X, y, features, FOLD=5, RANDOM_STATE=42, N_ITER=50):\n","    lgbm_default = LGBMClassifier(\n","        boosting_type='gbdt', #'gbdt'\n","        n_estimators=500, #100\n","        objective='binary', #'binary' or 'multiclass'\n","        )\n","\n","    lgbm_params = {\n","        'num_leaves': [8, 16, 32, 64], #31\n","        'max_depth': [3, 7, 14, 21], #-1\n","        'learning_rate': [0.01, 0.1, 1], #0.1\n","        'min_data_in_leaf': [20, 40, 80], #20\n","        'min_sum_hessian_in_leaf': [1e-5, 1e-2, 1, 1e2, 1e4], #1e-3\n","        'bagging_fraction': [i / 10.0 for i in range(7, 11)], #0.7\n","        'bagging_freq': [0, 5, 10, 20, 30], #0\n","        'feature_fraction': [i / 10.0 for i in range(3, 7)], #1\n","        'lambda_l1': [0, 1e-3, 1e-1],\n","        'lambda_l2': [0, 1e-3, 1e-1]\n","        }\n","\n","    print('PARAMETER GRID')\n","    print(lgbm_params)\n","    lgbm_search = RandomizedSearchCV(\n","        estimator=lgbm_default,\n","        param_distributions=lgbm_params,\n","        scoring='f1_macro',\n","        # n_jobs=-1,\n","        pre_dispatch='2*n_jobs',\n","        cv=FOLD,\n","        verbose=1,\n","        random_state=RANDOM_STATE,\n","        n_iter=N_ITER\n","        )\n","    \n","    print()\n","    print('SEARCHING...')\n","    lgbm_search.fit(X=X[features], y=y)\n","    print('Best parameter: {}'.format(lgbm_search.best_params_))\n","    print('Best score: {}'.format(lgbm_search.best_score_))\n","\n","    print()\n","    print('FINAL FEATURE')\n","    feature_importance = pd.DataFrame(\n","        lgbm_search.best_estimator_.feature_importances_,\n","        index=feat,\n","        columns=['importance']\n","    ).sort_values('importance', ascending=False)\n","    final_features = feature_importance[feature_importance['importance'] > 0].index\n","    print('All feature: {}'.format(len(features)))\n","    print('CV feature: {}'.format(len(final_features)))\n","\n","    return lgbm_search.best_params_, final_features, lgbm_search.cv_results_"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iITXM9Zt6yuI","executionInfo":{"status":"ok","timestamp":1610805180280,"user_tz":-420,"elapsed":2385909,"user":{"displayName":"Rizky P. Nugroho","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhfGfTSVeninJ0B1l8mJI7zRStmfZQEXN4TN6EVgA=s64","userId":"11961087904784672174"}},"outputId":"a0dd8f7f-3614-4888-991c-19351371d31a"},"source":["best_params, final_features, cv_result = params_search(train_df, train_df['Label'], feat, FOLD=5, N_ITER=20)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["PARAMETER GRID\n","{'num_leaves': [8, 16, 32, 64], 'max_depth': [3, 7, 14, 21], 'learning_rate': [0.01, 0.1, 1], 'min_data_in_leaf': [20, 40, 80], 'min_sum_hessian_in_leaf': [1e-05, 0.01, 1, 100.0, 10000.0], 'bagging_fraction': [0.7, 0.8, 0.9, 1.0], 'bagging_freq': [0, 5, 10, 20, 30], 'feature_fraction': [0.3, 0.4, 0.5, 0.6], 'lambda_l1': [0, 0.001, 0.1], 'lambda_l2': [0, 0.001, 0.1]}\n","\n","SEARCHING...\n","Fitting 5 folds for each of 20 candidates, totalling 100 fits\n"],"name":"stdout"},{"output_type":"stream","text":["[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n","[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed: 38.7min finished\n"],"name":"stderr"},{"output_type":"stream","text":["Best parameter: {'num_leaves': 32, 'min_sum_hessian_in_leaf': 1e-05, 'min_data_in_leaf': 80, 'max_depth': 14, 'learning_rate': 0.1, 'lambda_l2': 0, 'lambda_l1': 0.001, 'feature_fraction': 0.4, 'bagging_freq': 20, 'bagging_fraction': 1.0}\n","Best score: 0.8540954492587722\n","\n","FINAL FEATURE\n","All feature: 1331\n","CV feature: 1327\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6Gtn3PDG6KQI","executionInfo":{"status":"ok","timestamp":1610812552725,"user_tz":-420,"elapsed":1107,"user":{"displayName":"Rizky P. Nugroho","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhfGfTSVeninJ0B1l8mJI7zRStmfZQEXN4TN6EVgA=s64","userId":"11961087904784672174"}},"outputId":"fd622068-f7b0-4465-f47b-0d2c933eec19"},"source":["joblib.dump((best_params, final_features, cv_result), OUTPUTS_DIR+'models/randomsearch_result.pkl')"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Colab Projects/product-pair-matching/outputs/models/randomsearch_result.pkl']"]},"metadata":{"tags":[]},"execution_count":66}]},{"cell_type":"markdown","metadata":{"id":"IulyznzsDXDO"},"source":["## Train Model"]},{"cell_type":"markdown","metadata":{"id":"r3CdU40DDbSQ"},"source":["### Cross Validation"]},{"cell_type":"code","metadata":{"id":"Z2f8pImH8Gbc"},"source":["def lgb_f1_score(y_true, y_hat):\n","    y_hat = np.round(y_hat) # scikits f1 doesn't like probabilities\n","    return 'f1_macro', f1_score(y_true, y_hat, 'macro'), True"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"30DeGSCr2r5n"},"source":["def cv_model(X, y, feat, params, stratified=False):\n","    \n","    classifier = []\n","        \n","    if stratified:\n","        folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n","    else:\n","        folds = KFold(n_splits=5, shuffle=True, random_state=42)\n","        \n","    oof = np.zeros(len(X))\n","    for fold_, (trn_idx, val_idx) in enumerate(folds.split(X[feat].values, y)):\n","        X_train = X[feat].iloc[trn_idx]\n","        y_train = y.iloc[trn_idx]\n","        X_valid = X[feat].iloc[val_idx]\n","        y_valid = y.iloc[val_idx]\n","\n","        print(\"Fold n°{}\".format(fold_))\n","        len_train = len(X.iloc[trn_idx])\n","        print('Train: {0} data, 0: {1}, 1: {2}'.format(len_train,\n","                                                       len_train-np.sum(y[trn_idx]),\n","                                                       np.sum(y[trn_idx]))) \n","        len_valid = len(X.iloc[val_idx])\n","        print('Valid: {0} data, 0: {1}, 1: {2}'.format(len_valid,\n","                                                       len_valid-np.sum(y[val_idx]),\n","                                                       np.sum(y[val_idx])))\n","        \n","        clf = LGBMClassifier(**params) \n","        clf.fit(X_train, y_train,\n","                eval_set=[(X_train, y_train), (X_valid, y_valid)],\n","                eval_names=['train', 'valid'],\n","                eval_metric=lgb_f1_score,\n","                early_stopping_rounds=250,\n","                verbose=100\n","                )\n","\n","        \n","        oof[val_idx] = clf.predict(X_valid, num_iteration=clf.best_iteration_)\n","        classifier.append(clf)\n","        print()\n","\n","    print('OOF:', f1_score(y, np.round(oof)))\n","    return classifier, oof"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7EbipA71UGxX"},"source":["params = best_params\n","params['objective']='binary'\n","params['metric']='f1_macro'\n","params['n_estimators']=1000\n","params['is_unbalance']=True"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ujGyIDnJgHme","executionInfo":{"status":"ok","timestamp":1610811702098,"user_tz":-420,"elapsed":330837,"user":{"displayName":"Rizky P. Nugroho","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhfGfTSVeninJ0B1l8mJI7zRStmfZQEXN4TN6EVgA=s64","userId":"11961087904784672174"}},"outputId":"5aa3b981-fd29-4de5-c870-0579f9e66bd6"},"source":["cv_classifier, oof = cv_model(train_df, train_df['Label'], final_features, params, stratified=False)\n","print('Save model...')\n","joblib.dump(cv_classifier, OUTPUTS_DIR+'models/cv_classifier.pkl')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Fold n°0\n","Train: 8144 data, 0: 3481, 1: 4663\n","Valid: 2037 data, 0: 856, 1: 1181\n","Training until validation scores don't improve for 250 rounds.\n","[100]\ttrain's f1_macro: 0.974956\tvalid's f1_macro: 0.856626\n","[200]\ttrain's f1_macro: 0.999678\tvalid's f1_macro: 0.866282\n","[300]\ttrain's f1_macro: 0.999785\tvalid's f1_macro: 0.869951\n","[400]\ttrain's f1_macro: 0.999785\tvalid's f1_macro: 0.871068\n","Early stopping, best iteration is:\n","[204]\ttrain's f1_macro: 0.999785\tvalid's f1_macro: 0.867556\n","\n","Fold n°1\n","Train: 8145 data, 0: 3471, 1: 4674\n","Valid: 2036 data, 0: 866, 1: 1170\n","Training until validation scores don't improve for 250 rounds.\n","[100]\ttrain's f1_macro: 0.974808\tvalid's f1_macro: 0.848428\n","[200]\ttrain's f1_macro: 0.999572\tvalid's f1_macro: 0.873511\n","[300]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.876026\n","[400]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.873175\n","[500]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.875965\n","Early stopping, best iteration is:\n","[294]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.876538\n","\n","Fold n°2\n","Train: 8145 data, 0: 3439, 1: 4706\n","Valid: 2036 data, 0: 898, 1: 1138\n","Training until validation scores don't improve for 250 rounds.\n","[100]\ttrain's f1_macro: 0.97731\tvalid's f1_macro: 0.856743\n","[200]\ttrain's f1_macro: 0.999362\tvalid's f1_macro: 0.867216\n","[300]\ttrain's f1_macro: 0.999787\tvalid's f1_macro: 0.864522\n","[400]\ttrain's f1_macro: 0.999894\tvalid's f1_macro: 0.872072\n","[500]\ttrain's f1_macro: 0.999894\tvalid's f1_macro: 0.874439\n","Early stopping, best iteration is:\n","[317]\ttrain's f1_macro: 0.999894\tvalid's f1_macro: 0.869801\n","\n","Fold n°3\n","Train: 8145 data, 0: 3480, 1: 4665\n","Valid: 2036 data, 0: 857, 1: 1179\n","Training until validation scores don't improve for 250 rounds.\n","[100]\ttrain's f1_macro: 0.977656\tvalid's f1_macro: 0.844344\n","[200]\ttrain's f1_macro: 0.999678\tvalid's f1_macro: 0.856007\n","[300]\ttrain's f1_macro: 0.999678\tvalid's f1_macro: 0.855124\n","[400]\ttrain's f1_macro: 0.999786\tvalid's f1_macro: 0.857772\n","[500]\ttrain's f1_macro: 0.999786\tvalid's f1_macro: 0.858648\n","Early stopping, best iteration is:\n","[301]\ttrain's f1_macro: 0.999786\tvalid's f1_macro: 0.855502\n","\n","Fold n°4\n","Train: 8145 data, 0: 3477, 1: 4668\n","Valid: 2036 data, 0: 860, 1: 1176\n","Training until validation scores don't improve for 250 rounds.\n","[100]\ttrain's f1_macro: 0.977009\tvalid's f1_macro: 0.849203\n","[200]\ttrain's f1_macro: 0.99925\tvalid's f1_macro: 0.853047\n","[300]\ttrain's f1_macro: 0.999786\tvalid's f1_macro: 0.854083\n","[400]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.855491\n","[500]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.860692\n","Early stopping, best iteration is:\n","[304]\ttrain's f1_macro: 0.999893\tvalid's f1_macro: 0.857143\n","\n","OOF: 0.8652279036446674\n","Save model...\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Colab Projects/product-pair-matching/outputs/models/cv_classifier.pkl']"]},"metadata":{"tags":[]},"execution_count":56}]},{"cell_type":"markdown","metadata":{"id":"B8BhYh7NDetI"},"source":["### All Data"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cv5BuzqzkAIg","executionInfo":{"status":"ok","timestamp":1610811820853,"user_tz":-420,"elapsed":446693,"user":{"displayName":"Rizky P. Nugroho","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhfGfTSVeninJ0B1l8mJI7zRStmfZQEXN4TN6EVgA=s64","userId":"11961087904784672174"}},"outputId":"cb08c917-e5d3-4bb2-a926-c868c9fa22fb"},"source":["print('TRAIN CLASSIFIER WITH ALL DATA')\n","all_classifier = LGBMClassifier(**params)\n","all_classifier.fit(train_df[final_features], train_df['Label'])\n","val_preds = all_classifier.predict(train_df[final_features])\n","print('F1: {}'.format(f1_score(train_df['Label'], val_preds)))\n","\n","print('Save model...')\n","joblib.dump(all_classifier, OUTPUTS_DIR+'models/all_classifier.pkl')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["TRAIN CLASSIFIER WITH ALL DATA\n","F1: 0.999828855040219\n","Save model...\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Colab Projects/product-pair-matching/outputs/models/all_classifier.pkl']"]},"metadata":{"tags":[]},"execution_count":57}]},{"cell_type":"markdown","metadata":{"id":"vicsnSNmDgyE"},"source":["### Combine Model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3CeQLi_kjR9K","executionInfo":{"status":"ok","timestamp":1610811821814,"user_tz":-420,"elapsed":444865,"user":{"displayName":"Rizky P. Nugroho","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhfGfTSVeninJ0B1l8mJI7zRStmfZQEXN4TN6EVgA=s64","userId":"11961087904784672174"}},"outputId":"772d57a9-2c25-435c-cf5a-1957f08671f7"},"source":["print('ENSEMBLING MODEL')\n","list_model = cv_classifier + [all_classifier]*5\n","print('Save model...')\n","joblib.dump(list_model, OUTPUTS_DIR+'models/cv_and_all_classifier.pkl')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["ENSEMBLING MODEL\n","Save model...\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Colab Projects/product-pair-matching/outputs/models/cv_and_all_classifier.pkl']"]},"metadata":{"tags":[]},"execution_count":58}]},{"cell_type":"markdown","metadata":{"id":"W7v7jytNDjN7"},"source":["## Predict Test Data"]},{"cell_type":"code","metadata":{"id":"zCMaS2wL6kbN"},"source":["def create_submission(temp_df, list_model, feat, csv=False, name='test.csv'):\n","    df = temp_df.copy()\n","    df['sum_na'] = df.apply(lambda x: sum(x.isnull().values), axis=1)\n","    preds = []\n","    for model in list_model:\n","        y_hat = model.predict(df[feat])\n","        preds.append(y_hat)\n","        print(y_hat)\n","    \n","    df['label'] = np.round(np.mean(preds, axis=0))\n","    submission = pd.DataFrame()\n","    submission['label'] = df['label'].astype(int) \n","    if csv:\n","        submission['label'].to_csv(name, index=True, index_label='pair_index')\n","    return submission[['label']]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Lew-rOZk7CRg","colab":{"base_uri":"https://localhost:8080/","height":374},"executionInfo":{"status":"ok","timestamp":1610812105517,"user_tz":-420,"elapsed":138846,"user":{"displayName":"Rizky P. Nugroho","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhfGfTSVeninJ0B1l8mJI7zRStmfZQEXN4TN6EVgA=s64","userId":"11961087904784672174"}},"outputId":"89955bd9-6d42-4000-ce00-84a7f1dda34e"},"source":["# Predict using all model and use average result\n","sub_df = create_submission(test_df, list_model, final_features, csv=True, name='fine_tuning_all.csv')\n","sub_df.head()\n","#0.85473"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n","[0 0 0 ... 0 1 0]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   label\n","0      0\n","1      0\n","2      0\n","3      1\n","4      1"]},"metadata":{"tags":[]},"execution_count":61}]},{"cell_type":"code","metadata":{"id":"NPRb5U1_Ipyo"},"source":["# We only got 0.85.. (ranked 16 in Private LB (late submission))\n","# Perhaps we have problem in feature extraction\n","# Suggestion:\n","# - Extract image feature with more simple pretrained model\n","#   and only use a few first layer\n","# - Extract text feature using smaller embedding size (maybe 64)\n","# Or we have silent bug (data maybe shuffled)"],"execution_count":null,"outputs":[]}]}