"""v9: same v6 features, blend weights (LightGBM .6 / CatBoost .4) and per-grade cut-offs, but final models are
refit on 100% of the training loans (v6 averaged five fold models trained on 80% each) with several seeds.
Tree count per model = best early-stopping round on one held-out 20% split x 1.15.
Usage: python v9_full_refit.py lgb SEED | cat SEED | blend"""
import sys, os, numpy as np, pandas as pd, warnings; warnings.filterwarnings('ignore')
sys.path.insert(0, os.getcwd())
from feats import load, build_all_v6, CATS
from sklearn.model_selection import train_test_split
tr, te = load(); X, Xt = build_all_v6(tr, te); y = tr.default.values
mode = sys.argv[1]
if mode == 'lgb':
    import lightgbm as lgb; s = int(sys.argv[2])
    kw = dict(learning_rate=0.02, num_leaves=63, min_child_samples=30, subsample=0.8, subsample_freq=1,
              colsample_bytree=0.6, reg_lambda=1, random_state=s, verbose=-1, n_jobs=1)
    a, b = train_test_split(np.arange(len(X)), test_size=.2, stratify=y, random_state=s)
    m = lgb.LGBMClassifier(n_estimators=5000, **kw).fit(X.iloc[a], y[a], eval_set=[(X.iloc[b], y[b])],
                                                         callbacks=[lgb.early_stopping(250, verbose=False)])
    it = int(m.best_iteration_ * 1.15); print('lgb', s, 'trees', it, flush=True)
    p = lgb.LGBMClassifier(n_estimators=it, **kw).fit(X, y).predict_proba(Xt)[:, 1]
    np.save(f'v9_lgb_{s}.npy', p)
elif mode == 'cat':
    from catboost import CatBoostClassifier; s = int(sys.argv[2])
    conv = lambda d: d.assign(**{c: d[c].astype(str) for c in CATS})
    kw = dict(learning_rate=0.05, depth=7, l2_leaf_reg=3, thread_count=1, random_seed=s, verbose=False, cat_features=CATS)
    a, b = train_test_split(np.arange(len(X)), test_size=.2, stratify=y, random_state=s)
    m = CatBoostClassifier(iterations=4000, od_type='Iter', od_wait=250, **kw)
    m.fit(conv(X.iloc[a]), y[a], eval_set=(conv(X.iloc[b]), y[b]), use_best_model=True)
    it = int(m.get_best_iteration() * 1.15); print('cat', s, 'trees', it, flush=True)
    p = CatBoostClassifier(iterations=it, **kw).fit(conv(X), y).predict_proba(conv(Xt))[:, 1]
    np.save(f'v9_cat_{s}.npy', p)
else:
    import glob
    L = np.mean([np.load(f) for f in sorted(glob.glob('v9_lgb_*.npy'))], 0)
    C = np.mean([np.load(f) for f in sorted(glob.glob('v9_cat_*.npy'))], 0)
    p = .6 * L + .4 * C
    th = te.loan_grade.map(lambda g: .56 if g in 'AB' else (.53 if g in 'CDE' else .49)).values
    sub = pd.DataFrame({'ID': te.ID, 'default': (p > th).astype(int)}); sub.to_csv('submission_v9.csv', index=False)
    v5 = pd.read_csv('/dev-server/public/submission/DHDL_Loan_Default_Submission_v5.csv')
    v6 = pd.read_csv('/dev-server/public/submission/DHDL_Loan_Default_Submission_v6.csv')
    print({'n_lgb': len(glob.glob('v9_lgb_*.npy')), 'n_cat': len(glob.glob('v9_cat_*.npy')),
           'rate': round(sub.default.mean() * 100, 2), 'agree_v5': round((sub.default == v5.default).mean() * 100, 2),
           'agree_v6': round((sub.default == v6.default).mean() * 100, 2), 'rows': len(sub)})
