"""v10 = v9 (full-refit LightGBM x8 + CatBoost x4, plus full-refit XGBoost) + the v8 near-cut-off specialist,
the only v8 idea that gained on all three splits (+0.045 / +0.003 / +0.014).
Blend weights, per-grade cut-offs and the specialist are fitted on out-of-fold predictions (pred_*_2026.npy from
v8_solution.py); they are applied to the full-refit test predictions (v9_*.npy)."""
import glob, itertools, json, numpy as np, pandas as pd
from sklearn.linear_model import LogisticRegression
tr = pd.read_csv('train.csv'); te = pd.read_csv('test.csv'); y = tr.default.values; n = len(tr)
L = lambda k: np.load(f'pred_{k}_2026.npy')[:n]
oof = [np.mean([L(f'lgb_s{i}') for i in range(1, 6)], 0), L('xgb'), L('cat')]
tst = [np.mean([np.load(f) for f in glob.glob('v9_lgb_*.npy')], 0), np.load('v9_xgb_1.npy'),
       np.mean([np.load(f) for f in glob.glob('v9_cat_*.npy')], 0)]
grp = lambda s: s.map(lambda g: 'AB' if g in 'AB' else ('CDE' if g in 'CDE' else 'FG')).values
gtr, gte = grp(tr.loan_grade), grp(te.loan_grade)
T = np.round(np.arange(.40, .61, .01), 2)
lo = lambda v: np.log(np.clip(v, 1e-6, 1 - 1e-6) / (1 - np.clip(v, 1e-6, 1 - 1e-6)))
W = [w for w in itertools.product(*[np.round(np.arange(0, 1.01, .1), 1)] * 3) if abs(sum(w) - 1) < 1e-9]
best = (-1,)
for w in W:
    p = sum(a * c for a, c in zip(w, oof)); acc = ((p[:, None] > T) == y[:, None]).mean(0)
    if acc.max() > best[0]: best = (acc.max(), w, T[acc.argmax()])
_, w, t = best
po = sum(a * c for a, c in zip(w, oof)); pt = sum(a * c for a, c in zip(w, tst))
th = {g: max(T, key=lambda tt: ((po[gtr == g] > tt) == y[gtr == g]).mean() - 1e-4 * abs(tt - t)) for g in ('AB', 'CDE', 'FG')}
pred_o = np.array([po[i] > th[gtr[i]] for i in range(n)]); pred_t = np.array([pt[i] > th[gte[i]] for i in range(len(te))])
cats = ['AB', 'CDE', 'FG']
def feats(comps, p, g):
    G = np.c_[[g == c for c in cats]].T.astype(float)
    return np.c_[np.c_[[lo(c) for c in comps]].T, G, lo(p)[:, None] * G]
near_o = np.abs(po - .5) < .2; near_t = np.abs(pt - .5) < .2
lr = LogisticRegression(C=0.5, max_iter=2000).fit(feats(oof, po, gtr)[near_o], y[near_o])
spec_o = pred_o.copy(); spec_o[near_o] = lr.predict_proba(feats(oof, po, gtr)[near_o])[:, 1] > .5
spec_t = pred_t.copy(); spec_t[near_t] = lr.predict_proba(feats(tst, pt, gte)[near_t])[:, 1] > .5
sub = pd.DataFrame({'ID': te.ID, 'default': spec_t.astype(int)}); sub.to_csv('submission_v10.csv', index=False)
v9 = pd.read_csv('/dev-server/public/submission/DHDL_Loan_Default_Submission_v9.csv')
S = {'weights': list(w), 'threshold': float(t), 'grade_thresholds': {k: float(v) for k, v in th.items()},
     'oof_acc_blend': round(100 * (pred_o == y).mean(), 3), 'oof_acc_with_specialist_in_sample': round(100 * (spec_o == y).mean(), 3),
     'test_flipped_by_specialist': int((spec_t != pred_t).sum()), 'agree_v9': round(100 * (sub.default.values == v9.default.values).mean(), 2),
     'test_default_rate': round(100 * sub.default.mean(), 2), 'rows': len(sub)}
json.dump(S, open('summary_v10.json', 'w'), indent=1); print(S)
