# ===== v2: richer features + CatBoost + stacking =====
"""DHDL Loan Default v2 — richer features + LightGBM/XGBoost/CatBoost, saved OOF for stacking.
Usage: python v2_train.py <model> [seed]   model in {lgb, lgb127, xgb, cb}
v1 files are never touched; outputs go to /tmp/ld/v2_*.npy"""
import pandas as pd, numpy as np, json, sys, time
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, roc_auc_score
import lightgbm as lgb, xgboost as xgb
from catboost import CatBoostClassifier

U = "/tmp/ld/"
which = sys.argv[1]; SEED = int(sys.argv[2]) if len(sys.argv) > 2 else 42
tr = pd.read_csv(U + "Train_set..csv"); te = pd.read_csv(U + "Test_set..csv")
y = tr["default"].values
CATS = ["loan_grade", "home_ownership", "income_verification_status", "loan_purpose", "state_code", "application_type", "loan_subgrade"]

def fe(d):
    d = d.copy()
    d["term_m"] = d["loan_term"].str.extract(r"(\d+)").astype(float)[0] * 12
    d["job_exp_num"] = d["job_experience"].map({"<5 Years": 2.5, "6-10 years": 8, "10+ years": 12})
    d["job_exp_missing"] = d["job_experience"].isna().astype(int)
    d["bal_missing"] = d["total_current_balance"].isna().astype(int)
    d["lwp_missing"] = d["last_week_pay"].isna().astype(int)
    d["subgrade_num"] = d["loan_subgrade"].map(lambda s: (ord(s[0]) - 65) * 5 + int(s[1]) if isinstance(s, str) else np.nan)
    d["int_per_loan"] = d["interest_receive"] / d["loan_amnt"]
    d["pay_progress"] = d["last_week_pay"] / d["term_m"]
    d["exp_interest"] = d["loan_amnt"] * d["interest_rate"] / 100 / 12 * d["last_week_pay"].fillna(0) / 4.33
    d["int_gap"] = d["interest_receive"] - d["exp_interest"]
    d["int_per_week"] = d["interest_receive"] / (d["last_week_pay"] + 1)
    d["revol_util"] = d["revolving_balance"] / (d["total_revolving_limit"] + 1)
    d["loan_to_income"] = d["loan_amnt"] / (d["annual_income"] + 1)
    d["installment"] = d["loan_amnt"] * (d["interest_rate"]/1200) / (1 - (1 + d["interest_rate"]/1200) ** (-d["term_m"]))
    d["inst_to_income"] = d["installment"] * 12 / (d["annual_income"] + 1)
    # ---- v2 additions (target-free) ----
    d["int_ratio_expected"] = d["interest_receive"] / (d["exp_interest"] + 1)
    d["total_expected_int"] = d["installment"] * d["term_m"] - d["loan_amnt"]
    d["int_share_received"] = d["interest_receive"] / (d["total_expected_int"].abs() + 1)
    d["weeks_left"] = d["term_m"] * 4.33 - d["last_week_pay"]
    d["bal_to_income"] = d["total_current_balance"] / (d["annual_income"] + 1)
    d["revol_to_loan"] = d["revolving_balance"] / (d["loan_amnt"] + 1)
    d["rate_x_progress"] = d["interest_rate"] * d["pay_progress"]
    return d

full = pd.concat([fe(tr), fe(te)], ignore_index=True)
# group comparisons: how each loan compares with peers of the same sub-grade / state (train+test features, no target)
for g in ["loan_subgrade", "state_code", "loan_purpose"]:
    for c in ["int_per_loan", "pay_progress", "interest_rate", "loan_to_income"]:
        full[f"{c}_vs_{g}"] = full[c] - full.groupby(g)[c].transform("mean")
    full[f"freq_{g}"] = full.groupby(g)[g].transform("count")
for c in CATS: full[c] = full[c].fillna("NA").astype("category")
full = full.drop(columns=["ID", "loan_term", "job_experience", "default"], errors="ignore")
X = full.iloc[:len(tr)].reset_index(drop=True); XT = full.iloc[len(tr):].reset_index(drop=True)
FEATS = list(X.columns)
skf = StratifiedKFold(5, shuffle=True, random_state=SEED)

def lgb_fp(leaves):
    P = dict(n_estimators=5000, learning_rate=0.03, num_leaves=leaves, n_jobs=1, min_child_samples=20, subsample=0.8,
             subsample_freq=1, colsample_bytree=0.6, reg_lambda=2.0, random_state=SEED, verbose=-1)
    def f(Xa, ya, Xb, yb):
        m = lgb.LGBMClassifier(**P).fit(Xa, ya, eval_set=[(Xb, yb)], callbacks=[lgb.early_stopping(200, verbose=False)])
        return m.predict_proba(Xb)[:, 1], m.predict_proba(XT)[:, 1]
    return f
def xgb_fp(Xa, ya, Xb, yb):
    m = xgb.XGBClassifier(n_estimators=5000, learning_rate=0.04, max_depth=7, n_jobs=1, subsample=0.8, colsample_bytree=0.6,
                          min_child_weight=2, enable_categorical=True, tree_method="hist", early_stopping_rounds=200,
                          random_state=SEED, eval_metric="logloss")
    m.fit(Xa, ya, eval_set=[(Xb, yb)], verbose=False)
    return m.predict_proba(Xb)[:, 1], m.predict_proba(XT)[:, 1]
def cbp(d):
    d = d.copy()
    for c in CATS: d[c] = d[c].astype(str)
    return d
def cb_fp(Xa, ya, Xb, yb):
    m = CatBoostClassifier(iterations=1500, learning_rate=0.08, depth=6, random_seed=SEED, verbose=0, od_type="Iter",
                           od_wait=100, cat_features=CATS, thread_count=4)
    m.fit(cbp(Xa), ya, eval_set=(cbp(Xb), yb))
    return m.predict_proba(cbp(Xb))[:, 1], m.predict_proba(cbp(XT))[:, 1]

fp = dict(lgb=lgb_fp(63), lgb127=lgb_fp(127), xgb=xgb_fp, cb=cb_fp)[which]
oof = np.zeros(len(X)); tp = np.zeros(len(XT)); t = time.time()
for a, b in skf.split(X, y):
    pv, pt = fp(X.iloc[a], y[a], X.iloc[b], y[b]); oof[b] = pv; tp += pt / 5
    print(which, SEED, "fold", round(time.time() - t), flush=True)
print(f"{which} s{SEED}: acc@.5={accuracy_score(y, oof > .5):.5f} auc={roc_auc_score(y, oof):.5f} {time.time()-t:.0f}s", flush=True)
np.save(f"{U}v2_{which}_s{SEED}_oof.npy", oof); np.save(f"{U}v2_{which}_s{SEED}_test.npy", tp)

"""v2 stacking + honest (nested) threshold + v2 submission/evidence. Never overwrites v1 files."""
import json, glob, os, numpy as np, pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix

D = "/tmp/ld/"; V1 = 89.60
tr = pd.read_csv(D + "Train_set..csv"); te = pd.read_csv(D + "Test_set..csv"); ss = pd.read_csv(D + "Sample_Submission..csv")
y = tr["default"].values
NAMES = {"lgb": "LightGBM v2 · 63 leaves", "lgb127": "LightGBM v2 · 127 leaves", "xgb": "XGBoost v2", "cb": "CatBoost v2"}
oof, tst = {}, {}
for k in NAMES:
    f = sorted(glob.glob(f"{D}v2_{k}_s*_oof.npy"))
    if not f: continue
    oof[k] = np.mean([np.load(x) for x in f], 0); tst[k] = np.mean([np.load(x.replace("_oof", "_test")) for x in f], 0)
keys = list(oof)
TS = np.arange(0.30, 0.71, 0.005)
def thr(p, yy): a = [accuracy_score(yy, p > t) for t in TS]; i = int(np.argmax(a)); return float(TS[i]), a[i]

def logit(p): p = np.clip(p, 1e-6, 1 - 1e-6); return np.log(p / (1 - p))
Z = np.column_stack([logit(oof[k]) for k in keys]); ZT = np.column_stack([logit(tst[k]) for k in keys])
cands = {k: (oof[k], tst[k]) for k in keys}
cands["average"] = (np.mean([oof[k] for k in keys], 0), np.mean([tst[k] for k in keys], 0))
# stacked meta-model: out-of-fold meta predictions (second layer, different split)
sk2 = StratifiedKFold(5, shuffle=True, random_state=2026); meta = np.zeros(len(y))
for a, b in sk2.split(Z, y):
    meta[b] = LogisticRegression(C=1.0, max_iter=1000).fit(Z[a], y[a]).predict_proba(Z[b])[:, 1]
cands["stack"] = (meta, LogisticRegression(C=1.0, max_iter=1000).fit(Z, y).predict_proba(ZT)[:, 1])

rows = []
for k, (p, _) in cands.items():
    t, a = thr(p, y)
    # nested: choose threshold on 4/5 of rows, score on the held-out 1/5
    nest = []
    for a_, b_ in StratifiedKFold(5, shuffle=True, random_state=7).split(p.reshape(-1, 1), y):
        tt, _ = thr(p[a_], y[a_]); nest.append(accuracy_score(y[b_], p[b_] > tt))
    rows.append(dict(id=k, name=NAMES.get(k, "Simple average" if k == "average" else "Stacked (logistic meta-model)"),
                     acc05=round(accuracy_score(y, p > .5) * 100, 2), auc=round(roc_auc_score(y, p), 4),
                     threshold=round(t, 3), accTuned=round(a * 100, 2), accNested=round(float(np.mean(nest)) * 100, 2)))
rows.sort(key=lambda r: -r["accNested"]); print(pd.DataFrame(rows).to_string())
best = rows[0]; p, pt = cands[best["id"]]; t = best["threshold"]
pred = (p > t).astype(int); tn, fp, fn, tp = confusion_matrix(y, pred).ravel()
gain = round(best["accTuned"] - V1, 2); promoted = gain >= 0.15
sub = pd.DataFrame({"ID": te["ID"], "default": (pt > t).astype(int)})
assert len(sub) == 39933 and set(sub["default"].unique()) <= {0, 1} and (sub["ID"].values == ss["ID"].values).all()
sub.to_csv(D + "DHDL_Loan_Default_Submission_v2.csv", index=False)
v1 = pd.read_csv("/dev-server/public/submission/DHDL_Loan_Default_Submission.csv")
agree = float((v1["default"].values == sub["default"].values).mean())
ev = dict(version="v2", v1Accuracy=V1, models=rows, final=dict(id=best["id"], name=best["name"], threshold=t,
          cvAccuracy=best["accTuned"], nestedAccuracy=best["accNested"], auc=best["auc"], gainVsV1=gain, promoted=promoted,
          confusion=dict(tn=int(tn), fp=int(fp), fn=int(fn), tp=int(tp)), testDefaultRate=round(float(sub["default"].mean()) * 100, 2),
          agreementWithV1=round(agree * 100, 2), rows=len(sub)),
          seeds=sorted({int(x.split("_s")[-1].split("_")[0]) for x in glob.glob(D + "v2_*_oof.npy")}),
          newFeatures=["interest received ÷ expected to date", "share of total lifetime interest received", "weeks left on loan",
                       "balance ÷ income", "revolving balance ÷ loan", "rate × repayment progress",
                       "peer comparisons within sub-grade, state and purpose (12)", "category frequencies (3)"])
json.dump(ev, open("/dev-server/src/data/loan-default-v2-evidence.json", "w"), indent=2)
print(json.dumps(ev["final"], indent=1))
