"""DHDL Loan Default v3 — full solution: features + models (v3_train.py) and stacking (v3_stack.py).
Run: python v3_train.py <lgb|lgbw|xgb|cb|et|mlp> <seed>  for each model/seed, then python v3_stack.py"""
# ===== v3_train.py =====
"""DHDL Loan Default v3 — v2 features + repayment-schedule rebuild + in-fold target encoding + diverse models.
Usage: python v3_train.py <model> [seed]   model in {lgb, xgb, cb, et, mlp, lgbw}
v1/v2 files are never touched; outputs go to /tmp/ld/v3_*.npy"""
import sys, time, numpy as np, pandas as pd
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, roc_auc_score

U = "/tmp/ld/"
which = sys.argv[1]; SEED = int(sys.argv[2]) if len(sys.argv) > 2 else 42
tr = pd.read_csv(U + "Train_set..csv"); te = pd.read_csv(U + "Test_set..csv"); y = tr["default"].values
CATS = ["loan_grade", "home_ownership", "income_verification_status", "loan_purpose", "state_code", "application_type", "loan_subgrade"]

def fe(d):
    d = d.copy()
    d["term_m"] = d["loan_term"].str.extract(r"(\d+)").astype(float)[0] * 12
    d["job_exp_num"] = d["job_experience"].map({"<5 Years": 2.5, "6-10 years": 8, "10+ years": 12})
    d["job_exp_missing"] = d["job_experience"].isna().astype(int)
    d["bal_missing"] = d["total_current_balance"].isna().astype(int)
    d["lwp_missing"] = d["last_week_pay"].isna().astype(int)
    d["subgrade_num"] = d["loan_subgrade"].map(lambda s: (ord(s[0]) - 65) * 5 + int(s[1]) if isinstance(s, str) else np.nan)
    d["int_per_loan"] = d["interest_receive"] / d["loan_amnt"]
    d["pay_progress"] = d["last_week_pay"] / d["term_m"]
    d["exp_interest"] = d["loan_amnt"] * d["interest_rate"] / 100 / 12 * d["last_week_pay"].fillna(0) / 4.33
    d["int_gap"] = d["interest_receive"] - d["exp_interest"]
    d["int_per_week"] = d["interest_receive"] / (d["last_week_pay"] + 1)
    d["revol_util"] = d["revolving_balance"] / (d["total_revolving_limit"] + 1)
    d["loan_to_income"] = d["loan_amnt"] / (d["annual_income"] + 1)
    r = d["interest_rate"] / 1200
    d["installment"] = d["loan_amnt"] * r / (1 - (1 + r) ** (-d["term_m"]))
    d["inst_to_income"] = d["installment"] * 12 / (d["annual_income"] + 1)
    d["int_ratio_expected"] = d["interest_receive"] / (d["exp_interest"] + 1)
    d["total_expected_int"] = d["installment"] * d["term_m"] - d["loan_amnt"]
    d["int_share_received"] = d["interest_receive"] / (d["total_expected_int"].abs() + 1)
    d["weeks_left"] = d["term_m"] * 4.33 - d["last_week_pay"]
    d["bal_to_income"] = d["total_current_balance"] / (d["annual_income"] + 1)
    d["revol_to_loan"] = d["revolving_balance"] / (d["loan_amnt"] + 1)
    d["rate_x_progress"] = d["interest_rate"] * d["pay_progress"]
    # ---- v3: exact amortisation schedule rebuild (target-free) ----
    k = (d["last_week_pay"].fillna(0) / 4.33).clip(lower=0)            # months elapsed
    k = np.minimum(k, d["term_m"])
    g = (1 + r) ** k
    d["sched_balance"] = d["loan_amnt"] * g - d["installment"] * (g - 1) / r   # balance if every payment made
    d["sched_int_paid"] = d["installment"] * k - (d["loan_amnt"] - d["sched_balance"])
    d["int_vs_sched"] = d["interest_receive"] / (d["sched_int_paid"] + 1)
    d["int_minus_sched"] = d["interest_receive"] - d["sched_int_paid"]
    d["implied_months_paid"] = d["interest_receive"] / (d["loan_amnt"] * r + 1e-6)
    d["months_gap"] = k - d["implied_months_paid"]
    d["sched_bal_share"] = d["sched_balance"] / d["loan_amnt"]
    d["lwp_mod"] = d["last_week_pay"] % 4.33
    d["amnt_round"] = (d["loan_amnt"] % 1000 == 0).astype(int)
    return d

full = pd.concat([fe(tr), fe(te)], ignore_index=True)
for gcol in ["loan_subgrade", "state_code", "loan_purpose"]:
    for c in ["int_per_loan", "pay_progress", "interest_rate", "loan_to_income", "months_gap"]:
        full[f"{c}_vs_{gcol}"] = full[c] - full.groupby(gcol)[c].transform("mean")
    full[f"freq_{gcol}"] = full.groupby(gcol)[gcol].transform("count")
full["sg_term"] = full["loan_subgrade"].astype(str) + "_" + full["term_m"].astype(str)
full["freq_amnt"] = full.groupby("loan_amnt")["loan_amnt"].transform("count")
TE_COLS = ["loan_subgrade", "state_code", "loan_purpose", "sg_term"]
for c in CATS: full[c] = full[c].fillna("NA").astype("category")
full = full.drop(columns=["ID", "loan_term", "job_experience", "default"], errors="ignore")
X0 = full.iloc[:len(tr)].reset_index(drop=True); XT0 = full.iloc[len(tr):].reset_index(drop=True)

def add_te(Xa, ya, others, seed):
    """Smoothed target means: out-of-fold for training rows, full-train mapping for other frames. No leakage."""
    Xa = Xa.copy(); others = [o.copy() for o in others]; prior = ya.mean(); m = 50
    for c in TE_COLS:
        ka = Xa[c].astype(str).values; col = np.zeros(len(Xa))
        for a, b in StratifiedKFold(5, shuffle=True, random_state=seed).split(Xa, ya):
            s = pd.DataFrame({"k": ka[a], "y": ya[a]}).groupby("k")["y"].agg(["sum", "count"])
            mp = (s["sum"] + prior * m) / (s["count"] + m)
            col[b] = pd.Series(ka[b]).map(mp).fillna(prior).values
        Xa[f"te_{c}"] = col
        s = pd.DataFrame({"k": ka, "y": ya}).groupby("k")["y"].agg(["sum", "count"]); mp = (s["sum"] + prior * m) / (s["count"] + m)
        for o in others: o[f"te_{c}"] = o[c].astype(str).map(mp).astype(float).fillna(prior).values
    return (Xa, *others)

def prep(d, cat=True):
    d = d.drop(columns=["sg_term"])
    if cat: return d
    d = d.copy()
    for c in CATS: d[c] = d[c].cat.codes
    return d

def fit_predict(Xa, ya, Xb, yb, XT):
    if which in ("lgb", "lgbw"):
        import lightgbm as lgb
        P = dict(n_estimators=6000, learning_rate=0.02, num_leaves=127, min_child_samples=30, subsample=0.8, subsample_freq=1,
                 colsample_bytree=0.5, reg_lambda=3.0, n_jobs=4, random_state=SEED, verbose=-1)
        if which == "lgbw": P.update(num_leaves=255, max_depth=12, colsample_bytree=0.35, min_child_samples=15, learning_rate=0.015)
        m = lgb.LGBMClassifier(**P).fit(prep(Xa), ya, eval_set=[(prep(Xb), yb)], callbacks=[lgb.early_stopping(300, verbose=False)])
        return m.predict_proba(prep(Xb))[:, 1], m.predict_proba(prep(XT))[:, 1]
    if which == "xgb":
        import xgboost as xgb
        m = xgb.XGBClassifier(n_estimators=6000, learning_rate=0.03, max_depth=8, n_jobs=4, subsample=0.8, colsample_bytree=0.5,
                              min_child_weight=3, reg_lambda=3, enable_categorical=True, tree_method="hist", early_stopping_rounds=300,
                              random_state=SEED, eval_metric="logloss", max_cat_to_onehot=1)
        m.fit(prep(Xa), ya, eval_set=[(prep(Xb), yb)], verbose=False)
        return m.predict_proba(prep(Xb))[:, 1], m.predict_proba(prep(XT))[:, 1]
    if which == "cb":
        from catboost import CatBoostClassifier
        def c(d):
            d = prep(d).copy()
            for k in CATS: d[k] = d[k].astype(str)
            return d
        m = CatBoostClassifier(iterations=3000, learning_rate=0.06, depth=7, l2_leaf_reg=5, random_seed=SEED, verbose=0,
                               od_type="Iter", od_wait=150, cat_features=CATS, thread_count=4)
        m.fit(c(Xa), ya, eval_set=(c(Xb), yb))
        return m.predict_proba(c(Xb))[:, 1], m.predict_proba(c(XT))[:, 1]
    if which == "et":
        from sklearn.ensemble import ExtraTreesClassifier
        f = lambda d: prep(d, False).fillna(-999)
        m = ExtraTreesClassifier(500, min_samples_leaf=3, max_features=0.4, n_jobs=4, random_state=SEED).fit(f(Xa), ya)
        return m.predict_proba(f(Xb))[:, 1], m.predict_proba(f(XT))[:, 1]
    if which == "mlp":
        from sklearn.neural_network import MLPClassifier
        from sklearn.preprocessing import QuantileTransformer
        f = lambda d: prep(d, False).fillna(prep(Xa, False).median(numeric_only=True))
        q = QuantileTransformer(output_distribution="normal", random_state=SEED).fit(f(Xa))
        m = MLPClassifier((256, 128, 64), alpha=1e-3, batch_size=512, learning_rate_init=1e-3, max_iter=60, early_stopping=True,
                          n_iter_no_change=6, random_state=SEED).fit(q.transform(f(Xa)), ya)
        return m.predict_proba(q.transform(f(Xb)))[:, 1], m.predict_proba(q.transform(f(XT)))[:, 1]

oof = np.zeros(len(X0)); tp = np.zeros(len(XT0)); t = time.time()
for a, b in StratifiedKFold(5, shuffle=True, random_state=SEED).split(X0, y):
    Xa, Xb, XT = add_te(X0.iloc[a].reset_index(drop=True), y[a], [X0.iloc[b].reset_index(drop=True), XT0], SEED)
    pv, pt = fit_predict(Xa, y[a], Xb, y[b], XT); oof[b] = pv; tp += pt / 5
    print(which, SEED, "fold", round(time.time() - t), flush=True)
print(f"{which} s{SEED}: acc@.5={accuracy_score(y, oof > .5):.5f} auc={roc_auc_score(y, oof):.5f} {time.time()-t:.0f}s", flush=True)
np.save(f"{U}v3_{which}_s{SEED}_oof.npy", oof); np.save(f"{U}v3_{which}_s{SEED}_test.npy", tp)

# ===== v3_stack.py =====
"""v3 stacking (logistic + LightGBM meta, hill-climb weights), nested cut-off, decision rule vs v2. Never overwrites v1/v2."""
import json, glob, numpy as np, pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix

D = "/tmp/ld/"; R = "/dev-server/"
V2 = json.load(open(R + "src/data/loan-default-v2-evidence.json")); V2ACC, V2NEST = V2["final"]["cvAccuracy"], V2["final"]["nestedAccuracy"]
tr = pd.read_csv(D + "Train_set..csv"); te = pd.read_csv(D + "Test_set..csv"); ss = pd.read_csv(D + "Sample_Submission..csv"); y = tr["default"].values
NAMES = {"lgb": "LightGBM v3 · 127 leaves", "lgbw": "LightGBM v3 · 255 leaves, deep", "xgb": "XGBoost v3", "cb": "CatBoost v3",
         "et": "ExtraTrees v3", "mlp": "Neural network (tabular MLP)"}
oof, tst, per_seed = {}, {}, {}
for k in NAMES:
    f = sorted(glob.glob(f"{D}v3_{k}_s*_oof.npy"))
    if not f: continue
    oof[k] = np.mean([np.load(x) for x in f], 0); tst[k] = np.mean([np.load(x.replace("_oof", "_test")) for x in f], 0)
    per_seed[k] = {int(x.split("_s")[-1].split("_")[0]): round(accuracy_score(y, np.load(x) > .5) * 100, 2) for x in f}
keys = list(oof)
TS = np.arange(0.30, 0.71, 0.005)
def thr(p, yy): a = [accuracy_score(yy, p > t) for t in TS]; i = int(np.argmax(a)); return float(TS[i]), a[i]
def logit(p): p = np.clip(p, 1e-6, 1 - 1e-6); return np.log(p / (1 - p))
Z = np.column_stack([logit(oof[k]) for k in keys]); ZT = np.column_stack([logit(tst[k]) for k in keys])
cands = {k: (oof[k], tst[k]) for k in keys}

# hill-climbing weights (Caruana) — chosen out-of-fold on a second split to keep it honest
def hill(P, yy, iters=40):
    w = np.zeros(P.shape[1]); best = -1
    for _ in range(iters):
        sc = []
        for j in range(P.shape[1]):
            w2 = w.copy(); w2[j] += 1; p = P @ w2 / w2.sum(); sc.append(roc_auc_score(yy, p))
        j = int(np.argmax(sc))
        if sc[j] <= best: break
        best = sc[j]; w[j] += 1
    return w / w.sum()
P = np.column_stack([oof[k] for k in keys]); PT = np.column_stack([tst[k] for k in keys])
hc = np.zeros(len(y))
for a, b in StratifiedKFold(5, shuffle=True, random_state=2026).split(P, y): hc[b] = P[b] @ hill(P[a], y[a])
W = hill(P, y); cands["hill"] = (hc, PT @ W)

lr = np.zeros(len(y))
for a, b in StratifiedKFold(5, shuffle=True, random_state=2026).split(Z, y):
    lr[b] = LogisticRegression(C=1.0, max_iter=2000).fit(Z[a], y[a]).predict_proba(Z[b])[:, 1]
cands["stack"] = (lr, LogisticRegression(C=1.0, max_iter=2000).fit(Z, y).predict_proba(ZT)[:, 1])

import lightgbm as lgb
raw = tr[["interest_rate", "loan_amnt", "last_week_pay"]].values; rawT = te[["interest_rate", "loan_amnt", "last_week_pay"]].values
M = np.column_stack([Z, raw]); MT = np.column_stack([ZT, rawT]); gm = np.zeros(len(y)); gt = np.zeros(len(te))
mp = dict(n_estimators=400, learning_rate=0.03, num_leaves=15, min_child_samples=200, subsample=0.8, subsample_freq=1, n_jobs=4, verbose=-1, random_state=1)
for a, b in StratifiedKFold(5, shuffle=True, random_state=2026).split(M, y):
    m = lgb.LGBMClassifier(**mp).fit(M[a], y[a]); gm[b] = m.predict_proba(M[b])[:, 1]; gt += m.predict_proba(MT)[:, 1] / 5
cands["stack_lgb"] = (gm, gt)
cands["stack_blend"] = ((lr + gm) / 2, (cands["stack"][1] + gt) / 2)

LABEL = {"hill": "Hill-climb weighted blend", "stack": "Stacked (logistic meta-model)", "stack_lgb": "Stacked (LightGBM meta-model)",
         "stack_blend": "Two-level stack (logistic + LightGBM meta)"}
rows = []
for k, (p, _) in cands.items():
    t, a = thr(p, y); nest = []
    for s in (7, 42, 2026):
        for a_, b_ in StratifiedKFold(5, shuffle=True, random_state=s).split(p.reshape(-1, 1), y):
            tt, _ = thr(p[a_], y[a_]); nest.append(accuracy_score(y[b_], p[b_] > tt))
    rows.append(dict(id=k, name=NAMES.get(k, LABEL.get(k)), acc05=round(accuracy_score(y, p > .5) * 100, 2), auc=round(roc_auc_score(y, p), 4),
                     threshold=round(t, 3), accTuned=round(a * 100, 2), accNested=round(float(np.mean(nest)) * 100, 2)))
rows.sort(key=lambda r: -r["accNested"]); print(pd.DataFrame(rows).to_string())
best = rows[0]; p, pt = cands[best["id"]]; t = best["threshold"]
pred = (p > t).astype(int); tn, fp, fn, tpv = confusion_matrix(y, pred).ravel()
gain_nested = round(best["accNested"] - V2NEST, 2); promoted = gain_nested >= 0.10

# error analysis of the final model
err = pd.DataFrame({"grade": tr["loan_grade"], "term": tr["loan_term"], "wrong": pred != y, "near": np.abs(p - t) < 0.1})
errors = dict(totalWrong=int((pred != y).sum()), nearCutoffShare=round(float(err.loc[err.wrong, "near"].mean()) * 100, 1),
              byGrade=[dict(group=g, errRate=round(float(v) * 100, 2)) for g, v in err.groupby("grade")["wrong"].mean().items()],
              byTerm=[dict(group=g, errRate=round(float(v) * 100, 2)) for g, v in err.groupby("term")["wrong"].mean().items()])

sub = pd.DataFrame({"ID": te["ID"], "default": (pt > t).astype(int)})
assert len(sub) == 39933 and set(sub["default"].unique()) <= {0, 1} and (sub["ID"].values == ss["ID"].values).all()
sub.to_csv(R + "public/submission/DHDL_Loan_Default_Submission_v3.csv", index=False)
v2s = pd.read_csv(R + "public/submission/DHDL_Loan_Default_Submission_v2.csv")
adv = json.load(open(D + "adversarial.json")) if glob.glob(D + "adversarial.json") else None
ev = dict(version="v3", v2Accuracy=V2ACC, v2Nested=V2NEST, v1Accuracy=V2["v1Accuracy"], models=rows, perSeedAcc05=per_seed,
          hillWeights={k: round(float(w), 3) for k, w in zip(keys, W)},
          final=dict(id=best["id"], name=best["name"], threshold=t, cvAccuracy=best["accTuned"], nestedAccuracy=best["accNested"], auc=best["auc"],
                     gainVsV2=round(best["accTuned"] - V2ACC, 2), gainNestedVsV2=gain_nested, promoted=promoted,
                     confusion=dict(tn=int(tn), fp=int(fp), fn=int(fn), tp=int(tpv)), testDefaultRate=round(float(sub["default"].mean()) * 100, 2),
                     agreementWithV2=round(float((v2s["default"].values == sub["default"].values).mean()) * 100, 2), rows=len(sub)),
          errors=errors, adversarial=adv,
          seeds=sorted({int(x.split("_s")[-1].split("_")[0]) for x in glob.glob(D + "v3_*_oof.npy")}),
          newFeatures=["rebuilt repayment schedule: expected balance and interest if every payment were made",
                       "interest received vs schedule (ratio and gap)", "implied months paid vs months elapsed",
                       "in-fold smoothed default rates for sub-grade, state, purpose, sub-grade × term",
                       "peer comparisons of months-behind within sub-grade, state, purpose", "loan-amount frequency and round-amount flag"])
json.dump(ev, open(R + "src/data/loan-default-v3-evidence.json", "w"), indent=2)
print(json.dumps(ev["final"], indent=1))
