# DHDL Loan Default — full solution (seed 42). Run: python DHDL_Loan_Default_Solution.py <model>, tune.py <i>, finalize.py
# ===== pipeline.py =====
import pandas as pd, numpy as np, json, sys, time
from sklearn.model_selection import StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, f1_score, precision_score, recall_score
import lightgbm as lgb, xgboost as xgb
from catboost import CatBoostClassifier

SEED = 42
U = "/tmp/ld/"
tr = pd.read_csv(U + "Train_set..csv"); te = pd.read_csv(U + "Test_set..csv")
y = tr["default"].values
CATS = ["loan_grade", "home_ownership", "income_verification_status", "loan_purpose", "state_code", "application_type", "loan_subgrade"]

def fe(d):
    d = d.copy()
    d["term_m"] = d["loan_term"].str.extract(r"(\d+)").astype(float)[0] * 12
    exp = {"<5 Years": 2.5, "6-10 years": 8, "10+ years": 12}
    d["job_exp_num"] = d["job_experience"].map(exp)
    d["job_exp_missing"] = d["job_experience"].isna().astype(int)
    d["bal_missing"] = d["total_current_balance"].isna().astype(int)
    d["lwp_missing"] = d["last_week_pay"].isna().astype(int)
    d["subgrade_num"] = d["loan_subgrade"].map(lambda s: (ord(s[0]) - 65) * 5 + int(s[1]) if isinstance(s, str) else np.nan)
    d["int_per_loan"] = d["interest_receive"] / d["loan_amnt"]
    d["pay_progress"] = d["last_week_pay"] / d["term_m"]
    d["exp_interest"] = d["loan_amnt"] * d["interest_rate"] / 100 / 12 * d["last_week_pay"].fillna(0) / 4.33
    d["int_gap"] = d["interest_receive"] - d["exp_interest"]
    d["int_per_week"] = d["interest_receive"] / (d["last_week_pay"] + 1)
    d["revol_util"] = d["revolving_balance"] / (d["total_revolving_limit"] + 1)
    d["loan_to_income"] = d["loan_amnt"] / (d["annual_income"] + 1)
    d["installment"] = d["loan_amnt"] * (d["interest_rate"]/1200) / (1 - (1 + d["interest_rate"]/1200) ** (-d["term_m"]))
    d["inst_to_income"] = d["installment"] * 12 / (d["annual_income"] + 1)
    for c in CATS: d[c] = d[c].fillna("NA").astype("category")
    return d.drop(columns=["ID", "loan_term", "job_experience", "default"], errors="ignore")

X = fe(tr); XT = fe(te)
for c in CATS: XT[c] = pd.Categorical(XT[c], categories=X[c].cat.categories)
FEATS = list(X.columns)
skf = StratifiedKFold(5, shuffle=True, random_state=SEED)

def cv(name, fit_pred):
    oof = np.zeros(len(X)); tp = np.zeros(len(XT)); tra = []
    t = time.time()
    for f, (a, b) in enumerate(skf.split(X, y)):
        p_val, p_test, p_tr = fit_pred(X.iloc[a], y[a], X.iloc[b], y[b])
        oof[b] = p_val; tp += p_test / 5; tra.append(accuracy_score(y[a], p_tr > .5))
    acc = accuracy_score(y, oof > .5)
    print(f"{name}: acc={acc:.5f} auc={roc_auc_score(y, oof):.5f} train_acc={np.mean(tra):.5f} {time.time()-t:.0f}s", flush=True)
    return dict(oof=oof, test=tp, train_acc=float(np.mean(tra)))

num = [c for c in FEATS if c not in CATS]
def lr_fp(Xa, ya, Xb, yb):
    def prep(d, s=None):
        m = pd.get_dummies(d[num + ["loan_grade", "home_ownership", "loan_purpose", "income_verification_status"]], dtype=float).fillna(0)
        return m
    A, B, T = prep(Xa), prep(Xb), prep(XT)
    B = B.reindex(columns=A.columns, fill_value=0); T = T.reindex(columns=A.columns, fill_value=0)
    sc = StandardScaler().fit(A); m = LogisticRegression(max_iter=2000).fit(sc.transform(A), ya)
    return m.predict_proba(sc.transform(B))[:, 1], m.predict_proba(sc.transform(T))[:, 1], m.predict_proba(sc.transform(A))[:, 1]

def codes(d): 
    d = d.copy()
    for c in CATS: d[c] = d[c].cat.codes
    return d.fillna(-999)
def rf_fp(Xa, ya, Xb, yb):
    m = RandomForestClassifier(400, min_samples_leaf=2, n_jobs=1, random_state=SEED).fit(codes(Xa), ya)
    return m.predict_proba(codes(Xb))[:, 1], m.predict_proba(codes(XT))[:, 1], m.predict_proba(codes(Xa))[:, 1]

LGBP = dict(n_estimators=5000, learning_rate=0.05, num_leaves=63, n_jobs=1, min_child_samples=20, subsample=0.8, subsample_freq=1, colsample_bytree=0.7, reg_lambda=1.0, random_state=SEED, verbose=-1)
def lgb_fp(Xa, ya, Xb, yb):
    m = lgb.LGBMClassifier(**LGBP).fit(Xa, ya, eval_set=[(Xb, yb)], callbacks=[lgb.early_stopping(200, verbose=False)])
    return m.predict_proba(Xb)[:, 1], m.predict_proba(XT)[:, 1], m.predict_proba(Xa)[:, 1]

def xgb_fp(Xa, ya, Xb, yb):
    m = xgb.XGBClassifier(n_estimators=5000, learning_rate=0.05, max_depth=7, n_jobs=1, subsample=0.8, colsample_bytree=0.7, min_child_weight=2, enable_categorical=True, tree_method="hist", early_stopping_rounds=200, random_state=SEED, eval_metric="logloss")
    m.fit(Xa, ya, eval_set=[(Xb, yb)], verbose=False)
    return m.predict_proba(Xb)[:, 1], m.predict_proba(XT)[:, 1], m.predict_proba(Xa)[:, 1]

def cb_prep(d):
    d = d.copy()
    for c in CATS: d[c] = d[c].astype(str)
    return d
def cb_fp(Xa, ya, Xb, yb):
    m = CatBoostClassifier(iterations=4000, learning_rate=0.06, depth=8, random_seed=SEED, verbose=0, od_type="Iter", od_wait=200, cat_features=CATS, thread_count=1)
    m.fit(cb_prep(Xa), ya, eval_set=(cb_prep(Xb), yb))
    return m.predict_proba(cb_prep(Xb))[:, 1], m.predict_proba(cb_prep(XT))[:, 1], m.predict_proba(cb_prep(Xa))[:, 1]

which = sys.argv[1]
fp = dict(lr=lr_fp, rf=rf_fp, lgb=lgb_fp, xgb=xgb_fp, cb=cb_fp)[which]
r = cv(which, fp)
np.save(f"/tmp/ld/{which}_oof.npy", r["oof"]); np.save(f"/tmp/ld/{which}_test.npy", r["test"])
json.dump({"train_acc": r["train_acc"]}, open(f"/tmp/ld/{which}_meta.json", "w"))
if which == "lgb":
    m = lgb.LGBMClassifier(**{**LGBP, "n_estimators": 800}).fit(X, y)
    imp = pd.Series(m.booster_.feature_importance("gain"), FEATS).sort_values(ascending=False)
    (imp / imp.sum()).to_json("/tmp/ld/importance.json")

# ===== tune.py =====
# import sys, json, numpy as np
# sys.argv = ['x', 'none']
# exec(open('/tmp/ld/pipeline.py').read().split('which = sys.argv')[0])
# import sys as _s
# cfg_id = int(_s.orig_argv[-1])
# GRID = [
#   dict(num_leaves=31, min_child_samples=20, colsample_bytree=0.7),
#   dict(num_leaves=127, min_child_samples=30, colsample_bytree=0.6),
#   dict(num_leaves=255, min_child_samples=50, colsample_bytree=0.5),
#   dict(num_leaves=63, min_child_samples=10, colsample_bytree=0.9),
# ]
# P = {**LGBP, **GRID[cfg_id], "learning_rate": 0.04}
# def fp(Xa, ya, Xb, yb):
#     m = lgb.LGBMClassifier(**P).fit(Xa, ya, eval_set=[(Xb, yb)], callbacks=[lgb.early_stopping(200, verbose=False)])
#     return m.predict_proba(Xb)[:, 1], m.predict_proba(XT)[:, 1], m.predict_proba(Xa)[:, 1]
# r = cv(f"lgb_t{cfg_id}", fp)
# np.save(f"/tmp/ld/lgbt{cfg_id}_oof.npy", r["oof"]); np.save(f"/tmp/ld/lgbt{cfg_id}_test.npy", r["test"])
# json.dump({"train_acc": r["train_acc"], "params": GRID[cfg_id]}, open(f"/tmp/ld/lgbt{cfg_id}_meta.json", "w"))

# ===== finalize.py =====
# import json, os, numpy as np, pandas as pd
# from itertools import product
# from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, precision_score, recall_score, f1_score
# D = "/tmp/ld/"
# tr = pd.read_csv(D + "Train_set..csv"); te = pd.read_csv(D + "Test_set..csv"); ss = pd.read_csv(D + "Sample_Submission..csv")
# y = tr["default"].values
# NAMES = {"lr": "Logistic Regression", "rf": "Random Forest", "xgb": "XGBoost", "lgb": "LightGBM (base)", "cb": "CatBoost",
#          "lgbt0": "LightGBM · 31 leaves", "lgbt1": "LightGBM · 127 leaves", "lgbt2": "LightGBM · 255 leaves", "lgbt3": "LightGBM · 63 leaves, dense"}
# oof, tst, models = {}, {}, []
# for k in NAMES:
#     if not os.path.exists(D + f"{k}_oof.npy"): continue
#     oof[k] = np.load(D + f"{k}_oof.npy"); tst[k] = np.load(D + f"{k}_test.npy")
#     meta = json.load(open(D + f"{k}_meta.json"))
#     acc = accuracy_score(y, oof[k] > .5)
#     models.append(dict(id=k, name=NAMES[k], cvAccuracy=round(acc * 100, 2), cvAuc=round(roc_auc_score(y, oof[k]), 4),
#                        trainAccuracy=round(meta["train_acc"] * 100, 2), gap=round((meta["train_acc"] - acc) * 100, 2)))
# models.sort(key=lambda m: -m["cvAccuracy"])
# print(pd.DataFrame(models).to_string())
# 
# def best_thr(p):
#     ts = np.arange(0.30, 0.71, 0.005)
#     a = [accuracy_score(y, p > t) for t in ts]; i = int(np.argmax(a)); return float(ts[i]), a[i]
# 
# # Blend: best LightGBM + CatBoost + XGBoost, grid on weights (OOF)
# lg = max([k for k in oof if k.startswith("lgb")], key=lambda k: accuracy_score(y, oof[k] > .5))
# cands = [c for c in [lg, "cb", "xgb"] if c in oof]
# best = None
# for w in product(np.arange(0, 1.01, 0.1), repeat=len(cands)):
#     if abs(sum(w) - 1) > 1e-6: continue
#     p = sum(wi * oof[c] for wi, c in zip(w, cands)); t, a = best_thr(p)
#     if best is None or a > best[2]: best = (w, t, a)
# w, thr, blend_acc = best
# p_oof = sum(wi * oof[c] for wi, c in zip(w, cands)); p_test = sum(wi * tst[c] for wi, c in zip(w, cands))
# single_t, single_a = best_thr(oof[lg])
# use_blend = blend_acc > single_a + 1e-4
# print("blend", dict(zip(cands, w)), thr, blend_acc, "single", lg, single_t, single_a)
# if not use_blend:
#     p_oof, p_test, thr, w, cands = oof[lg], tst[lg], single_t, (1.0,), [lg]
# pred = (p_oof > thr).astype(int)
# tn, fp, fn, tp = confusion_matrix(y, pred).ravel()
# final = dict(components={c: round(float(wi), 2) for c, wi in zip(cands, w)}, threshold=round(thr, 3),
#              cvAccuracy=round(accuracy_score(y, pred) * 100, 2), cvAuc=round(roc_auc_score(y, p_oof), 4),
#              precision=round(precision_score(y, pred) * 100, 2), recall=round(recall_score(y, pred) * 100, 2),
#              f1=round(f1_score(y, pred) * 100, 2), tp=int(tp), tn=int(tn), fp=int(fp), fn=int(fn),
#              singleBest=dict(id=lg, threshold=round(single_t, 3), cvAccuracy=round(single_a * 100, 2)), blendUsed=bool(use_blend))
# 
# sub = pd.DataFrame({"ID": te["ID"], "default": (p_test > thr).astype(int)})
# assert len(sub) == 39933 and set(sub["ID"]) == set(ss["ID"]) and sub["default"].isin([0, 1]).all()
# sub.to_csv(D + "DHDL_Loan_Default_Submission.csv", index=False)
# bk = sorted([k for k in oof if k.startswith("lgb") and k != lg], key=lambda k: -best_thr(oof[k])[1])[0]
# bk_t, bk_a = best_thr(oof[bk])
# final["backup"] = dict(id=bk, threshold=round(bk_t, 3), cvAccuracy=round(bk_a * 100, 2))
# sub2 = pd.DataFrame({"ID": te["ID"], "default": (tst[bk] > bk_t).astype(int)})
# sub2.to_csv(D + "DHDL_Loan_Default_Submission_Backup.csv", index=False)
# final["testPredictedDefaultRate"] = round(sub["default"].mean() * 100, 2)
# final["backupPredictedDefaultRate"] = round(sub2["default"].mean() * 100, 2)
# final["backupAgreement"] = round((sub["default"] == sub2["default"]).mean() * 100, 2)
# 
# # EDA
# def rate(col, order=None):
#     g = tr.groupby(col, dropna=False)["default"].agg(["mean", "size"]).reset_index()
#     g[col] = g[col].astype(object).where(g[col].notna(), "Missing").astype(str)
#     if order: g = g.set_index(col).reindex([o for o in order if o in set(g[col])]).reset_index()
#     return [dict(group=r[col], rate=round(r["mean"] * 100, 2), n=int(r["size"])) for _, r in g.iterrows()]
# tr["_lwp_band"] = pd.cut(tr["last_week_pay"], [-1, 13, 26, 52, 104, 1e9], labels=["0-13", "14-26", "27-52", "53-104", "105+"]).astype(object)
# tr["_int_band"] = pd.qcut(tr["interest_rate"], 5).astype(str)
# tr["_irecv_band"] = pd.qcut(tr["interest_receive"], 5, duplicates="drop").astype(str)
# imp = json.load(open(D + "importance.json"))
# imp = sorted(imp.items(), key=lambda x: -x[1])[:15]
# miss = tr.isna().sum(); miss = {k: int(v) for k, v in miss.items() if v > 0}
# ev = dict(
#   data=dict(trainRows=len(tr), testRows=len(te), features=tr.shape[1] - 2, defaultRate=round(y.mean() * 100, 2),
#             baselineAccuracy=round((1 - y.mean()) * 100, 2), missing=miss, duplicates=int(tr.duplicated("ID").sum()),
#             engineeredFeatures=14, seed=42, folds=5),
#   eda=dict(grade=rate("loan_grade", list("ABCDEFG")), term=rate("loan_term"), home=rate("home_ownership"),
#            verification=rate("income_verification_status"), purpose=sorted(rate("loan_purpose"), key=lambda r: -r["n"])[:8],
#            weeksPaid=rate("_lwp_band", ["0-13", "14-26", "27-52", "53-104", "105+"]),
#            interestRate=rate("_int_band"), interestReceived=rate("_irecv_band"), jobExperience=rate("job_experience")),
#   models=models, final=final,
#   importance=[dict(feature=k, pct=round(v * 100, 2)) for k, v in imp],
#   leaderboardTop=88.28,
# )
# json.dump(ev, open(D + "loan-default-evidence.json", "w"), indent=1)
# print(json.dumps(final, indent=1))
