metrics

from sklearn.metrics import (classification_report, confusion_matrix,
                             average_precision_score, precision_recall_curve)
import numpy as np

print(classification_report(y_test, y_pred, digits=3))
print(confusion_matrix(y_test, y_pred))

# For rare positives, PR AUC - not ROC AUC
p = model.predict_proba(X_test)[:, 1]
print("PR AUC:", average_precision_score(y_test, p))

# Choose the operating point deliberately: what recall can the
# review team actually absorb?
prec, rec, thr = precision_recall_curve(y_test, p)
for target in (0.5, 0.7, 0.9):
    i = np.argmin(np.abs(rec - target))
    print(f"recall {rec[i]:.2f} -> precision {prec[i]:.2f} @ {thr[i]:.4f}")

Every aggregate metric compresses this table. Look at the table first, then decide which compression is right.

from sklearn.metrics import (classification_report, confusion_matrix,
                             ConfusionMatrixDisplay)

print(classification_report(y_test, y_pred, digits=3,
                            target_names=["fees", "fraud", "service", "tech"]))

#               precision    recall  f1-score   support
#         fees      0.881     0.912     0.896      1204
#        fraud      0.774     0.612     0.684       312   <- recall problem
#      service      0.856     0.889     0.872       998
#         tech      0.802     0.771     0.786       486
#
#     accuracy                          0.851      3000
#    macro avg      0.828     0.796     0.809      3000   <- classes equal
# weighted avg      0.848     0.851     0.848      3000   <- by frequency

print(confusion_matrix(y_test, y_pred))

# macro vs weighted matters: weighted hides poor performance on small
# classes, and 'fraud' is usually the small class you cannot afford
# to miss.

predict() uses 0.5, which is almost never the right cut for imbalanced problems. Pick the point your operation can absorb.

from sklearn.metrics import precision_recall_curve, average_precision_score
import numpy as np

p = model.predict_proba(X_test)[:, 1]
print("PR AUC:", round(average_precision_score(y_test, p), 4))

prec, rec, thr = precision_recall_curve(y_test, p)

print(f"{'recall':>8} {'precision':>10} {'threshold':>10} {'alerts/day':>11}")
for target in (0.90, 0.80, 0.70, 0.50):
    i = np.argmin(np.abs(rec[:-1] - target))
    alerts = (p >= thr[i]).sum() / DAYS_IN_TEST
    print(f"{rec[i]:8.2f} {prec[i]:10.2f} {thr[i]:10.4f} {alerts:11.0f}")

#   recall  precision  threshold  alerts/day
#     0.90       0.08     0.0142         840   <- team cannot work 840
#     0.80       0.19     0.0631         310
#     0.70       0.34     0.1487         152   <- feasible
#     0.50       0.61     0.4012          61
#
# The threshold is a capacity decision, not a modelling one. Bring this
# table to the operations team rather than picking a number yourself.

Both metrics on the same predictions. The gap is the argument.

from sklearn.metrics import roc_auc_score, average_precision_score
import numpy as np

# 0.5% positive rate
print("positive rate:", y_test.mean())          # 0.005

print("ROC AUC:", round(roc_auc_score(y_test, p), 4))          # 0.9412
print("PR AUC: ", round(average_precision_score(y_test, p), 4)) # 0.3140

# ROC AUC of 0.94 sounds excellent. But the baseline for PR AUC is the
# positive rate itself - 0.005 - so 0.314 means the model is 60x better
# than random, while precision at usable recall is still around 0.3.
#
# ROC AUC stays high because the enormous true-negative count dominates
# the false-positive rate denominator. It is not wrong, it is answering
# a question you did not ask.

When the business cost of the two error types differs, encode that and search against it rather than against F1.

from sklearn.metrics import make_scorer, confusion_matrix

COST_FN = 500.0     # missing a fraudulent transaction
COST_FP = 12.0      # an analyst reviewing a clean one

def expected_cost(y_true, y_pred):
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    return fp * COST_FP + fn * COST_FN

# greater_is_better=False makes the searcher minimise it
cost_scorer = make_scorer(expected_cost, greater_is_better=False)

search = RandomizedSearchCV(pipe, params, scoring=cost_scorer,
                            cv=5, n_iter=30, n_jobs=-1)
search.fit(X_train, y_train)
print("best expected cost per fold:", -search.best_score_)

# This is usually a more honest objective than F1, which weights the
# two error types equally for no reason connected to your business.