model_selection

from sklearn.model_selection import TimeSeriesSplit, cross_validate

# Expanding window: fold 1 trains on the earliest slice and tests on the
# next, fold 2 trains on both and tests on the third, and so on.
cv = TimeSeriesSplit(n_splits=5, test_size=30_000)

scores = cross_validate(
    pipeline, X, y, cv=cv,
    scoring=["average_precision", "roc_auc", "f1_macro"],
    return_train_score=True,    # gap between train and test reveals overfit
    n_jobs=-1,
)
print(scores["test_average_precision"])

Four situations, four splitters. Choosing by habit rather than by data shape is the most expensive mistake in this subpackage.

from sklearn.model_selection import (train_test_split, StratifiedKFold,
                                     GroupKFold, TimeSeriesSplit)

# 1. Plain holdout. stratify=y keeps class ratios in both halves -
#    without it, a rare class can vanish from the test set entirely.
X_tr, X_te, y_tr, y_te = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42)

# 2. Classification cross-validation
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# 3. Rows share an entity. The SAME CUSTOMER must not appear in both
#    train and test, or you are testing memorisation.
cv = GroupKFold(n_splits=5)
for tr, te in cv.split(X, y, groups=df["customer_id"]):
    pass

# 4. Time-ordered. Expanding window: always trains on the past.
cv = TimeSeriesSplit(n_splits=5, test_size=30_000, gap=1_000)
for tr, te in cv.split(X):
    print(f"train rows 0-{tr[-1]}, test rows {te[0]}-{te[-1]}")
# gap= leaves a buffer between train and test, which matters when your
# label takes time to mature (a default is only known 90 days later).

This is the function that settles the 10-million-rows question. Train at increasing sample sizes and look at where the curve flattens.

from sklearn.model_selection import learning_curve
import numpy as np

sizes, train_scores, val_scores = learning_curve(
    pipeline, X, y,
    train_sizes=np.array([0.05, 0.1, 0.2, 0.4, 0.7, 1.0]),
    cv=5, scoring="f1_macro", n_jobs=-1, random_state=42,
)

for n, tr, va in zip(sizes, train_scores.mean(1), val_scores.mean(1)):
    print(f"{n:>8,} rows   train {tr:.3f}   val {va:.3f}   gap {tr-va:.3f}")

#    2,500 rows   train 0.981   val 0.742   gap 0.239   <- overfitting
#    5,000 rows   train 0.954   val 0.798   gap 0.156
#   10,000 rows   train 0.921   val 0.844   gap 0.077
#   20,000 rows   train 0.903   val 0.869   gap 0.034
#   35,000 rows   train 0.897   val 0.878   gap 0.019
#   50,000 rows   train 0.894   val 0.881   gap 0.013   <- flat: stop here
#
# Reading it: a large gap means more data will help. A flat validation
# curve means it will not, and you need better features or a different
# model instead.

RandomizedSearchCV beats GridSearchCV on anything but a tiny space, because most hyperparameters barely matter and grid search spends equal effort on all of them.

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import loguniform, randint

search = RandomizedSearchCV(
    pipeline,
    param_distributions={
        # Double underscore addresses a step inside the Pipeline
        "tfidf__ngram_range": [(1, 1), (1, 2), (1, 3)],
        "tfidf__min_df": randint(2, 10),
        "clf__C": loguniform(1e-2, 1e2),     # sample on a log scale,
                                            # because C spans orders of
                                            # magnitude
    },
    n_iter=40,                  # 40 random draws, not every combination
    cv=StratifiedKFold(5, shuffle=True, random_state=42),
    scoring="f1_macro",
    n_jobs=-1,
    random_state=42,
    refit=True,                 # refits the best on all data
    return_train_score=True,
)

search.fit(X_train, y_train)
print(search.best_params_, round(search.best_score_, 4))

import pandas as pd
res = pd.DataFrame(search.cv_results_)
print(res.nlargest(5, "mean_test_score")[
    ["mean_test_score", "std_test_score", "mean_fit_time", "params"]])
# Look at std_test_score too. A high mean with a high std is a lucky
# fold, not a better model.

One fit, many numbers. Also surfaces the train/test gap, which single-metric scoring hides.

from sklearn.model_selection import cross_validate

scores = cross_validate(
    pipeline, X, y,
    cv=TimeSeriesSplit(n_splits=5),
    scoring=["average_precision", "roc_auc", "f1_macro", "balanced_accuracy"],
    return_train_score=True,
    n_jobs=-1,
)

for k in sorted(scores):
    if k.startswith(("test_", "train_")):
        print(f"{k:32} {scores[k].mean():.4f} +/- {scores[k].std():.4f}")

# Note how much lower average_precision is than roc_auc on imbalanced
# data. That difference is the whole argument for using PR AUC.