feature_extraction

from sklearn.feature_extraction.text import TfidfVectorizer

vec = TfidfVectorizer(
    ngram_range=(1, 2),    # unigrams + bigrams; catches "not fraudulent"
    min_df=3,              # drop terms in fewer than 3 docs (noise)
    max_df=0.9,            # drop terms in >90% of docs (uninformative)
    sublinear_tf=True,     # log-scale counts; usually helps
    strip_accents="unicode",
    stop_words=None,       # careful: "not" and "no" are stop words
)
X = vec.fit_transform(train_texts)      # sparse matrix
print(X.shape, vec.get_feature_names_out()[:10])

Four arguments change results more than the choice of classifier downstream.

from sklearn.feature_extraction.text import TfidfVectorizer

docs = ["I was charged an overdraft fee incorrectly",
        "This was not a fraudulent transaction",
        "My card was declined at the supermarket",
        "Charged twice for the same overdraft"]

vec = TfidfVectorizer(
    ngram_range=(1, 2),      # THE important one: bigrams let a bag of
                             # words see "not fraudulent" as one feature
    min_df=1,                # raise to 3-5 on real data to drop noise
    max_df=0.9,              # drop terms in >90% of docs
    sublinear_tf=True,       # log-scale counts
    strip_accents="unicode",
    lowercase=True,
)

X = vec.fit_transform(docs)
print(X.shape)                            # (4, 42) - sparse
print([t for t in vec.get_feature_names_out() if " " in t][:6])
# ['an overdraft', 'charged an', 'charged twice', 'fee incorrectly', ...]

# Inspect what a single document became
import numpy as np
row = X[1].toarray().ravel()
terms = vec.get_feature_names_out()
for i in np.argsort(row)[-5:][::-1]:
    print(f"  {terms[i]:22} {row[i]:.3f}")

Payment narratives, merchant names and SWIFT fields are abbreviated and inconsistently spelled. Character n-grams survive that where word tokens do not.

from sklearn.feature_extraction.text import TfidfVectorizer

names = ["NORTHWOOD CAP LTD", "Northwood Capital Limited",
         "NRTHWD CAPITAL", "Southwood Capital Ltd"]

# Word-level: "NRTHWD" shares no tokens with "Northwood"
word = TfidfVectorizer(analyzer="word")

# Character-level: they share many 3-grams, so similarity survives
char = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5))

from sklearn.metrics.pairwise import cosine_similarity
for name, v in [("word", word), ("char", char)]:
    M = v.fit_transform(names)
    print(name, cosine_similarity(M[0:1], M).round(2))
# word [[1.   0.   0.   0.35]]      <- misses the abbreviation entirely
# char [[1.   0.62 0.41 0.55]]      <- catches it

# analyzer="char_wb" keeps n-grams inside word boundaries, which works
# better than plain "char" for name matching.