Named entity recognition

aggregation_strategy is the argument that matters — without it you get raw sub-word pieces instead of whole entities.

from transformers import pipeline

ner = pipeline("token-classification",
               model="dslim/bert-base-NER",
               aggregation_strategy="simple")   # merges B- and I- pieces

text = ("Jane Okafor at Northwood Capital wired GBP 2,400,000 "
        "to Meridian Holdings on 14 March.")

for e in ner(text):
    print(f"{e['entity_group']:6} {e['word']:22} {e['score']:.3f} "
          f"[{e['start']}:{e['end']}]")

# start/end are character offsets into the original string. Keep them -
# they are what lets you redact precisely instead of string-replacing,
# which would also hit unrelated occurrences of the same substring.

The hybrid is not laziness. Structured identifiers have checksums, so a rule is both faster and more reliable than any model.

import re

PATTERNS = {
    "IBAN":  re.compile(r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b"),
    "SORT":  re.compile(r"\b\d{2}-\d{2}-\d{2}\b"),
    "CARD":  re.compile(r"\b(?:\d[ -]*?){13,19}\b"),
    "EMAIL": re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b"),
}

def redact(text, ner_pipe):
    spans = []
    # 1. deterministic identifiers
    for tag, rx in PATTERNS.items():
        spans += [(m.start(), m.end(), tag) for m in rx.finditer(text)]
    # 2. model-detected names and organisations
    for e in ner_pipe(text):
        if e["entity_group"] in ("PER", "ORG") and e["score"] > 0.85:
            spans.append((e["start"], e["end"], e["entity_group"]))

    # replace right-to-left so earlier offsets stay valid
    for s, e, tag in sorted(spans, key=lambda x: -x[0]):
        text = text[:s] + f"[{tag}]" + text[e:]
    return text

This is the part people get wrong. Sub-word pieces after the first must be masked with -100 so the loss ignores them; otherwise you train on labels that mean nothing.

def align_labels(examples, tokenizer, label2id):
    enc = tokenizer(examples["tokens"], truncation=True,
                    is_split_into_words=True, max_length=256)
    out = []
    for i, tags in enumerate(examples["ner_tags"]):
        word_ids = enc.word_ids(batch_index=i)
        prev, labels = None, []
        for wid in word_ids:
            if wid is None:
                labels.append(-100)          # [CLS], [SEP], padding
            elif wid != prev:
                labels.append(tags[wid])     # first piece carries the label
            else:
                labels.append(-100)          # continuation pieces ignored
            prev = wid
        out.append(labels)
    enc["labels"] = out
    return enc