When to use: Large datasets where full pairwise comparison is too slow.
def deduplicate_with_blocking(df, block_col="zip_code", match_cols=None, threshold=0.80):
    """
    Scalable fuzzy deduplication using blocking.

    Blocking dramatically reduces comparisons:
    - 100K records with no blocking: 5 billion comparisons
    - 100K records with 1000 zip codes: ~5 million comparisons (1000x faster)

    Args:
        df: Input DataFrame
        block_col: Column to group records by (records only compared within same block)
        match_cols: Columns to use for similarity scoring
        threshold: Minimum average similarity to consider a match
    """
    if match_cols is None:
        match_cols = ["first_name", "last_name", "email"]

    duplicates_to_remove = set()

    for block_value, group in df.groupby(block_col):
        if len(group) < 2:
            continue

        indices = group.index.tolist()
        for i in range(len(indices)):
            if indices[i] in duplicates_to_remove:
                continue
            for j in range(i + 1, len(indices)):
                if indices[j] in duplicates_to_remove:
                    continue

                # Score similarity across multiple columns
                scores = []
                for col in match_cols:
                    val_a = str(group.loc[indices[i], col])
                    val_b = str(group.loc[indices[j], col])
                    scores.append(similarity(val_a, val_b))

                avg_score = sum(scores) / len(scores)
                if avg_score >= threshold:
                    # Keep the record with fewer nulls
                    nulls_i = group.loc[indices[i]].isnull().sum()
                    nulls_j = group.loc[indices[j]].isnull().sum()
                    remove_idx = indices[j] if nulls_i <= nulls_j else indices[i]
                    duplicates_to_remove.add(remove_idx)

    return df.drop(index=duplicates_to_remove)
Included with Data Cleaning Playbook

Ready for the complete Data Cleaning Playbook?
