Two pieces of a real data pipeline, implemented completely: near-duplicate detection at scale via MinHash + LSH, and a lightweight trained quality classifier of the kind used to score raw crawl documents.
python
import numpy as np
from hashlib import blake2b
def get_shingles(text, k=5):
words = text.lower().split()
if len(words) < k:
return {" ".join(words)}
return {" ".join(words[i:i + k]) for i in range(len(words) - k + 1)}
def minhash_signature(shingles, num_perm=128):
# num_perm independent hash functions simulated via a fixed permutation trick
signature = np.full(num_perm, np.inf)
for sh in shingles:
base = int.from_bytes(blake2b(sh.encode(), digest_size=8).digest(), "big")
for i in range(num_perm):
# cheap pseudo-independent hash family: (a_i * base + b_i) mod large_prime
a, b = 2 * i + 1, i
h = (a * base + b) % (2 ** 61 - 1)
signature[i] = min(signature[i], h)
return signature
def jaccard_estimate(sig_a, sig_b):
return float((sig_a == sig_b).mean())