Four implementations, in increasing order of how much they've won in practice. Read RoPE's implementation carefully — it's the one you'll actually find inside LLaMA, Mistral, Qwen, and most other current open-weight models.
python
import numpy as np
def sinusoidal_encoding(max_len, d_model):
pos = np.arange(max_len)[:, None] # (max_len, 1)
i = np.arange(d_model)[None, :] # (1, d_model)
angle_rates = 1.0 / (10000 ** (2 * (i // 2) / d_model))
angles = pos * angle_rates
pe = np.zeros((max_len, d_model))
pe[:, 0::2] = np.sin(angles[:, 0::2]) # even dims: sine
pe[:, 1::2] = np.cos(angles[:, 1::2]) # odd dims: cosine
return pe
pe = sinusoidal_encoding(max_len=100, d_model=64)
token_embeddings = np.random.randn(100, 64) * 0.1
x = token_embeddings + pe # simple elementwise addition, done once, before layer 1