Recurrent Networks · L3 Code

Implementing a vanilla RNN cell from scratch and an LSTM-based sequence model in PyTorch.

python

import numpy as np

def tanh(z):
    return np.tanh(z)

class VanillaRNN:
    def __init__(self, input_dim, hidden_dim, seed=0):
        rng = np.random.default_rng(seed)
        self.Wxh = rng.normal(0, 0.1, (hidden_dim, input_dim))
        self.Whh = rng.normal(0, 0.1, (hidden_dim, hidden_dim))
        self.bh = np.zeros(hidden_dim)
        self.hidden_dim = hidden_dim

    def forward(self, x_seq):             # x_seq: (seq_len, input_dim)
        h = np.zeros(self.hidden_dim)
        hiddens = []
        for x_t in x_seq:
            h = tanh(self.Wxh @ x_t + self.Whh @ h + self.bh)
            hiddens.append(h.copy())
        return np.stack(hiddens)           # (seq_len, hidden_dim)

rnn = VanillaRNN(input_dim=8, hidden_dim=16)
seq = np.random.randn(10, 8)               # a sequence of 10 timesteps
hidden_states = rnn.forward(seq)
print(hidden_states.shape)                 # (10, 16)

A vanilla RNN's forward pass over a full sequence, from scratch.

Notice Wxh and Whh are the *same two matrices* reused at every single timestep — this weight sharing across time is exactly analogous to a convolution's weight sharing across space.