THE EVOLVING TRANSFORMER · STOP 05
Two diffs on the LLaMA stack. First a sliding window and a rolling KV buffer. Then the dense SwiGLU becomes 8 experts, top-2. Neither model is weight-tied.
Everything else is the LLaMA decoder: pre-norm, RMSNorm, SwiGLU, RoPE, GQA, untied head. Mistral 7B multiplies the causal mask by a band of width W = 4,096. A rolling buffer stores token i in slot i mod W, so the cache never holds more than 512 MiB. Mixtral then replaces the dense SwiGLU with 8 SwiGLU experts and a top-2 softmax router. The FFN is still SwiGLU — silu(gate) × up — not a plain SiLU MLP.
The sliding window is a desk that only holds the last four pages. Older pages still matter: each layer can pass a summary one window further back. The router is a clerk who picks two specialists per word and averages their answers.
The mask is still S×S in this teaching code, so compute is not yet O(N×W). A banded kernel or the rolling buffer is what makes the bill drop. Sliding window → · MoE →
| Knob | Mistral 7B | Mixtral 8×7B |
|---|---|---|
| attention | GQA + window 4,096 | same idea |
| FFN | dense SwiGLU | 8 experts, top-2 SwiGLU |
| cache / token | 128 KiB | same per token |
| weight tying | no | no |
Teaching window W = 4, sequence 8, Mixtral 8 experts top-2, hidden 32.
8 experts · top-2 · aux balance
full causal · window band · rolling KV
| Tensor | Shape | |
|---|---|---|
| sliding mask | (16, 16) | |
| ones on row 7 of a W=4 band | 4 | positions 4…7 |
| router weights | (2, 8, 2) | sum to 1 |
| Mixtral logits | (2, 8, 50) |
After 32 layers a 4,096-token window can carry a signal about 131,072 tokens back. Token 9 in a W = 4 buffer sits in slot 1 and overwrites token 5.
"""Mistral-style decoder: LLaMA + sliding-window mask + rolling KV buffer.
Mistral 7B published: 32 / 32 / 8 / 128, hidden 4096, vocab 32,000,
window W = 4,096. SwiGLU (not a plain SiLU FFN). NOT weight-tied.
The mask is causal ∩ a band of width W. The rolling buffer stores
token i in slot i mod W so the cache never grows past W.
"""
from dataclasses import dataclass
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class MistralConfig:
dmodel: int = 32
N: int = 2
H: int = 4
n_kv_heads: int = 2
vocabsize: int = 50
max_seq_len: int = 16
dff: int = 64
norm_eps: float = 1e-5
sliding_window: int = 4
rope_theta: float = 10_000.0
@property
def dk(self):
return self.dmodel // self.H
def make_sliding_window_mask(seq_len, window, device=None):
"""Token i attends to [max(0, i-W+1) .. i]. Shape (S, S)."""
causal = torch.tril(torch.ones(seq_len, seq_len, device=device))
band = torch.triu(torch.ones(seq_len, seq_len, device=device), diagonal=-(window - 1))
return causal * band
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-5):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
class SwiGLU(nn.Module):
def __init__(self, config):
super().__init__()
self.w_gate = nn.Linear(config.dmodel, config.dff, bias=False)
self.w_up = nn.Linear(config.dmodel, config.dff, bias=False)
self.w_down = nn.Linear(config.dff, config.dmodel, bias=False)
def forward(self, x):
return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))
class RotaryPositionalEmbedding(nn.Module):
def __init__(self, head_dim, max_seq_len, theta):
super().__init__()
freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
angles = torch.outer(torch.arange(max_seq_len).float(), freqs)
self.register_buffer("cos_cached", torch.cos(angles))
self.register_buffer("sin_cached", torch.sin(angles))
def forward(self, x, positions):
# positions: (S,) absolute indices — needed when the buffer wraps
even, odd = x[..., 0::2], x[..., 1::2]
cos = self.cos_cached[positions].view(1, 1, -1, even.size(-1))
sin = self.sin_cached[positions].view(1, 1, -1, even.size(-1))
rot = torch.stack([even * cos - odd * sin, even * sin + odd * cos], dim=-1)
return rot.flatten(-2)
class RollingKVCache:
"""W slots. Token i overwrites slot i mod W."""
def __init__(self, window):
self.window = window
self.k = None
self.v = None
self.n_seen = 0
def append(self, k_new, v_new):
B, H, T_new, D = k_new.shape
if self.k is None:
self.k = torch.zeros(B, H, self.window, D, device=k_new.device, dtype=k_new.dtype)
self.v = torch.zeros_like(self.k)
for t in range(T_new):
slot = (self.n_seen + t) % self.window
self.k[:, :, slot] = k_new[:, :, t]
self.v[:, :, slot] = v_new[:, :, t]
self.n_seen += T_new
return self.k, self.v, self.n_seen
def repeat_kv(x, n_rep):
if n_rep == 1:
return x
B, n_kv, S, d = x.shape
return x.unsqueeze(2).expand(B, n_kv, n_rep, S, d).reshape(B, n_kv * n_rep, S, d)
class SlidingWindowGQA(nn.Module):
def __init__(self, config):
super().__init__()
self.H = config.H
self.n_kv = config.n_kv_heads
self.n_rep = config.H // config.n_kv_heads
self.dk = config.dk
self.window = config.sliding_window
self.wq = nn.Linear(config.dmodel, config.H * config.dk, bias=False)
self.wk = nn.Linear(config.dmodel, config.n_kv_heads * config.dk, bias=False)
self.wv = nn.Linear(config.dmodel, config.n_kv_heads * config.dk, bias=False)
self.wo = nn.Linear(config.dmodel, config.dmodel, bias=False)
def forward(self, x, rope, mask, start_pos=0, cache=None):
B, S, _ = x.shape
q = self.wq(x).view(B, S, self.H, self.dk).transpose(1, 2)
k = self.wk(x).view(B, S, self.n_kv, self.dk).transpose(1, 2)
v = self.wv(x).view(B, S, self.n_kv, self.dk).transpose(1, 2)
q_pos = torch.arange(start_pos, start_pos + S, device=x.device)
q, k = rope(q, q_pos), rope(k, q_pos)
if cache is not None:
k, v, n_seen = cache.append(k, v)
# valid slots are the last min(n_seen, W) writes
t_all = min(n_seen, self.window)
else:
t_all = S
k_rep, v_rep = repeat_kv(k[:, :, :t_all] if cache is None else k, self.n_rep), None
if cache is None:
v_rep = repeat_kv(v, self.n_rep)
scores = (q @ k_rep.transpose(-1, -2)) / math.sqrt(self.dk)
scores = scores.masked_fill(mask[:S, :S] == 0, float("-inf"))
else:
v_rep = repeat_kv(v, self.n_rep)
scores = (q @ k_rep.transpose(-1, -2)) / math.sqrt(self.dk)
out = scores.softmax(dim=-1) @ v_rep
return self.wo(out.transpose(1, 2).contiguous().view(B, S, self.H * self.dk))
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = SlidingWindowGQA(config)
self.ff = SwiGLU(config)
self.rms_1 = RMSNorm(config.dmodel, config.norm_eps)
self.rms_2 = RMSNorm(config.dmodel, config.norm_eps)
def forward(self, x, rope, mask, start_pos=0, cache=None):
x = x + self.attn(self.rms_1(x), rope, mask, start_pos, cache)
return x + self.ff(self.rms_2(x))
class Mistral(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.wte = nn.Embedding(config.vocabsize, config.dmodel)
self.rope = RotaryPositionalEmbedding(config.dk, config.max_seq_len, config.rope_theta)
self.blocks = nn.ModuleList([Block(config) for _ in range(config.N)])
self.norm = RMSNorm(config.dmodel, config.norm_eps)
self.lm_head = nn.Linear(config.dmodel, config.vocabsize, bias=False)
self.register_buffer(
"mask",
make_sliding_window_mask(config.max_seq_len, config.sliding_window),
)
def forward(self, idx, start_pos=0, caches=None):
x = self.wte(idx)
for i, block in enumerate(self.blocks):
cache = None if caches is None else caches[i]
x = block(x, self.rope, self.mask, start_pos, cache)
return self.lm_head(self.norm(x))
"""Mixtral: Mistral's stack with a sparse SwiGLU MoE (8 experts, top-2).
Published Mixtral 8×7B: 8 experts, top-2, SwiGLU (not a plain SiLU FFN).
NOT weight-tied. Attention, embeddings and norms are shared — that is
why the total is ~46.7B, not 8 × 7.3B.
The dispatch loop is a teaching loop: production batches tokens per expert.
"""
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
from mistral import (
MistralConfig,
RMSNorm,
RotaryPositionalEmbedding,
SlidingWindowGQA,
SwiGLU,
make_sliding_window_mask,
)
@dataclass
class MixtralConfig(MistralConfig):
num_experts: int = 8
top_k: int = 2
class MoERouter(nn.Module):
def __init__(self, config):
super().__init__()
self.top_k = config.top_k
self.router = nn.Linear(config.dmodel, config.num_experts, bias=False)
def forward(self, x):
# softmax over experts, then keep top-k and renormalise
probs = F.softmax(self.router(x), dim=-1)
weight, indices = torch.topk(probs, self.top_k, dim=-1)
weight = weight / weight.sum(dim=-1, keepdim=True)
return weight, indices, probs
def router_aux_loss(probs, indices, num_experts):
"""Switch-style load-balancing term. Stops routing collapse."""
# fraction of tokens sent to each expert (hard assignment)
one_hot = F.one_hot(indices, num_experts).float().sum(dim=-2)
tokens = one_hot.size(0) * one_hot.size(1)
load = one_hot.sum(dim=(0, 1)) / (tokens * indices.size(-1))
importance = probs.mean(dim=(0, 1))
return num_experts * (load * importance).sum()
class MoE(nn.Module):
def __init__(self, config):
super().__init__()
self.experts = nn.ModuleList([SwiGLU(config) for _ in range(config.num_experts)])
self.router = MoERouter(config)
self.top_k = config.top_k
self.num_experts = config.num_experts
def forward(self, x):
weights, indices, probs = self.router(x)
output = torch.zeros_like(x)
for k in range(self.top_k):
for i, expert in enumerate(self.experts):
chosen = indices[:, :, k] == i
if chosen.any():
output[chosen] += weights[:, :, k][chosen].unsqueeze(-1) * expert(x[chosen])
aux = router_aux_loss(probs, indices, self.num_experts)
return output, aux
class MixtralBlock(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = SlidingWindowGQA(config)
self.moe = MoE(config)
self.rms_1 = RMSNorm(config.dmodel, config.norm_eps)
self.rms_2 = RMSNorm(config.dmodel, config.norm_eps)
def forward(self, x, rope, mask):
x = x + self.attn(self.rms_1(x), rope, mask)
moe_out, aux = self.moe(self.rms_2(x))
return x + moe_out, aux
class Mixtral(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.wte = nn.Embedding(config.vocabsize, config.dmodel)
self.rope = RotaryPositionalEmbedding(config.dk, config.max_seq_len, config.rope_theta)
self.blocks = nn.ModuleList([MixtralBlock(config) for _ in range(config.N)])
self.norm = RMSNorm(config.dmodel, config.norm_eps)
self.lm_head = nn.Linear(config.dmodel, config.vocabsize, bias=False)
self.register_buffer(
"mask",
make_sliding_window_mask(config.max_seq_len, config.sliding_window),
)
def forward(self, idx):
x = self.wte(idx)
aux_total = x.new_zeros(())
for block in self.blocks:
x, aux = block(x, self.rope, self.mask)
aux_total = aux_total + aux
return self.lm_head(self.norm(x)), aux_total / self.config.N
Test note. Assert the band width, the rolling-buffer slots after token 9, a Mistral forward, a Mixtral forward with 8 experts, router weights summing to 1, and both heads untied.
Calling the FFN “SiLU”. Mistral and Mixtral use SwiGLU: silu on the gate, times the up projection.
Tying the head. Both published configs set tie_word_embeddings: false.
Skipping the aux loss. Without a load-balancing term, top-2 routing collapses onto a few experts.
Writing 8 × 7.3B = 58B. Attention, embeddings and norms are shared. Mixtral is about 46.7B total, 12.9B active.
Tokens 4, 5, 6 and 7 — four positions.
Slot 9 mod 4 = 1. It overwrites token 5.
Two of eight, plus the shared attention path.