THE EVOLVING TRANSFORMER · STOP 09
Keep every image patch, project it into a decoder, and use a prefix-LM mask: the image is bidirectional, the text is causal. The decoder uses RoPE, not a learned wpe.
CLIP scored a pair. PaliGemma writes tokens. The vision tower keeps all 16 patches (no CLS). A linear map sends them into the decoder width. They sit in front of the caption. The mask is a prefix-LM: every image token may see every other image token; each text token may see all image tokens and earlier text, never later text. The decoder uses RoPE over the combined length — not a GPT-2-style wpe on the concat. The gated MLP is GeGLU (GELU on the gate), which is what Gemma 1 actually uses.
Tape a photograph to the top of a notepad. While you look at the photo, every patch may talk to every other patch. When you write the caption underneath, you may look at the photo and at the words you have already written, but not at the words you have not.
Fusion, SigLIP, and the prefix-LM idea are taught in Deep Cuts. This stop is the mask and the RoPE, in one file. Vision-language models →
| Tensor | Shape | Note |
|---|---|---|
| images | (2, 3, 16, 16) | |
| vision patches | (2, 16, 32) | no CLS |
| projected patches | (2, 16, 32) | decoder width |
| text ids | (2, 8) | |
| prefix-LM mask | (24, 24) | 16 + 8 |
| logits | (2, 24, 40) | image + text positions |
Mask check: image row 0 may see columns 0…15 and none of 16…23. Text row 20 may see 0…15 and 16…20.
projector prefix · prefix-LM mask · (2,24,40)
"""PaliGemma-style VLM: SigLIP patches + a prefix-LM decoder with RoPE.
The old stop used a plain causal mask over [image + text]. That is wrong.
PaliGemma is a prefix-LM: image tokens attend to each other, text is causal
and may look at every image token.
The decoder uses RoPE (not a learned wpe over the concat). The gated MLP
is GeGLU — GELU on the gate — as in Gemma 1.
"""
from dataclasses import dataclass
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class PaliGemmaConfig:
image_size: int = 16
patch_size: int = 4
C: int = 3
vision_dmodel: int = 32
vision_H: int = 4
vision_N: int = 2
vision_dff: int = 64
vocab_size: int = 40
max_text_len: int = 8
gemma_dmodel: int = 32
gemma_H: int = 4
gemma_N: int = 2
gemma_dff: int = 64
rope_theta: float = 10_000.0
norm_eps: float = 1e-6
@property
def num_patches(self):
return (self.image_size // self.patch_size) ** 2
@property
def vision_dk(self):
return self.vision_dmodel // self.vision_H
@property
def gemma_dk(self):
return self.gemma_dmodel // self.gemma_H
@property
def total_seq_len(self):
return self.num_patches + self.max_text_len
def prefix_lm_mask(n_image, n_text, device=None):
"""Image is bidirectional. Text is causal and can see every image token."""
S = n_image + n_text
mask = torch.zeros(S, S, device=device)
mask[:n_image, :n_image] = 1 # image ↔ image
mask[n_image:, :n_image] = 1 # text → image
mask[n_image:, n_image:] = torch.tril(torch.ones(n_text, n_text, device=device))
return mask
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
class RotaryPositionalEmbedding(nn.Module):
def __init__(self, head_dim, max_seq_len, theta):
super().__init__()
freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
angles = torch.outer(torch.arange(max_seq_len).float(), freqs)
self.register_buffer("cos_cached", torch.cos(angles))
self.register_buffer("sin_cached", torch.sin(angles))
def forward(self, x):
B, H, S, D = x.shape
even, odd = x[..., 0::2], x[..., 1::2]
cos = self.cos_cached[:S].view(1, 1, S, -1)
sin = self.sin_cached[:S].view(1, 1, S, -1)
return torch.stack([even * cos - odd * sin, even * sin + odd * cos], dim=-1).view(B, H, S, D)
class VisionBlock(nn.Module):
def __init__(self, config):
super().__init__()
d, h, dk = config.vision_dmodel, config.vision_H, config.vision_dk
self.h, self.dk = h, dk
self.ln1 = nn.LayerNorm(d)
self.ln2 = nn.LayerNorm(d)
self.wq = nn.Linear(d, d)
self.wk = nn.Linear(d, d)
self.wv = nn.Linear(d, d)
self.wo = nn.Linear(d, d)
self.ff = nn.Sequential(nn.Linear(d, config.vision_dff), nn.GELU(), nn.Linear(config.vision_dff, d))
def forward(self, x):
B, S, _ = x.shape
h = self.ln1(x)
q = self.wq(h).view(B, S, self.h, self.dk).transpose(1, 2)
k = self.wk(h).view(B, S, self.h, self.dk).transpose(1, 2)
v = self.wv(h).view(B, S, self.h, self.dk).transpose(1, 2)
scores = (q @ k.transpose(-1, -2)) / math.sqrt(self.dk)
x = x + self.wo((scores.softmax(-1) @ v).transpose(1, 2).contiguous().view(B, S, -1))
return x + self.ff(self.ln2(x))
class SigLIPVision(nn.Module):
"""All patches kept — no CLS. Positions are a small lookup table."""
def __init__(self, config):
super().__init__()
self.proj = nn.Conv2d(config.C, config.vision_dmodel, config.patch_size, config.patch_size, bias=False)
self.pos = nn.Embedding(config.num_patches, config.vision_dmodel)
self.blocks = nn.ModuleList([VisionBlock(config) for _ in range(config.vision_N)])
self.ln = nn.LayerNorm(config.vision_dmodel)
self.register_buffer("position_ids", torch.arange(config.num_patches).unsqueeze(0))
def forward(self, images):
x = self.proj(images).flatten(2).transpose(1, 2) + self.pos(self.position_ids)
for block in self.blocks:
x = block(x)
return self.ln(x)
class GeGLU(nn.Module):
"""Gemma's gated MLP: GELU(gate) * up, then down. Not SwiGLU."""
def __init__(self, config):
super().__init__()
d, df = config.gemma_dmodel, config.gemma_dff
self.gate = nn.Linear(d, df, bias=False)
self.up = nn.Linear(d, df, bias=False)
self.down = nn.Linear(df, d, bias=False)
def forward(self, x):
return self.down(F.gelu(self.gate(x), approximate="tanh") * self.up(x))
class GemmaAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.h = config.gemma_H
self.dk = config.gemma_dk
d = config.gemma_dmodel
self.wq = nn.Linear(d, d, bias=False)
self.wk = nn.Linear(d, d, bias=False)
self.wv = nn.Linear(d, d, bias=False)
self.wo = nn.Linear(d, d, bias=False)
def forward(self, x, rope, mask):
B, S, _ = x.shape
q = rope(self.wq(x).view(B, S, self.h, self.dk).transpose(1, 2))
k = rope(self.wk(x).view(B, S, self.h, self.dk).transpose(1, 2))
v = self.wv(x).view(B, S, self.h, self.dk).transpose(1, 2)
scores = (q @ k.transpose(-1, -2)) / math.sqrt(self.dk)
scores = scores.masked_fill(mask[:, :, :S, :S] == 0, float("-inf"))
out = (scores.softmax(-1) @ v).transpose(1, 2).contiguous().view(B, S, -1)
return self.wo(out)
class GemmaBlock(nn.Module):
def __init__(self, config):
super().__init__()
self.rms_1 = RMSNorm(config.gemma_dmodel, config.norm_eps)
self.rms_2 = RMSNorm(config.gemma_dmodel, config.norm_eps)
self.attn = GemmaAttention(config)
self.mlp = GeGLU(config)
def forward(self, x, rope, mask):
x = x + self.attn(self.rms_1(x), rope, mask)
return x + self.mlp(self.rms_2(x))
class GemmaDecoder(nn.Module):
def __init__(self, config):
super().__init__()
self.n_image = config.num_patches
self.wte = nn.Embedding(config.vocab_size, config.gemma_dmodel)
self.rope = RotaryPositionalEmbedding(config.gemma_dk, config.total_seq_len, config.rope_theta)
self.blocks = nn.ModuleList([GemmaBlock(config) for _ in range(config.gemma_N)])
self.norm = RMSNorm(config.gemma_dmodel, config.norm_eps)
self.lm_head = nn.Linear(config.gemma_dmodel, config.vocab_size, bias=False)
self.register_buffer(
"mask",
prefix_lm_mask(config.num_patches, config.max_text_len).view(
1, 1, config.total_seq_len, config.total_seq_len
),
)
def forward(self, image_tokens, text_ids):
text = self.wte(text_ids)
x = torch.cat([image_tokens, text], dim=1)
for block in self.blocks:
x = block(x, self.rope, self.mask)
return self.lm_head(self.norm(x))
class PaliGemma(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.vision = SigLIPVision(config)
self.projector = nn.Linear(config.vision_dmodel, config.gemma_dmodel)
self.language = GemmaDecoder(config)
def forward(self, images, text_ids):
patches = self.projector(self.vision(images))
return self.language(patches, text_ids)
Test note. Forward two tiles and two captions. Assert logits (2, 24, 40). Check the prefix-LM mask (image bidirectional, text causal). Confirm the decoder has RoPE and no learned wpe.
A plain causal mask over [image + text]. That forces early patches to ignore later patches. PaliGemma's image prefix is bidirectional.
A learned wpe on the concat. Gemma uses RoPE. Positions run through the image prefix and then the text.
Calling the decoder MLP SwiGLU. Gemma 1 is GeGLU — GELU on the gate, not SiLU.
Yes. The image prefix is bidirectional.
No. The image does not look at the text.
wpe?That is Gemma. The same rotary engine as LLaMA, run over patches then words.