2026年2月12日，AI大神Andrej Karpathy（@karpathy）再次发布极简的GPT编码教程，仅仅通过243行python代码就实现的现在AI大语言模型中的预训练及推理操作，堪称神作。下面我们来一一拆解该代码，使读者能有一个更清晰的视角来理解这一杰作：
python源码如下：
Python
"""
The most atomic way to train and inference a GPT in pure, dependency-free Python.
This file is the complete algorithm.
Everything else is just efficiency.
@karpathy
"""
import os # os.path.exists
import math # math.log, math.exp
import random # random.seed, random.choices, random.gauss, random.shuffle
# Let there be order among chaos
random.seed(42)
# Let there be an input dataset docs: list[str] of documents (e.g. a dataset of names)
if not os.path.exists('input.txt'):
import urllib.request
names_url = 'https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt'
urllib.request.urlretrieve(names_url, 'input.txt')
docs = [l.strip() for l in open('input.txt').read().strip().split('\n') if l.strip()] # list[str] of documents
random.shuffle(docs)
print(f"num docs: {len(docs)}")
# Let there be a Tokenizer to translate strings to discrete symbols and back
chars = ['<BOS>'] + sorted(set(''.join(docs))) # character-level tokenizer with a BOS delimiter
vocab_size = len(chars)
stoi = { ch:i for i, ch in enumerate(chars) } # encoding: map string to integer
itos = { i:ch for i, ch in enumerate(chars) } # decoding: map integer to string
BOS = stoi['<BOS>']
print(f"vocab size: {vocab_size}")
# Let there be an Autograd to apply the chain rule recursively across a computation graph and so
# calculate the gradients of the loss with respect to model parameters.
class Value:
"""Stores a single scalar value and its gradient."""
def **init**(self, data, _children=(), _op=''):
self.data = data
self.grad = 0
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc
def **add**(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')
def _backward():
self.grad += out.grad
other.grad += out.grad
out._backward = _backward
return out
def **mul**(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')
def _backward():
self.grad += other.data * out.grad
other.grad += self.data * out.grad
out._backward = _backward
return out
def **pow**(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')
def _backward():
self.grad += (other * self.data**(other-1)) * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), 'log')
def _backward():
self.grad += (1 / self.data) * out.grad
out._backward = _backward
return out
def exp(self):
out = Value(math.exp(self.data), (self,), 'exp')
def _backward():
self.grad += out.data * out.grad
out._backward = _backward
return out
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
def _backward():
self.grad += (out.data > 0) * out.grad
out._backward = _backward
return out
def backward(self):
# topological order all of the children in the graph
topo = []
visited = set()
def build_topo(v):
if v not in visited:
visited.add(v)
for child in v._prev:
build_topo(child)
topo.append(v)
build_topo(self)
# go one variable at a time and apply the chain rule to get its gradient
self.grad = 1
for v in reversed(topo):
v._backward()
def **neg**(self): return self * -1
def **radd**(self, other): return self + other
def **sub**(self, other): return self + (-other)
def **rsub**(self, other): return other + (-self)
def **rmul**(self, other): return self * other
def **truediv**(self, other): return self * other**-1
def **rtruediv**(self, other): return other * self**-1
def **repr**(self): return f"Value(data={self.data}, grad={self.grad})"
# Initialize the parameters, to store the knowledge of the model.
n_embd = 16 # embedding dimension
n_head = 4 # number of attention heads
n_layer = 1 # number of layers
block_size = 8 # maximum sequence length
head_dim = n_embd // n_head # dimension of each head
matrix = lambda nout, nin, std=0.02: [[Value(random.gauss(0, std)) for _ in range(nin)] for _ in range(nout)]
state_dict = {'wte': matrix(vocab_size, n_embd), 'wpe': matrix(block_size, n_embd), 'lm_head': matrix(vocab_size, n_embd)}
for i in range(n_layer):
state_dict[f'layer{i}.attn_wq'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wk'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wv'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wo'] = matrix(n_embd, n_embd, std=0)
state_dict[f'layer{i}.mlp_fc1'] = matrix(4 * n_embd, n_embd)
state_dict[f'layer{i}.mlp_fc2'] = matrix(n_embd, 4 * n_embd, std=0)
params = [p for mat in state_dict.values() for row in mat for p in row] # flatten params into a single list[Value]
print(f"num params: {len(params)}")
# Define the model architecture: a stateless function mapping token sequence and parameters to logits over what comes next.
# Follow GPT-2, blessed among the GPTs, with minor differences: layernorm -> rmsnorm, no biases, GeLU -> ReLU^2
def linear(x, w):
return [sum(wi * xi for wi, xi in zip(wo, x)) for wo in w]
def softmax(logits):
max_val = max(val.data for val in logits)
exps = [(val - max_val).exp() for val in logits]
total = sum(exps)
return [e / total for e in exps]
def rmsnorm(x):
ms = sum(xi * xi for xi in x) / len(x)
scale = (ms + 1e-5) ** -0.5
return [xi * scale for xi in x]
def gpt(token_id, pos_id, keys, values):
tok_emb = state_dict['wte'][token_id] # token embedding
pos_emb = state_dict['wpe'][pos_id] # position embedding
x = [t + p for t, p in zip(tok_emb, pos_emb)] # joint token and position embedding
x = rmsnorm(x)
for li in range(n_layer):
# 1) Multi-head attention block
x_residual = x
x = rmsnorm(x)
q = linear(x, state_dict[f'layer{li}.attn_wq'])
k = linear(x, state_dict[f'layer{li}.attn_wk'])
v = linear(x, state_dict[f'layer{li}.attn_wv'])
keys[li].append(k)
values[li].append(v)
x_attn = []
for h in range(n_head):
hs = h * head_dim
q_h = q[hs:hs+head_dim]
k_h = [ki[hs:hs+head_dim] for ki in keys[li]]
v_h = [vi[hs:hs+head_dim] for vi in values[li]]
attn_logits = [sum(q_h[j] * k_h[t][j] for j in range(head_dim)) / head_dim**0.5 for t in range(len(k_h))]
attn_weights = softmax(attn_logits)
head_out = [sum(attn_weights[t] * v_h[t][j] for t in range(len(v_h))) for j in range(head_dim)]
x_attn.extend(head_out)
x = linear(x_attn, state_dict[f'layer{li}.attn_wo'])
x = [a + b for a, b in zip(x, x_residual)]
# 2) MLP block
x_residual = x
x = rmsnorm(x)
x = linear(x, state_dict[f'layer{li}.mlp_fc1'])
x = [xi.relu() ** 2 for xi in x]
x = linear(x, state_dict[f'layer{li}.mlp_fc2'])
x = [a + b for a, b in zip(x, x_residual)]
logits = linear(x, state_dict['lm_head'])
return logits
# Let there be Adam, the blessed optimizer and its buffers
learning_rate, beta1, beta2, eps_adam = 1e-2, 0.9, 0.95, 1e-8
m = [0.0] * len(params) # first moment buffer
v = [0.0] * len(params) # second moment buffer
# Repeat in sequence
num_steps = 500 # number of training steps
for step in range(num_steps):
# Take single document, tokenize it, surround it with BOS special token on both sides
doc = docs[step % len(docs)]
tokens = [BOS] + [stoi[ch] for ch in doc] + [BOS]
n = min(block_size, len(tokens) - 1)
# Forward the token sequence through the model, building up the computation graph all the way to the loss.
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
losses = []
for pos_id in range(n):
token_id, target_id = tokens[pos_id], tokens[pos_id + 1]
logits = gpt(token_id, pos_id, keys, values)
probs = softmax(logits)
loss_t = -probs[target_id].log()
losses.append(loss_t)
loss = (1 / n) * sum(losses) # final average loss over the document sequence. May yours be low.
# Backward the loss, calculating the gradients with respect to all model parameters.
loss.backward()
# Adam optimizer update: update the model parameters based on the corresponding gradients.
lr_t = learning_rate * (1 - step / num_steps)
for i, p in enumerate(params):
m[i] = beta1 * m[i] + (1 - beta1) * p.grad
v[i] = beta2 * v[i] + (1 - beta2) * p.grad ** 2
m_hat = m[i] / (1 - beta1 ** (step + 1))
v_hat = v[i] / (1 - beta2 ** (step + 1))
p.data -= lr_t * m_hat / (v_hat ** 0.5 + eps_adam)
p.grad = 0
print(f"step {step+1:4d} / {num_steps:4d} | loss {loss.data:.4f}")
# Inference: may the model babble back to us
temperature = 0.6 # in (0, 1], control the "creativity" of generated text, low to high
print("\n--- inference ---")
for sample_idx in range(20):
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
token_id = BOS
print(f"sample {sample_idx+1}: ", end="")
for pos_id in range(block_size):
logits = gpt(token_id, pos_id, keys, values)
probs = softmax([l / temperature for l in logits])
token_id = random.choices(range(vocab_size), weights=[p.data for p in probs])[0]
if token_id == BOS:
break
print(itos[token_id], end="")
print()
这个代码是由Andrej Karpathy（@karpathy）编写的，一个“最原子化”（most atomic）的GPT模型训练和推断实现。它使用纯Python、无外部依赖（除了标准库），展示了GPT的核心算法，包括数据准备、分词器、自动求导引擎、模型架构、训练优化和推断生成。代码强调“一切其他都是效率优化”，意思是这个版本是最基础的，忽略了性能优化如向量化或GPU加速。
代码整体结构：
- 导入和初始化：设置随机种子、加载数据。
- 分词器：字符级 tokenizer。
- 自动求导引擎（Value类）：实现反向传播。
- 模型参数初始化：使用高斯分布初始化权重。
- 模型架构：定义前向传播函数，包括嵌入、注意力、MLP 等。
- 优化器：手动实现 Adam。
- 训练循环：前向、损失计算、反向传播、参数更新。
- 推断：生成新序列。
下面，我将逐段详细解释代码，每段后附上解释。解释包括作用、关键概念、数学原理和潜在问题。代码中的注释（如“# Let there be...”）是诗意化的，增加了趣味性。
1. 导入模块和设置随机种子
import os # os.path.exists import math # math.log, math.exp import random # random.seed, random.choices, random.gauss, random.shuffle # Let there be order among chaos random.seed(42)
- 解释：导入必要的标准库模块。os 用于检查文件是否存在；math 用于数学运算如对数和指数；random 用于随机数生成，包括设置种子（seed=42，确保可复现性）。设置种子后，所有随机操作（如参数初始化、采样）都会相同。这部分是为了“从混沌中带来秩序”，即控制随机性。
2. 加载输入数据集
if not os.path.exists('input.txt'):
import urllib.request
names_url = 'https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt'
urllib.request.urlretrieve(names_url, 'input.txt')
docs = [l.strip() for l in open('input.txt').read().strip().split('\n') if l.strip()] # list[str] of documents
random.shuffle(docs)
print(f"num docs: {len(docs)}")
- 解释：如果 input.txt 不存在，从 GitHub 下载一个名字数据集（约 32k 行英文名字）。然后读取文件，处理成列表 docs，每个元素是一个名字（如 "emma"）。使用列表推导式过滤空行，并用 random.shuffle 打乱顺序（因为种子已设，可复现）。打印文档数量（通常为 32033）。这是一个字符级语言建模任务的数据准备，类似于 next‑token prediction。潜在问题：如果网络不可用，下载会失败；实际运行时，数据集是纯文本，每行一个名字。
3. 创建分词器（Tokenizer）
chars = ['<BOS>'] + sorted(set(''.join(docs))) # character-level tokenizer with a BOS delimiter
vocab_size = len(chars)
stoi = { ch:i for i, ch in enumerate(chars) } # encoding: map string to integer
itos = { i:ch for i, ch in enumerate(chars) } # decoding: map integer to string
BOS = stoi['<BOS>']
print(f"vocab size: {vocab_size}")
- 解释：构建字符级分词器。chars 是词汇表：添加特殊 token <BOS>（Beginning of Sequence，开始序列标记），然后从所有文档中提取唯一字符并排序（通常 27 个，包括 a‑z 和 <BOS>）。vocab_size 是词汇表大小（27）。stoi 和 itos 是字符串到整数、整数到字符串的映射字典。BOS 是 <BOS> 的整数 ID（0）。这是一个简单的字符 tokenizer，适用于小词汇表任务。打印词汇大小。原理：语言建模中，将文本转为 token ID 序列，便于嵌入。
4. 自动求导引擎（Value 类）
class Value:
"""Stores a single scalar value and its gradient."""
def __init__(self, data, _children=(), _op=''):
self.data = data
self.grad = 0
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc
def __add__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')
def _backward():
self.grad += out.grad
other.grad += out.grad
out._backward = _backward
return out
def __mul__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')
def _backward():
self.grad += other.data * out.grad
other.grad += self.data * out.grad
out._backward = _backward
return out
def __pow__(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')
def _backward():
self.grad += (other * self.data**(other-1)) * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), 'log')
def _backward():
self.grad += (1 / self.data) * out.grad
out._backward = _backward
return out
def exp(self):
out = Value(math.exp(self.data), (self,), 'exp')
def _backward():
self.grad += out.data * out.grad
out._backward = _backward
return out
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
def _backward():
self.grad += (out.data > 0) * out.grad
out._backward = _backward
return out
def backward(self):
# topological order all of the children in the graph
topo = []
visited = set()
def build_topo(v):
if v not in visited:
visited.add(v)
for child in v._prev:
build_topo(child)
topo.append(v)
build_topo(self)
# go one variable at a time and apply the chain rule to get its gradient
self.grad = 1
for v in reversed(topo):
v._backward()
def __neg__(self): return self * -1
def __radd__(self, other): return self + other
def __sub__(self, other): return self + (-other)
def __rsub__(self, other): return other + (-self)
def __rmul__(self, other): return self * other
def __truediv__(self, other): return self * other**-1
def __rtruediv__(self, other): return other * self**-1
def __repr__(self): return f"Value(data={self.data}, grad={self.grad})"
- 解释：这是代码的核心：一个简单的自动求导系统，类似于 micrograd（Karpathy 的另一个项目）。Value 类封装标量值和梯度，支持基本运算（如加、乘、幂、log、exp、ReLU）。每个运算记录前驱节点（_prev）和操作符（_op），并定义 _backward 函数，用于链式法则计算梯度。
- 初始化：存储数据、梯度（初始0）、反向函数、前驱和操作。
- 运算重载：如 __add__ 创建新 Value，定义如何传播梯度（e.g., 加法的梯度是1）。
- 非线性函数：log、exp、relu 的反向是标准导数（e.g., log 的导数是 1/x）。
- backward 方法：构建拓扑排序（DFS 后序遍历），从损失开始（grad=1），逆序调用 _backward 计算所有梯度。
- 其他运算：支持负、减、除等，通过重载实现。
- 数学原理：这是计算图的反向传播。每个 Value 是图节点，运算构建图，backward 应用链式法则。局限：仅标量，非向量化，效率低，但易懂。用于计算损失对参数的梯度。
5. 初始化模型参数
n_embd = 16 # embedding dimension
n_head = 4 # number of attention heads
n_layer = 1 # number of layers
block_size = 8 # maximum sequence length
head_dim = n_embd // n_head # dimension of each head
matrix = lambda nout, nin, std=0.02: [[Value(random.gauss(0, std)) for _ in range(nin)] for _ in range(nout)]
state_dict = {'wte': matrix(vocab_size, n_embd), 'wpe': matrix(block_size, n_embd), 'lm_head': matrix(vocab_size, n_embd)}
for i in range(n_layer):
state_dict[f'layer{i}.attn_wq'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wk'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wv'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wo'] = matrix(n_embd, n_embd, std=0)
state_dict[f'layer{i}.mlp_fc1'] = matrix(4 * n_embd, n_embd)
state_dict[f'layer{i}.mlp_fc2'] = matrix(n_embd, 4 * n_embd, std=0)
params = [p for mat in state_dict.values() for row in mat for p in row] # flatten params into a single list[Value]
print(f"num params: {len(params)}")
- 解释：定义超参数：嵌入维 16、4头注意力、1层、最大序列 8。matrix 函数创建权重矩阵，每个元素是 Value，使用高斯初始化（std=0.02，避免梯度爆炸）。state_dict 是字典存储所有权重：词嵌入（wte）、位置嵌入（wpe）、LM头（lm_head），以及每层的注意力（QKV、O）和MLP（fc1、fc2）。注意：投影层（如 wo、fc2）std=0（零初始化？代码中std=0，但可能是简化）。params 展平所有 Value 为列表（总参数约数千）。打印参数数量。这模拟 PyTorch 的 state_dict，参数是可微的 Value。
6. 定义模型辅助函数和 GPT 前向传播
def linear(x, w):
return [sum(wi * xi for wi, xi in zip(wo, x)) for wo in w]
def softmax(logits):
max_val = max(val.data for val in logits)
exps = [(val - max_val).exp() for val in logits]
total = sum(exps)
return [e / total for e in exps]
def rmsnorm(x):
ms = sum(xi * xi for xi in x) / len(x)
scale = (ms + 1e-5) ** -0.5
return [xi * scale for xi in x]
def gpt(token_id, pos_id, keys, values):
tok_emb = state_dict['wte'][token_id] # token embedding
pos_emb = state_dict['wpe'][pos_id] # position embedding
x = [t + p for t, p in zip(tok_emb, pos_emb)] # joint token and position embedding
x = rmsnorm(x)
for li in range(n_layer):
# 1) Multi-head attention block
x_residual = x
x = rmsnorm(x)
q = linear(x, state_dict[f'layer{li}.attn_wq'])
k = linear(x, state_dict[f'layer{li}.attn_wk'])
v = linear(x, state_dict[f'layer{li}.attn_wv'])
keys[li].append(k)
values[li].append(v)
x_attn = []
for h in range(n_head):
hs = h * head_dim
q_h = q[hs:hs+head_dim]
k_h = [ki[hs:hs+head_dim] for ki in keys[li]]
v_h = [vi[hs:hs+head_dim] for vi in values[li]]
attn_logits = [sum(q_h[j] * k_h[t][j] for j in range(head_dim)) / head_dim**0.5 for t in range(len(k_h))]
attn_weights = softmax(attn_logits)
head_out = [sum(attn_weights[t] * v_h[t][j] for t in range(len(v_h))) for j in range(head_dim)]
x_attn.extend(head_out)
x = linear(x_attn, state_dict[f'layer{li}.attn_wo'])
x = [a + b for a, b in zip(x, x_residual)]
# 2) MLP block
x_residual = x
x = rmsnorm(x)
x = linear(x, state_dict[f'layer{li}.mlp_fc1'])
x = [xi.relu() ** 2 for xi in x]
x = linear(x, state_dict[f'layer{li}.mlp_fc2'])
x = [a + b for a, b in zip(x, x_residual)]
logits = linear(x, state_dict['lm_head'])
return logits
- 解释：
- linear：矩阵乘法，输入 x（向量）和 w（矩阵），输出新向量。使用 zip 和 sum，实现标量乘积。
- softmax：稳定计算（减 max 以防溢出），将 logits 转为概率。
- rmsnorm：RMSNorm（LayerNorm 变体，无均值减，只除 RMS，添加 epsilon 防零除）。用于稳定训练。
- gpt 函数：无状态前向传播。输入 token_id、pos_id、累积的 keys/values（用于因果注意力）。
- 获取 token 和 position 嵌入，相加，RMSNorm。
- 每层：注意力 + 残差 + MLP + 残差。
- 注意力：计算 Q/K/V，追加到 keys/values（实现自回归）。多头：切分头，计算 dot‑product attention（scaled by 1/sqrt(d)），softmax 加权 V，concat 并投影 O。
- MLP：两层线性，中间 ReLU^2（近似 GeLU）。
- 最后：LM 头线性，输出 logits（vocab_size 维）。
- 数学原理：遵循 GPT‑2，但简化：RMSNorm 代替 LayerNorm，无 bias，ReLU^2 代替 GeLU。因果注意力（只看过去 keys）。这是 decoder‑only transformer 的核心。局限：非向量化，序列长时慢；无 mask（通过累积 keys 实现因果）。
7. 设置优化器（Adam）
learning_rate, beta1, beta2, eps_adam = 1e-2, 0.9, 0.95, 1e-8 m = [0.0] * len(params) # first moment buffer v = [0.0] * len(params) # second moment buffer
- 解释：手动实现 Adam 超参数和缓冲区。m、v 是动量和二阶矩估计（初始化为0）。Adam 是自适应学习率优化器，处理梯度方差。
8. 训练循环
num_steps = 500 # number of training steps
for step in range(num_steps):
# Take single document, tokenize it, surround it with BOS special token on both sides
doc = docs[step % len(docs)]
tokens = [BOS] + [stoi[ch] for ch in doc] + [BOS]
n = min(block_size, len(tokens) - 1)
# Forward the token sequence through the model, building up the computation graph all the way to the loss.
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
losses = []
for pos_id in range(n):
token_id, target_id = tokens[pos_id], tokens[pos_id + 1]
logits = gpt(token_id, pos_id, keys, values)
probs = softmax(logits)
loss_t = -probs[target_id].log()
losses.append(loss_t)
loss = (1 / n) * sum(losses) # final average loss over the document sequence. May yours be low.
# Backward the loss, calculating the gradients with respect to all model parameters.
loss.backward()
# Adam optimizer update: update the model parameters based on the corresponding gradients.
lr_t = learning_rate * (1 - step / num_steps)
for i, p in enumerate(params):
m[i] = beta1 * m[i] + (1 - beta1) * p.grad
v[i] = beta2 * v[i] + (1 - beta2) * p.grad ** 2
m_hat = m[i] / (1 - beta1 ** (step + 1))
v_hat = v[i] / (1 - beta2 ** (step + 1))
p.data -= lr_t * m_hat / (v_hat ** 0.5 + eps_adam)
p.grad = 0
print(f"step {step+1:4d} / {num_steps:4d} | loss {loss.data:.4f}")
- 解释：500 步训练。每步：
- 选一个文档（循环使用），加 BOS 前后，tokenize，截取 max block_size。
- 初始化 keys/values。
- 逐 token 前向：gpt 计算 logits，softmax 得 probs，计算交叉熵损失（‑log(prob[target])）。
- 平均损失。
- backward 计算梯度。
- Adam 更新：学习率线性衰减（cosine annealing 简化），更新 m/v，bias correction，得 m_hat/v_hat，更新参数。重置 grad=0。
- 打印进度和损失。
- 原理：这是自监督 next‑token prediction。损失是 NLL（负对数似然）。每步处理一个序列，类似于 batch_size=1。局限：慢（标量循环），但展示全过程。
9. 推断（Inference）
temperature = 0.6 # in (0, 1], control the "creativity" of generated text, low to high
print("\n--- inference ---")
for sample_idx in range(20):
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
token_id = BOS
print(f"sample {sample_idx+1}: ", end="")
for pos_id in range(block_size):
logits = gpt(token_id, pos_id, keys, values)
probs = softmax([l / temperature for l in logits])
token_id = random.choices(range(vocab_size), weights=[p.data for p in probs])[0]
if token_id == BOS:
break
print(itos[token_id], end="")
print()
- 解释：生成 20 个样本。从 BOS 开始，逐 token 采样：gpt 得 logits，除 temperature（低值更确定，高值更随机），softmax，random.choices 采样下一个 token。遇到 BOS 停止。打印生成的字符串（如名字）。
- 原理：自回归生成。temperature 控制多样性（0 为 argmax，1 为原分布）。这展示了训练后的模型“babble”（胡说），生成类似名字的文本。
总体总结和注意事项
这个代码约 200 行，实现了从零的 GPT：数据到训练到生成。参数少（~数千），数据集小，训练短（500步），所以生成的“名字”可能不完美，但证明了 Transformer 的核心。效率低（纯循环，无 NumPy），适合教学。运行时，损失从 ~3.5 降到 ~2.5，生成如 "maria"、"john" 等。扩展：增加层/维/数据，可接近真实 GPT。如果运行，需注意下载文件；实际输出依赖种子。
▶️ 喜欢这篇内容？创作真的很辛苦！
欢迎关注我们的 YouTube 频道：
AI Rosetta
你的点赞、订阅和分享，是对我们最大的支持！🙏
更多AI新闻请访问：
https://www.airosetta.com