2026年2月12日，AI 大神 Andrej Karpathy（@karpathy）再次發布極簡嘅GPT 編碼教程，只用 243 行 Python 代碼就實現咗而家 AI 大語言模型嘅預訓練同推理流程，堪稱神作。下面我哋逐段拆解呢份代碼，令讀者可以更清晰咁理解呢件傑作：
Python 源碼如下：
Python
"""
The most atomic way to train and inference a GPT in pure, dependency-free Python.
This file is the complete algorithm.
Everything else is just efficiency.
@karpathy
"""
import os # os.path.exists
import math # math.log, math.exp
import random # random.seed, random.choices, random.gauss, random.shuffle
# Let there be order among chaos
random.seed(42)
# Let there be an input dataset docs: list[str] of documents (e.g. a dataset of names)
if not os.path.exists('input.txt'):
import urllib.request
names_url = 'https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt'
urllib.request.urlretrieve(names_url, 'input.txt')
docs = [l.strip() for l in open('input.txt').read().strip().split('\n') if l.strip()] # list[str] of documents
random.shuffle(docs)
print(f"num docs: {len(docs)}")
# Let there be a Tokenizer to translate strings to discrete symbols and back
chars = ['<BOS>'] + sorted(set(''.join(docs))) # character-level tokenizer with a BOS delimiter
vocab_size = len(chars)
stoi = { ch:i for i, ch in enumerate(chars) } # encoding: map string to integer
itos = { i:ch for i, ch in enumerate(chars) } # decoding: map integer to string
BOS = stoi['<BOS>']
print(f"vocab size: {vocab_size}")
# Let there be an Autograd to apply the chain rule recursively across a computation graph and so
# calculate the gradients of the loss with respect to model parameters.
class Value:
"""Stores a single scalar value and its gradient."""
def __init__(self, data, _children=(), _op=''):
self.data = data
self.grad = 0
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc
def __add__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')
def _backward():
self.grad += out.grad
other.grad += out.grad
out._backward = _backward
return out
def __mul__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')
def _backward():
self.grad += other.data * out.grad
other.grad += self.data * out.grad
out._backward = _backward
return out
def __pow__(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')
def _backward():
self.grad += (other * self.data**(other-1)) * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), 'log')
def _backward():
self.grad += (1 / self.data) * out.grad
out._backward = _backward
return out
def exp(self):
out = Value(math.exp(self.data), (self,), 'exp')
def _backward():
self.grad += out.data * out.grad
out._backward = _backward
return out
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
def _backward():
self.grad += (out.data > 0) * out.grad
out._backward = _backward
return out
def backward(self):
# topological order all of the children in the graph
topo = []
visited = set()
def build_topo(v):
if v not in visited:
visited.add(v)
for child in v._prev:
build_topo(child)
topo.append(v)
build_topo(self)
# go one variable at a time and apply the chain rule to get its gradient
self.grad = 1
for v in reversed(topo):
v._backward()
def __neg__(self): return self * -1
def __radd__(self, other): return self + other
def __sub__(self, other): return self + (-other)
def __rsub__(self, other): return other + (-self)
def __rmul__(self, other): return self * other
def __truediv__(self, other): return self * other**-1
def __rtruediv__(self, other): return other * self**-1
def __repr__(self): return f"Value(data={self.data}, grad={self.grad})"
# Initialize the parameters, to store the knowledge of the model.
n_embd = 16 # embedding dimension
n_head = 4 # number of attention heads
n_layer = 1 # number of layers
block_size = 8 # maximum sequence length
head_dim = n_embd // n_head # dimension of each head
matrix = lambda nout, nin, std=0.02: [[Value(random.gauss(0, std)) for _ in range(nin)] for _ in range(nout)]
state_dict = {'wte': matrix(vocab_size, n_embd), 'wpe': matrix(block_size, n_embd), 'lm_head': matrix(vocab_size, n_embd)}
for i in range(n_layer):
state_dict[f'layer{i}.attn_wq'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wk'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wv'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wo'] = matrix(n_embd, n_embd, std=0)
state_dict[f'layer{i}.mlp_fc1'] = matrix(4 * n_embd, n_embd)
state_dict[f'layer{i}.mlp_fc2'] = matrix(n_embd, 4 * n_embd, std=0)
params = [p for mat in state_dict.values() for row in mat for p in row] # flatten params into a single list[Value]
print(f"num params: {len(params)}")
# Define the model architecture: a stateless function mapping token sequence and parameters to logits over what comes next.
# Follow GPT-2, blessed among the GPTs, with minor differences: layernorm -> rmsnorm, no biases, GeLU -> ReLU^2
def linear(x, w):
return [sum(wi * xi for wi, xi in zip(wo, x)) for wo in w]
def softmax(logits):
max_val = max(val.data for val in logits)
exps = [(val - max_val).exp() for val in logits]
total = sum(exps)
return [e / total for e in exps]
def rmsnorm(x):
ms = sum(xi * xi for xi in x) / len(x)
scale = (ms + 1e-5) ** -0.5
return [xi * scale for xi in x]
def gpt(token_id, pos_id, keys, values):
tok_emb = state_dict['wte'][token_id] # token embedding
pos_emb = state_dict['wpe'][pos_id] # position embedding
x = [t + p for t, p in zip(tok_emb, pos_emb)] # joint token and position embedding
x = rmsnorm(x)
for li in range(n_layer):
# 1) Multi-head attention block
x_residual = x
x = rmsnorm(x)
q = linear(x, state_dict[f'layer{li}.attn_wq'])
k = linear(x, state_dict[f'layer{li}.attn_wk'])
v = linear(x, state_dict[f'layer{li}.attn_wv'])
keys[li].append(k)
values[li].append(v)
x_attn = []
for h in range(n_head):
hs = h * head_dim
q_h = q[hs:hs+head_dim]
k_h = [ki[hs:hs+head_dim] for ki in keys[li]]
v_h = [vi[hs:hs+head_dim] for vi in values[li]]
attn_logits = [sum(q_h[j] * k_h[t][j] for j in range(head_dim)) / head_dim**0.5 for t in range(len(k_h))]
attn_weights = softmax(attn_logits)
head_out = [sum(attn_weights[t] * v_h[t][j] for t in range(len(v_h))) for j in range(head_dim)]
x_attn.extend(head_out)
x = linear(x_attn, state_dict[f'layer{li}.attn_wo'])
x = [a + b for a, b in zip(x, x_residual)]
# 2) MLP block
x_residual = x
x = rmsnorm(x)
x = linear(x, state_dict[f'layer{li}.mlp_fc1'])
x = [xi.relu() ** 2 for xi in x]
x = linear(x, state_dict[f'layer{li}.mlp_fc2'])
x = [a + b for a, b in zip(x, x_residual)]
logits = linear(x, state_dict['lm_head'])
return logits
# Let there be Adam, the blessed optimizer and its buffers
learning_rate, beta1, beta2, eps_adam = 1e-2, 0.9, 0.95, 1e-8
m = [0.0] * len(params) # first moment buffer
v = [0.0] * len(params) # second moment buffer
# Repeat in sequence
num_steps = 500 # number of training steps
for step in range(num_steps):
# Take single document, tokenize it, surround it with BOS special token on both sides
doc = docs[step % len(docs)]
tokens = [BOS] + [stoi[ch] for ch in doc] + [BOS]
n = min(block_size, len(tokens) - 1)
# Forward the token sequence through the model, building up the computation graph all the way to the loss.
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
losses = []
for pos_id in range(n):
token_id, target_id = tokens[pos_id], tokens[pos_id + 1]
logits = gpt(token_id, pos_id, keys, values)
probs = softmax(logits)
loss_t = -probs[target_id].log()
losses.append(loss_t)
loss = (1 / n) * sum(losses) # final average loss over the document sequence. May yours be low.
# Backward the loss, calculating the gradients with respect to all model parameters.
loss.backward()
# Adam optimizer update: update the model parameters based on the corresponding gradients.
lr_t = learning_rate * (1 - step / num_steps)
for i, p in enumerate(params):
m[i] = beta1 * m[i] + (1 - beta1) * p.grad
v[i] = beta2 * v[i] + (1 - beta2) * p.grad ** 2
m_hat = m[i] / (1 - beta1 ** (step + 1))
v_hat = v[i] / (1 - beta2 ** (step + 1))
p.data -= lr_t * m_hat / (v_hat ** 0.5 + eps_adam)
p.grad = 0
print(f"step {step+1:4d} / {num_steps:4d} | loss {loss.data:.4f}")
# Inference: may the model babble back to us
temperature = 0.6 # in (0, 1], control the "creativity" of generated text, low to high
print("\n--- inference ---")
for sample_idx in range(20):
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
token_id = BOS
print(f"sample {sample_idx+1}: ", end="")
for pos_id in range(block_size):
logits = gpt(token_id, pos_id, keys, values)
probs = softmax([l / temperature for l in logits])
token_id = random.choices(range(vocab_size), weights=[p.data for p in probs])[0]
if token_id == BOS:
break
print(itos[token_id], end="")
print()
呢份代碼由 Andrej Karpathy（@karpathy）編寫，係一個「最原子化」（most atomic）嘅 GPT 模型訓練同推斷實現。佢用純 Python、無外部依賴（只用標準庫），展示咗 GPT 嘅核心算法，包括數據準備、分詞器、自動求導引擎、模型架構、訓練優化同推斷生成。代碼強調「一切其他都係效率優化」，意思係呢個版本係最基礎嘅，忽略咗例如向量化或者 GPU 加速等性能優化。
代碼整體結構：
- 導入同初始化：設定隨機種子、載入數據。
- 分詞器：字符級 tokenizer。
- 自動求導引擎（Value 類）：實現反向傳播。
- 模型參數初始化：用高斯分佈初始化權重。
- 模型架構：定義前向傳播函數，包括嵌入、注意力、MLP 等。
- 優化器：手動實現 Adam。
- 訓練循環：前向、損失計算、反向傳播、參數更新。
- 推斷：生成新序列。
下面，我會逐段詳細解釋代碼，每段之後附上說明。說明會涵蓋作用、關鍵概念、數學原理同潛在問題。代碼入面嘅註釋（例如 “# Let there be...”）係詩意化嘅，增加咗趣味性。
1. 導入模塊同設定隨機種子
import os # os.path.exists import math # math.log, math.exp import random # random.seed, random.choices, random.gauss, random.shuffle # Let there be order among chaos random.seed(42)
- 解釋：導入必要嘅標準庫模塊。os 用嚟檢查文件是否存在；math 用於數學運算例如對數同指數；random 用於隨機數生成，包括設定種子（seed=42，確保可復現）。設定種子之後，所有隨機操作（例如參數初始化、採樣）都會相同。呢部分係為咗「由混沌帶嚟秩序」，即係控制隨機性。
2. 載入輸入數據集
if not os.path.exists('input.txt'):
import urllib.request
names_url = 'https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt'
urllib.request.urlretrieve(names_url, 'input.txt')
docs = [l.strip() for l in open('input.txt').read().strip().split('\n') if l.strip()] # list[str] of documents
random.shuffle(docs)
print(f"num docs: {len(docs)}")
- 解釋：如果 input.txt 唔存在，就會由 GitHub 下載一個名字數據集（大約 32k 行英文名）。然後讀取文件，處理成 docs 列表，每個元素係一個名字（例如 "emma"）。用列表推導式過濾空行，並用 random.shuffle 打亂順序（因為已設定種子，操作可復現）。打印文檔數量（通常為 32033）。呢個係字符級語言建模任務嘅數據準備，類似 next‑token prediction。潛在問題：如果網絡不可用，下載會失敗；實際運行時，數據集係純文本，每行一個名字。
3. 創建分詞器（Tokenizer）
chars = ['<BOS>'] + sorted(set(''.join(docs))) # character-level tokenizer with a BOS delimiter
vocab_size = len(chars)
stoi = { ch:i for i, ch in enumerate(chars) } # encoding: map string to integer
itos = { i:ch for i, ch in enumerate(chars) } # decoding: map integer to string
BOS = stoi['<BOS>']
print(f"vocab size: {vocab_size}")
- 解釋：構建字符級分詞器。chars 係詞彙表：加入特殊 token <BOS>（Beginning of Sequence，開始序列標記），然後由所有文檔提取唯一字符並排序（通常有 27 個字符，包括 a‑z 同 <BOS>）。vocab_size 係詞彙大小（27）。stoi 同 itos 分別係字符串到整數、整數到字符串嘅映射字典。BOS 係 <BOS> 嘅整數 ID（0）。呢個係一個簡單嘅字符 tokenizer，適合細詞彙表任務。打印詞彙大小。原理：語言建模時要將文本轉為 token ID 序列，方便嵌入處理。
4. 自動求導引擎（Value 類）
class Value:
"""Stores a single scalar value and its gradient."""
def __init__(self, data, _children=(), _op=''):
self.data = data
self.grad = 0
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc
def __add__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')
def _backward():
self.grad += out.grad
other.grad += out.grad
out._backward = _backward
return out
def __mul__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')
def _backward():
self.grad += other.data * out.grad
other.grad += self.data * out.grad
out._backward = _backward
return out
def __pow__(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')
def _backward():
self.grad += (other * self.data**(other-1)) * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), 'log')
def _backward():
self.grad += (1 / self.data) * out.grad
out._backward = _backward
return out
def exp(self):
out = Value(math.exp(self.data), (self,), 'exp')
def _backward():
self.grad += out.data * out.grad
out._backward = _backward
return out
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
def _backward():
self.grad += (out.data > 0) * out.grad
out._backward = _backward
return out
def backward(self):
# topological order all of the children in the graph
topo = []
visited = set()
def build_topo(v):
if v not in visited:
visited.add(v)
for child in v._prev:
build_topo(child)
topo.append(v)
build_topo(self)
# go one variable at a time and apply the chain rule to get its gradient
self.grad = 1
for v in reversed(topo):
v._backward()
def __neg__(self): return self * -1
def __radd__(self, other): return self + other
def __sub__(self, other): return self + (-other)
def __rsub__(self, other): return other + (-self)
def __rmul__(self, other): return self * other
def __truediv__(self, other): return self * other**-1
def __rtruediv__(self, other): return other * self**-1
def __repr__(self): return f"Value(data={self.data}, grad={self.grad})"
- 解釋：呢個係代碼嘅核心：一個簡單嘅自動求導系統，類似於 micrograd（Karpathy 之前嘅項目）。Value 類封裝咗標量值同梯度，支持基本運算（例如加、乘、冪、log、exp、ReLU）。每個運算會記錄前驅節點（_prev）同操作符（_op），並定義 _backward 函數，用嚟根據鏈式法則計算梯度。
- 初始化：存儲 data、grad（初始 0）、反向函數、前驅同操作。
- 運算重載：例如 __add__ 會建立新嘅 Value，並定義點樣傳播梯度（例如加法嘅梯度係 1）。
- 非線性函數：log、exp、relu 嘅反向係標準導數（例如 log 嘅導數係 1/x）。
- backward 方法：構建拓撲排序（DFS 後序遍歷），由損失開始（grad=1），逆序調用 _backward 計算所有梯度。
- 其他運算：支持負、減、除等，通過重載實現。
- 數學原理：呢個係計算圖嘅反向傳播。每個 Value 係圖節點，運算構建圖，backward 應用鏈式法則。局限係只支持標量，冇向量化，效率低，但易睇易教。用嚟計算損失對參數嘅梯度。
5. 初始化模型參數
n_embd = 16 # embedding dimension
n_head = 4 # number of attention heads
n_layer = 1 # number of layers
block_size = 8 # maximum sequence length
head_dim = n_embd // n_head # dimension of each head
matrix = lambda nout, nin, std=0.02: [[Value(random.gauss(0, std)) for _ in range(nin)] for _ in range(nout)]
state_dict = {'wte': matrix(vocab_size, n_embd), 'wpe': matrix(block_size, n_embd), 'lm_head': matrix(vocab_size, n_embd)}
for i in range(n_layer):
state_dict[f'layer{i}.attn_wq'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wk'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wv'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wo'] = matrix(n_embd, n_embd, std=0)
state_dict[f'layer{i}.mlp_fc1'] = matrix(4 * n_embd, n_embd)
state_dict[f'layer{i}.mlp_fc2'] = matrix(n_embd, 4 * n_embd, std=0)
params = [p for mat in state_dict.values() for row in mat for p in row] # flatten params into a single list[Value]
print(f"num params: {len(params)}")
- 解釋：定義超參數：嵌入維度 n_embd=16、注意力頭數 n_head=4、層數 n_layer=1、最大序列長 block_size=8。head_dim 係每個頭嘅維度。matrix 函數用嚟創建權重矩陣，每個元素係 Value，使用高斯初始化（std=0.02，避免梯度爆炸）。state_dict 係個字典存放所有權重：詞嵌入（wte）、位置嵌入（wpe）、LM 頭（lm_head），以及每層嘅注意力（QKV、O）同 MLP（fc1、fc2）。注意：某啲投影層（例如 wo、fc2）用咗 std=0（零初始化？代碼入面寫咗 std=0，可能係示例簡化）。params 會將所有 Value 展平成一個列表（總參數數量約幾千）。打印參數數量。呢個設計模擬咗 PyTorch 嘅 state_dict，參數係可微嘅 Value。
6. 定義模型輔助函數同 GPT 前向傳播
def linear(x, w):
return [sum(wi * xi for wi, xi in zip(wo, x)) for wo in w]
def softmax(logits):
max_val = max(val.data for val in logits)
exps = [(val - max_val).exp() for val in logits]
total = sum(exps)
return [e / total for e in exps]
def rmsnorm(x):
ms = sum(xi * xi for xi in x) / len(x)
scale = (ms + 1e-5) ** -0.5
return [xi * scale for xi in x]
def gpt(token_id, pos_id, keys, values):
tok_emb = state_dict['wte'][token_id] # token embedding
pos_emb = state_dict['wpe'][pos_id] # position embedding
x = [t + p for t, p in zip(tok_emb, pos_emb)] # joint token and position embedding
x = rmsnorm(x)
for li in range(n_layer):
# 1) Multi-head attention block
x_residual = x
x = rmsnorm(x)
q = linear(x, state_dict[f'layer{li}.attn_wq'])
k = linear(x, state_dict[f'layer{li}.attn_wk'])
v = linear(x, state_dict[f'layer{li}.attn_wv'])
keys[li].append(k)
values[li].append(v)
x_attn = []
for h in range(n_head):
hs = h * head_dim
q_h = q[hs:hs+head_dim]
k_h = [ki[hs:hs+head_dim] for ki in keys[li]]
v_h = [vi[hs:hs+head_dim] for vi in values[li]]
attn_logits = [sum(q_h[j] * k_h[t][j] for j in range(head_dim)) / head_dim**0.5 for t in range(len(k_h))]
attn_weights = softmax(attn_logits)
head_out = [sum(attn_weights[t] * v_h[t][j] for t in range(len(v_h))) for j in range(head_dim)]
x_attn.extend(head_out)
x = linear(x_attn, state_dict[f'layer{li}.attn_wo'])
x = [a + b for a, b in zip(x, x_residual)]
# 2) MLP block
x_residual = x
x = rmsnorm(x)
x = linear(x, state_dict[f'layer{li}.mlp_fc1'])
x = [xi.relu() ** 2 for xi in x]
x = linear(x, state_dict[f'layer{li}.mlp_fc2'])
x = [a + b for a, b in zip(x, x_residual)]
logits = linear(x, state_dict['lm_head'])
return logits
- 解釋：
- linear：矩陣乘法，輸入 x（向量）同 w（矩陣），輸出新向量。用 zip 同 sum 實現標量乘加。
- softmax：用穩定計算（先減 max 避免溢出），將 logits 轉成概率分布。
- rmsnorm：RMSNorm（LayerNorm 嘅變體，無減均值，只除以 RMS，並加入 epsilon 防止除零）。用於穩定訓練。
- gpt 函數：無狀態前向函數。輸入 token_id、pos_id，同累積嘅 keys/values（用嚟實現因果注意力）。
- 取 token 同 position 嵌入，相加，再做 RMSNorm。
- 每層包含：注意力 + 殘差 + MLP + 殘差。
- 注意力：計算 Q/K/V，將其追加到 keys/values（實現自回歸），多頭切分後計算 dot‑product attention（縮放因子 1/sqrt(d)），用 softmax 加權 V，concat 後再投影 O。
- MLP：兩層線性，中間用 ReLU^2（近似 GeLU）。
- 最後用 LM 頭線性，輸出 logits（vocab_size 維）。
- 數學原理：沿用 GPT‑2 架構，但簡化咗：RMSNorm 代替 LayerNorm、無 bias、ReLU^2 代替 GeLU。採用因果注意力（只參考過去 tokens）。局限係無向量化，序列長嘅時候會好慢；未顯式寫 mask（透過累積 keys 實現因果性）。
7. 設定優化器（Adam）
learning_rate, beta1, beta2, eps_adam = 1e-2, 0.9, 0.95, 1e-8 m = [0.0] * len(params) # first moment buffer v = [0.0] * len(params) # second moment buffer
- 解釋：手動實現 Adam 嘅超參數同緩衝區。m、v 分別係一階動量同二階矩估計（初始化為 0）。Adam 係一個自適應學習率優化器，可以處理梯度嘅變化。
8. 訓練循環
num_steps = 500 # number of training steps
for step in range(num_steps):
# Take single document, tokenize it, surround it with BOS special token on both sides
doc = docs[step % len(docs)]
tokens = [BOS] + [stoi[ch] for ch in doc] + [BOS]
n = min(block_size, len(tokens) - 1)
# Forward the token sequence through the model, building up the computation graph all the way to the loss.
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
losses = []
for pos_id in range(n):
token_id, target_id = tokens[pos_id], tokens[pos_id + 1]
logits = gpt(token_id, pos_id, keys, values)
probs = softmax(logits)
loss_t = -probs[target_id].log()
losses.append(loss_t)
loss = (1 / n) * sum(losses) # final average loss over the document sequence. May yours be low.
# Backward the loss, calculating the gradients with respect to all model parameters.
loss.backward()
# Adam optimizer update: update the model parameters based on the corresponding gradients.
lr_t = learning_rate * (1 - step / num_steps)
for i, p in enumerate(params):
m[i] = beta1 * m[i] + (1 - beta1) * p.grad
v[i] = beta2 * v[i] + (1 - beta2) * p.grad ** 2
m_hat = m[i] / (1 - beta1 ** (step + 1))
v_hat = v[i] / (1 - beta2 ** (step + 1))
p.data -= lr_t * m_hat / (v_hat ** 0.5 + eps_adam)
p.grad = 0
print(f"step {step+1:4d} / {num_steps:4d} | loss {loss.data:.4f}")
- 解釋：訓練做 500 步。每步嘅流程：
- 選一個文檔（循環使用），加上前後 BOS，tokenize，並截取到最大 block_size。
- 初始化 keys/values。
- 逐個 token 做前向：用 gpt 得到 logits，softmax 得到 probs，再計算交叉熵損失（‑log(prob[target])）。
- 對序列做平均損失。
- 調用 backward 計算梯度。
- 用 Adam 更新參數：學習率線性衰減，更新 m/v，做 bias‑correction 得到 m_hat/v_hat，然後更新參數，並將 p.grad 清零。
- 打印訓練進度同損失。
- 原理：呢個係自監督嘅 next‑token prediction。損失係 NLL（負對數似然）。每步處理一個序列，等同 batch_size=1。局限係計算慢（純標量循環），但示範咗整個流程。
9. 推斷（Inference）
temperature = 0.6 # in (0, 1], control the "creativity" of generated text, low to high
print("\n--- inference ---")
for sample_idx in range(20):
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
token_id = BOS
print(f"sample {sample_idx+1}: ", end="")
for pos_id in range(block_size):
logits = gpt(token_id, pos_id, keys, values)
probs = softmax([l / temperature for l in logits])
token_id = random.choices(range(vocab_size), weights=[p.data for p in probs])[0]
if token_id == BOS:
break
print(itos[token_id], end="")
print()
- 解釋：生成 20 個樣本。由 BOS 開始，逐個 token 采樣：gpt 得到 logits，除以 temperature（低值令生成更確定，高值更隨機），softmax，再用 random.choices 采樣下一個 token。若遇到 BOS 就停止。打印生成嘅字符串（例如人名）。
- 原理：自回歸生成。temperature 控制多樣性（0 近似取最大概率，1 係原分布）。呢部分展示咗訓練後模型點樣「babble」出類似名字嘅文本。
整體總結同注意事項
呢份代碼大約 200 行，從零開始實現咗一個 GPT：從數據到訓練再到生成。參數少（大約幾千）、數據細、訓練步數短（500 步），所以生成出嚟嘅「名字」未必完美，但證明咗 Transformer 嘅核心原理。效率低（純 Python 循環，唔用 NumPy 或 GPU），係以教學為主。運行時損失可能由約 3.5 降到 2.5，生成例如 "maria"、"john" 咁嘅名字。想擴展模型可以加多層、多啲維度或更多數據，就可以更貼近真實 GPT。如果運行，記住要注意下載步驟；實際輸出亦會受隨機種子影響。
▶️ 鍾意呢篇內容？創作真係好辛苦！
歡迎追蹤我哋嘅 YouTube 頻道：
AI Rosetta
你嘅讚好、訂閱同分享，係對我哋最大嘅支持！🙏
更多 AI 新聞請瀏覽：
https://www.airosetta.com