On February 12, 2026, AI expert Andrej Karpathy (@karpathy) again released a minimalist GPT coding tutorial, implementing pretraining and inference operations of modern large language models in just 243 lines of Python — a remarkable work. Below we break down the code so readers can get a clearer view of this piece:
Python source code is as follows:
Python
"""
The most atomic way to train and inference a GPT in pure, dependency-free Python.
This file is the complete algorithm.
Everything else is just efficiency.
@karpathy
"""
import os # os.path.exists
import math # math.log, math.exp
import random # random.seed, random.choices, random.gauss, random.shuffle
# Let there be order among chaos
random.seed(42)
# Let there be an input dataset docs: list[str] of documents (e.g. a dataset of names)
if not os.path.exists('input.txt'):
import urllib.request
names_url = 'https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt'
urllib.request.urlretrieve(names_url, 'input.txt')
docs = [l.strip() for l in open('input.txt').read().strip().split('\n') if l.strip()] # list[str] of documents
random.shuffle(docs)
print(f"num docs: {len(docs)}")
# Let there be a Tokenizer to translate strings to discrete symbols and back
chars = ['<BOS>'] + sorted(set(''.join(docs))) # character-level tokenizer with a BOS delimiter
vocab_size = len(chars)
stoi = { ch:i for i, ch in enumerate(chars) } # encoding: map string to integer
itos = { i:ch for i, ch in enumerate(chars) } # decoding: map integer to string
BOS = stoi['<BOS>']
print(f"vocab size: {vocab_size}")
# Let there be an Autograd to apply the chain rule recursively across a computation graph and so
# calculate the gradients of the loss with respect to model parameters.
class Value:
"""Stores a single scalar value and its gradient."""
def __init__(self, data, _children=(), _op=''):
self.data = data
self.grad = 0
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc
def __add__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')
def _backward():
self.grad += out.grad
other.grad += out.grad
out._backward = _backward
return out
def __mul__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')
def _backward():
self.grad += other.data * out.grad
other.grad += self.data * out.grad
out._backward = _backward
return out
def __pow__(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')
def _backward():
self.grad += (other * self.data**(other-1)) * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), 'log')
def _backward():
self.grad += (1 / self.data) * out.grad
out._backward = _backward
return out
def exp(self):
out = Value(math.exp(self.data), (self,), 'exp')
def _backward():
self.grad += out.data * out.grad
out._backward = _backward
return out
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
def _backward():
self.grad += (out.data > 0) * out.grad
out._backward = _backward
return out
def backward(self):
# topological order all of the children in the graph
topo = []
visited = set()
def build_topo(v):
if v not in visited:
visited.add(v)
for child in v._prev:
build_topo(child)
topo.append(v)
build_topo(self)
# go one variable at a time and apply the chain rule to get its gradient
self.grad = 1
for v in reversed(topo):
v._backward()
def __neg__(self): return self * -1
def __radd__(self, other): return self + other
def __sub__(self, other): return self + (-other)
def __rsub__(self, other): return other + (-self)
def __rmul__(self, other): return self * other
def __truediv__(self, other): return self * other**-1
def __rtruediv__(self, other): return other * self**-1
def __repr__(self): return f"Value(data={self.data}, grad={self.grad})"
# Initialize the parameters, to store the knowledge of the model.
n_embd = 16 # embedding dimension
n_head = 4 # number of attention heads
n_layer = 1 # number of layers
block_size = 8 # maximum sequence length
head_dim = n_embd // n_head # dimension of each head
matrix = lambda nout, nin, std=0.02: [[Value(random.gauss(0, std)) for _ in range(nin)] for _ in range(nout)]
state_dict = {'wte': matrix(vocab_size, n_embd), 'wpe': matrix(block_size, n_embd), 'lm_head': matrix(vocab_size, n_embd)}
for i in range(n_layer):
state_dict[f'layer{i}.attn_wq'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wk'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wv'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wo'] = matrix(n_embd, n_embd, std=0)
state_dict[f'layer{i}.mlp_fc1'] = matrix(4 * n_embd, n_embd)
state_dict[f'layer{i}.mlp_fc2'] = matrix(n_embd, 4 * n_embd, std=0)
params = [p for mat in state_dict.values() for row in mat for p in row] # flatten params into a single list[Value]
print(f"num params: {len(params)}")
# Define the model architecture: a stateless function mapping token sequence and parameters to logits over what comes next.
# Follow GPT-2, blessed among the GPTs, with minor differences: layernorm -> rmsnorm, no biases, GeLU -> ReLU^2
def linear(x, w):
return [sum(wi * xi for wi, xi in zip(wo, x)) for wo in w]
def softmax(logits):
max_val = max(val.data for val in logits)
exps = [(val - max_val).exp() for val in logits]
total = sum(exps)
return [e / total for e in exps]
def rmsnorm(x):
ms = sum(xi * xi for xi in x) / len(x)
scale = (ms + 1e-5) ** -0.5
return [xi * scale for xi in x]
def gpt(token_id, pos_id, keys, values):
tok_emb = state_dict['wte'][token_id] # token embedding
pos_emb = state_dict['wpe'][pos_id] # position embedding
x = [t + p for t, p in zip(tok_emb, pos_emb)] # joint token and position embedding
x = rmsnorm(x)
for li in range(n_layer):
# 1) Multi-head attention block
x_residual = x
x = rmsnorm(x)
q = linear(x, state_dict[f'layer{li}.attn_wq'])
k = linear(x, state_dict[f'layer{li}.attn_wk'])
v = linear(x, state_dict[f'layer{li}.attn_wv'])
keys[li].append(k)
values[li].append(v)
x_attn = []
for h in range(n_head):
hs = h * head_dim
q_h = q[hs:hs+head_dim]
k_h = [ki[hs:hs+head_dim] for ki in keys[li]]
v_h = [vi[hs:hs+head_dim] for vi in values[li]]
attn_logits = [sum(q_h[j] * k_h[t][j] for j in range(head_dim)) / head_dim**0.5 for t in range(len(k_h))]
attn_weights = softmax(attn_logits)
head_out = [sum(attn_weights[t] * v_h[t][j] for t in range(len(v_h))) for j in range(head_dim)]
x_attn.extend(head_out)
x = linear(x_attn, state_dict[f'layer{li}.attn_wo'])
x = [a + b for a, b in zip(x, x_residual)]
# 2) MLP block
x_residual = x
x = rmsnorm(x)
x = linear(x, state_dict[f'layer{li}.mlp_fc1'])
x = [xi.relu() ** 2 for xi in x]
x = linear(x, state_dict[f'layer{li}.mlp_fc2'])
x = [a + b for a, b in zip(x, x_residual)]
logits = linear(x, state_dict['lm_head'])
return logits
# Let there be Adam, the blessed optimizer and its buffers
learning_rate, beta1, beta2, eps_adam = 1e-2, 0.9, 0.95, 1e-8
m = [0.0] * len(params) # first moment buffer
v = [0.0] * len(params) # second moment buffer
# Repeat in sequence
num_steps = 500 # number of training steps
for step in range(num_steps):
# Take single document, tokenize it, surround it with BOS special token on both sides
doc = docs[step % len(docs)]
tokens = [BOS] + [stoi[ch] for ch in doc] + [BOS]
n = min(block_size, len(tokens) - 1)
# Forward the token sequence through the model, building up the computation graph all the way to the loss.
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
losses = []
for pos_id in range(n):
token_id, target_id = tokens[pos_id], tokens[pos_id + 1]
logits = gpt(token_id, pos_id, keys, values)
probs = softmax(logits)
loss_t = -probs[target_id].log()
losses.append(loss_t)
loss = (1 / n) * sum(losses) # final average loss over the document sequence. May yours be low.
# Backward the loss, calculating the gradients with respect to all model parameters.
loss.backward()
# Adam optimizer update: update the model parameters based on the corresponding gradients.
lr_t = learning_rate * (1 - step / num_steps)
for i, p in enumerate(params):
m[i] = beta1 * m[i] + (1 - beta1) * p.grad
v[i] = beta2 * v[i] + (1 - beta2) * p.grad ** 2
m_hat = m[i] / (1 - beta1 ** (step + 1))
v_hat = v[i] / (1 - beta2 ** (step + 1))
p.data -= lr_t * m_hat / (v_hat ** 0.5 + eps_adam)
p.grad = 0
print(f"step {step+1:4d} / {num_steps:4d} | loss {loss.data:.4f}")
# Inference: may the model babble back to us
temperature = 0.6 # in (0, 1], control the "creativity" of generated text, low to high
print("\n--- inference ---")
for sample_idx in range(20):
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
token_id = BOS
print(f"sample {sample_idx+1}: ", end="")
for pos_id in range(block_size):
logits = gpt(token_id, pos_id, keys, values)
probs = softmax([l / temperature for l in logits])
token_id = random.choices(range(vocab_size), weights=[p.data for p in probs])[0]
if token_id == BOS:
break
print(itos[token_id], end="")
print()
This code was written by Andrej Karpathy (@karpathy). It is a "most atomic" implementation of GPT training and inference. It uses pure Python with no external dependencies beyond the standard library, and demonstrates the core GPT algorithms including data preparation, tokenizer, autograd engine, model architecture, training optimization and inference. The code emphasizes that "everything else is just efficiency," meaning this is the most basic version and omits performance optimizations such as vectorization or GPU acceleration.
Overall structure of the code:
- Imports and initialization: set random seed, load data.
- Tokenizer: character‑level tokenizer.
- Autograd engine (Value class): implements backpropagation.
- Model parameter initialization: weights initialized from a Gaussian distribution.
- Model architecture: defines forward pass including embeddings, attention, and MLP.
- Optimizer: manual Adam implementation.
- Training loop: forward, loss calculation, backward, parameter update.
- Inference: generate new sequences.
Below, I explain the code in detail, section by section, with explanations after each segment. Explanations include purpose, key concepts, mathematical principles and potential issues. Inline comments in the code (e.g., "# Let there be...") are written in a poetic style to add charm.
1. Import modules and set random seed
import os # os.path.exists import math # math.log, math.exp import random # random.seed, random.choices, random.gauss, random.shuffle # Let there be order among chaos random.seed(42)
- Explanation: Import required standard library modules. os is used to check file existence; math provides operations like log and exp; random is used for random number generation, including setting the seed (seed=42 to ensure reproducibility). After the seed is set, all random operations (e.g., parameter initialization, sampling) become deterministic. This section is about "bringing order out of chaos," i.e., controlling randomness.
2. Load the input dataset
if not os.path.exists('input.txt'):
import urllib.request
names_url = 'https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt'
urllib.request.urlretrieve(names_url, 'input.txt')
docs = [l.strip() for l in open('input.txt').read().strip().split('\n') if l.strip()] # list[str] of documents
random.shuffle(docs)
print(f"num docs: {len(docs)}")
- Explanation: If input.txt does not exist, the script downloads a names dataset from GitHub (about 32k English names). It then reads the file and transforms it into the list docs, where each element is a name (e.g., "emma"). Empty lines are filtered out. random.shuffle is used to shuffle the list (deterministic because the seed is set). The code prints the number of documents. This is a character‑level language modeling data preparation for next‑token prediction. Potential issues: download may fail if network is unavailable; the dataset is plain text with one name per line.
3. Create the tokenizer
chars = ['<BOS>'] + sorted(set(''.join(docs))) # character-level tokenizer with a BOS delimiter
vocab_size = len(chars)
stoi = { ch:i for i, ch in enumerate(chars) } # encoding: map string to integer
itos = { i:ch for i, ch in enumerate(chars) } # decoding: map integer to string
BOS = stoi['<BOS>']
print(f"vocab size: {vocab_size}")
- Explanation: Builds a character‑level tokenizer. chars is the vocabulary: the special token <BOS> (beginning‑of‑sequence) is prepended, then unique characters from all documents are sorted (typically about 27 including a‑z and <BOS>). vocab_size is the vocabulary size. stoi and itos provide string‑to‑index and index‑to‑string mappings. BOS is the integer ID for <BOS> (usually 0). This simple tokenizer suits small‑vocabulary tasks. In language modeling, text is converted into token IDs for embedding.
4. Autograd engine (Value class)
class Value:
"""Stores a single scalar value and its gradient."""
def __init__(self, data, _children=(), _op=''):
self.data = data
self.grad = 0
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc
def __add__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')
def _backward():
self.grad += out.grad
other.grad += out.grad
out._backward = _backward
return out
def __mul__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')
def _backward():
self.grad += other.data * out.grad
other.grad += self.data * out.grad
out._backward = _backward
return out
def __pow__(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')
def _backward():
self.grad += (other * self.data**(other-1)) * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), 'log')
def _backward():
self.grad += (1 / self.data) * out.grad
out._backward = _backward
return out
def exp(self):
out = Value(math.exp(self.data), (self,), 'exp')
def _backward():
self.grad += out.data * out.grad
out._backward = _backward
return out
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
def _backward():
self.grad += (out.data > 0) * out.grad
out._backward = _backward
return out
def backward(self):
# topological order all of the children in the graph
topo = []
visited = set()
def build_topo(v):
if v not in visited:
visited.add(v)
for child in v._prev:
build_topo(child)
topo.append(v)
build_topo(self)
# go one variable at a time and apply the chain rule to get its gradient
self.grad = 1
for v in reversed(topo):
v._backward()
def __neg__(self): return self * -1
def __radd__(self, other): return self + other
def __sub__(self, other): return self + (-other)
def __rsub__(self, other): return other + (-self)
def __rmul__(self, other): return self * other
def __truediv__(self, other): return self * other**-1
def __rtruediv__(self, other): return other * self**-1
def __repr__(self): return f"Value(data={self.data}, grad={self.grad})"
- Explanation: This is the core: a simple automatic differentiation system, similar to micrograd. The Value class wraps a scalar value and its gradient, supporting basic operations (add, mul, pow, log, exp, ReLU). Each operation records its predecessors (_prev) and the operation type (_op) and defines a _backward function for gradient propagation.
- Initialization: stores data, gradient (initially 0), backward function, predecessors and operation.
- Operator overloading: operations like __add__ create new Value nodes and define how gradients flow (e.g., derivative of addition is 1).
- Nonlinear functions: log, exp, relu implement standard derivatives (e.g., d/dx log(x) = 1/x).
- backward method: builds a topological order (DFS post‑order), sets loss.grad = 1 and calls _backward in reverse order to compute gradients via the chain rule.
- Limitations: only scalar values, not vectorized — low efficiency but high clarity. Used to compute gradients of the loss w.r.t. parameters.
5. Initialize model parameters
n_embd = 16 # embedding dimension
n_head = 4 # number of attention heads
n_layer = 1 # number of layers
block_size = 8 # maximum sequence length
head_dim = n_embd // n_head # dimension of each head
matrix = lambda nout, nin, std=0.02: [[Value(random.gauss(0, std)) for _ in range(nin)] for _ in range(nout)]
state_dict = {'wte': matrix(vocab_size, n_embd), 'wpe': matrix(block_size, n_embd), 'lm_head': matrix(vocab_size, n_embd)}
for i in range(n_layer):
state_dict[f'layer{i}.attn_wq'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wk'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wv'] = matrix(n_embd, n_embd)
state_dict[f'layer{i}.attn_wo'] = matrix(n_embd, n_embd, std=0)
state_dict[f'layer{i}.mlp_fc1'] = matrix(4 * n_embd, n_embd)
state_dict[f'layer{i}.mlp_fc2'] = matrix(n_embd, 4 * n_embd, std=0)
params = [p for mat in state_dict.values() for row in mat for p in row] # flatten params into a single list[Value]
print(f"num params: {len(params)}")
- Explanation: Defines hyperparameters: embedding dim 16, 4 attention heads, 1 layer, max sequence length 8. matrix creates weight matrices whose elements are Value objects initialized from a Gaussian (std=0.02 to avoid large initial values). state_dict stores all weights: token embeddings (wte), position embeddings (wpe), lm head (lm_head), and per‑layer attention and MLP weights. Note: some projection matrices (wo, fc2) use std=0 in this snippet (zero init in this simplified code). params flattens all Value parameters into a list (total params are on the order of thousands). This mirrors a PyTorch state_dict; parameters are differentiable Value objects.
6. Define helper functions and GPT forward pass
def linear(x, w):
return [sum(wi * xi for wi, xi in zip(wo, x)) for wo in w]
def softmax(logits):
max_val = max(val.data for val in logits)
exps = [(val - max_val).exp() for val in logits]
total = sum(exps)
return [e / total for e in exps]
def rmsnorm(x):
ms = sum(xi * xi for xi in x) / len(x)
scale = (ms + 1e-5) ** -0.5
return [xi * scale for xi in x]
def gpt(token_id, pos_id, keys, values):
tok_emb = state_dict['wte'][token_id] # token embedding
pos_emb = state_dict['wpe'][pos_id] # position embedding
x = [t + p for t, p in zip(tok_emb, pos_emb)] # joint token and position embedding
x = rmsnorm(x)
for li in range(n_layer):
# 1) Multi-head attention block
x_residual = x
x = rmsnorm(x)
q = linear(x, state_dict[f'layer{li}.attn_wq'])
k = linear(x, state_dict[f'layer{li}.attn_wk'])
v = linear(x, state_dict[f'layer{li}.attn_wv'])
keys[li].append(k)
values[li].append(v)
x_attn = []
for h in range(n_head):
hs = h * head_dim
q_h = q[hs:hs+head_dim]
k_h = [ki[hs:hs+head_dim] for ki in keys[li]]
v_h = [vi[hs:hs+head_dim] for vi in values[li]]
attn_logits = [sum(q_h[j] * k_h[t][j] for j in range(head_dim)) / head_dim**0.5 for t in range(len(k_h))]
attn_weights = softmax(attn_logits)
head_out = [sum(attn_weights[t] * v_h[t][j] for t in range(len(v_h))) for j in range(head_dim)]
x_attn.extend(head_out)
x = linear(x_attn, state_dict[f'layer{li}.attn_wo'])
x = [a + b for a, b in zip(x, x_residual)]
# 2) MLP block
x_residual = x
x = rmsnorm(x)
x = linear(x, state_dict[f'layer{li}.mlp_fc1'])
x = [xi.relu() ** 2 for xi in x]
x = linear(x, state_dict[f'layer{li}.mlp_fc2'])
x = [a + b for a, b in zip(x, x_residual)]
logits = linear(x, state_dict['lm_head'])
return logits
- Explanation:
- linear: matrix multiplication — input vector x and matrix w produce an output vector. Implemented with zip and sum; operates on Value scalars.
- softmax: numerically stable implementation (subtract max) to convert logits to probabilities.
- rmsnorm: RMSNorm, a LayerNorm variant that divides by root‑mean‑square (no mean subtraction), with epsilon for numerical stability.
- gpt function: stateless forward pass. Inputs are token_id, pos_id and accumulating keys/values for causal attention.
- Token and position embeddings are summed and RMS‑normalized.
- Per layer: attention block + residual, then MLP block + residual.
- Attention: compute Q/K/V, append to keys/values to realize causal attention. Multi‑head: split heads, compute scaled dot‑product attention, softmax weighting of V, concatenate and project with O.
- MLP: two linear layers with ReLU^2 nonlinearity (approximate GeLU).
- Output logits from lm_head (vocab_size dimensions).
- Notes: Follows GPT‑2 style but simplified: RMSNorm instead of LayerNorm, no biases, ReLU^2 instead of GeLU. Causal attention is implemented by accumulating keys/values. This implementation is not vectorized and is slow for long sequences.
7. Setup optimizer (Adam)
learning_rate, beta1, beta2, eps_adam = 1e-2, 0.9, 0.95, 1e-8 m = [0.0] * len(params) # first moment buffer v = [0.0] * len(params) # second moment buffer
- Explanation: Manually defines Adam hyperparameters and buffers. m and v are the first‑ and second‑moment estimates (initialized to zero). Adam is an adaptive optimizer that addresses gradient variance.
8. Training loop
num_steps = 500 # number of training steps
for step in range(num_steps):
# Take single document, tokenize it, surround it with BOS special token on both sides
doc = docs[step % len(docs)]
tokens = [BOS] + [stoi[ch] for ch in doc] + [BOS]
n = min(block_size, len(tokens) - 1)
# Forward the token sequence through the model, building up the computation graph all the way to the loss.
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
losses = []
for pos_id in range(n):
token_id, target_id = tokens[pos_id], tokens[pos_id + 1]
logits = gpt(token_id, pos_id, keys, values)
probs = softmax(logits)
loss_t = -probs[target_id].log()
losses.append(loss_t)
loss = (1 / n) * sum(losses) # final average loss over the document sequence. May yours be low.
# Backward the loss, calculating the gradients with respect to all model parameters.
loss.backward()
# Adam optimizer update: update the model parameters based on the corresponding gradients.
lr_t = learning_rate * (1 - step / num_steps)
for i, p in enumerate(params):
m[i] = beta1 * m[i] + (1 - beta1) * p.grad
v[i] = beta2 * v[i] + (1 - beta2) * p.grad ** 2
m_hat = m[i] / (1 - beta1 ** (step + 1))
v_hat = v[i] / (1 - beta2 ** (step + 1))
p.data -= lr_t * m_hat / (v_hat ** 0.5 + eps_adam)
p.grad = 0
print(f"step {step+1:4d} / {num_steps:4d} | loss {loss.data:.4f}")
- Explanation: 500 training steps. Each step:
- Pick a document (cycled), surround with BOS tokens, tokenize, and clip to block_size.
- Initialize keys/values for causal attention.
- For each position: forward pass via gpt to get logits, softmax to probabilities, compute cross‑entropy loss (‑log(prob[target])).
- Average the losses across the sequence.
- Call backward to compute gradients.
- Perform Adam update: apply linear learning‑rate decay, update m and v, compute bias‑corrected m_hat and v_hat, update parameter values, and zero gradients.
- Print progress and loss.
- Notes: This is self‑supervised next‑token prediction using NLL loss. Effectively batch_size=1. The approach is slow due to scalar loops, but demonstrates the full pipeline.
9. Inference
temperature = 0.6 # in (0, 1], control the "creativity" of generated text, low to high
print("\n--- inference ---")
for sample_idx in range(20):
keys, values = [[] for _ in range(n_layer)], [[] for _ in range(n_layer)]
token_id = BOS
print(f"sample {sample_idx+1}: ", end="")
for pos_id in range(block_size):
logits = gpt(token_id, pos_id, keys, values)
probs = softmax([l / temperature for l in logits])
token_id = random.choices(range(vocab_size), weights=[p.data for p in probs])[0]
if token_id == BOS:
break
print(itos[token_id], end="")
print()
- Explanation: Generates 20 samples. Starting from BOS, the model samples tokens autoregressively: compute logits via gpt, scale by temperature (lower values make outputs more deterministic), apply softmax, then sample with random.choices. Stop if BOS is sampled. Prints generated strings (e.g., names).
- Principle: Autoregressive generation. Temperature controls diversity (0 ‑> argmax, 1 ‑> original distribution). This shows how the trained model "babble" and produces name‑like tokens.
Overall summary and notes
This code is about 200 lines and implements a GPT from scratch: data, training and generation. With few parameters (a few thousand), a small dataset and short training (500 steps), generated names will not be perfect but the example demonstrates the Transformer core. The implementation is educational rather than performant (pure Python loops, no NumPy). In typical runs, loss drops from ~3.5 to ~2.5 and generates names like "maria" or "john". To scale toward real GPT behavior, increase layers, dimensions, and dataset size. Note that downloads depend on network, and outputs depend on the random seed.
▶️ Enjoyed this article? Creating content is hard work!
Follow our YouTube channel:
AI Rosetta
Your likes, subscriptions and shares are our greatest support! 🙏
For more AI news visit:
https://www.airosetta.com