19:I[15621,["/_next/static/immutable/chunks/3gm-8e62xn1qd.js","/_next/static/immutable/chunks/0n2yq6f-f75k2.js"],"ArticleReader"] 1a:T8b5,

# hardcoded BF16 peak flops for various GPUs
# inspired by torchtitan: https://github.com/pytorch/torchtitan/blob/main/torchtitan/tools/utils.py
# and PR: https://github.com/karpathy/nanochat/pull/147
def get_peak_flops(device_name: str) -> float:
    name = device_name.lower()

    # --- NVIDIA Blackwell ---
    if "gb200" in name or "grace blackwell" in name:
        return 2.5e15
    if "b200" in name:
        return 2.25e15
    if "b100" in name:
        return 1.8e15

    # --- NVIDIA Hopper (H100/H200/H800) ---
    if "h200" in name:
        if "nvl" in name or "pcie" in name:
            return 836e12
        return 989e12  # H200 SXM
    if "h100" in name:
        if "nvl" in name:
            return 835e12
        if "pcie" in name:
            return 756e12
        return 989e12  # H100 SXM
    if "h800" in name:
        if "nvl" in name:
            return 989e12
        return 756e12  # H800 PCIe

    # --- NVIDIA Ampere data center ---
    if "a100" in name or "a800" in name:
        return 312e12
    if "a40" in name:
        return 149.7e12
    if "a30" in name:
        return 165e12

    # --- NVIDIA Ada data center ---
    if "l40s" in name or "l40-s" in name or "l40 s" in name:
        return 362e12
    if "l4" in name:
        return 121e12

    # --- AMD CDNA accelerators ---
    if "mi355" in name:
        return 2.5e15
    if "mi325" in name or "mi300x" in name:
        return 1.3074e15
    if "mi300a" in name:
        return 980.6e12
    if "mi250x" in name:
        return 383e12
    if "mi250" in name:
        return 362.1e12

    # --- Intel ---
    if "data center gpu max 1550" in name:
        # Ponte Vecchio (PVC) - dynamic based on compute units
        max_comp_units = torch.xpu.get_device_properties("xpu").max_compute_units
        return 512 * max_comp_units * 1300 * 10**6

    # --- Consumer RTX (for hobbyists) ---
    if "5090" in name:
        return 209.5e12
    if "4090" in name:
        return 165.2e12
    if "3090" in name:
        return 71e12

    # Unknown GPU - return inf so MFU shows as 0% rather than a wrong guess
    logger.warning(f"Peak flops undefined for: {device_name}, MFU will show as 0%")
    return float('inf')

1b:T713,# hardcoded BF16 peak flops for various GPUs # inspired by torchtitan: https://github.com/pytorch/torchtitan/blob/main/torchtitan/tools/utils.py # and PR: https://github.com/karpathy/nanochat/pull/147 def get_peak_flops(device_name: str) -> float: name = device_name.lower() # --- NVIDIA Blackwell --- if "gb200" in name or "grace blackwell" in name: return 2.5e15 if "b200" in name: return 2.25e15 if "b100" in name: return 1.8e15 # --- NVIDIA Hopper (H100/H200/H800) --- if "h200" in name: if "nvl" in name or "pcie" in name: return 836e12 return 989e12 # H200 SXM if "h100" in name: if "nvl" in name: return 835e12 if "pcie" in name: return 756e12 return 989e12 # H100 SXM if "h800" in name: if "nvl" in name: return 989e12 return 756e12 # H800 PCIe # --- NVIDIA Ampere data center --- if "a100" in name or "a800" in name: return 312e12 if "a40" in name: return 149.7e12 if "a30" in name: return 165e12 # --- NVIDIA Ada data center --- if "l40s" in name or "l40-s" in name or "l40 s" in name: return 362e12 if "l4" in name: return 121e12 # --- AMD CDNA accelerators --- if "mi355" in name: return 2.5e15 if "mi325" in name or "mi300x" in name: return 1.3074e15 if "mi300a" in name: return 980.6e12 if "mi250x" in name: return 383e12 if "mi250" in name: return 362.1e12 # --- Intel --- if "data center gpu max 1550" in name: # Ponte Vecchio (PVC) - dynamic based on compute units max_comp_units = torch.xpu.get_device_properties("xpu").max_compute_units return 512 * max_comp_units * 1300 * 10**6 # --- Consumer RTX (for hobbyists) --- if "5090" in name: return 209.5e12 if "4090" in name: return 165.2e12 if "3090" in name: return 71e12 # Unknown GPU - return inf so MFU shows as 0% rather than a wrong guess logger.warning(f"Peak flops undefined for: {device_name}, MFU will show as 0%") return float('inf')11:["$","main",null,{"className":"reader-page","children":[["$","div",null,{"className":"reader-kicker","children":[["$","$L10",null,{"href":"/sources/cee70eb6a2dae4c3","children":[["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":16,"height":16,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-arrow-left","children":[["$","path","1l729n",{"d":"m12 19-7-7 7-7"}],["$","path","x3x0zl",{"d":"M19 12H5"}],"$undefined"]}]," 返回 ","StudyingLover's Blog"]}],["$","span",null,{"children":"工程"}]]}],["$","header",null,{"className":"article-meta","children":[["$","span",null,{"children":["FROM · ","StudyingLover's Blog"]}],["$","div",null,{"children":[["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":15,"height":15,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-calendar-days","children":[["$","path","1cmpym",{"d":"M8 2v4"}],["$","path","4m81vk",{"d":"M16 2v4"}],["$","rect","1hopcy",{"width":"18","height":"18","x":"3","y":"4","rx":"2"}],["$","path","8toen8",{"d":"M3 10h18"}],["$","path","6423bh",{"d":"M8 14h.01"}],["$","path","1etili",{"d":"M12 14h.01"}],["$","path","1gbofw",{"d":"M16 14h.01"}],["$","path","lrp35t",{"d":"M8 18h.01"}],["$","path","mhygvu",{"d":"M12 18h.01"}],["$","path","kzsmim",{"d":"M16 18h.01"}],"$undefined"]}]," ","2026年1月18日"," · About the Author \nStudyingLover"]}]]}],["$","$L19",null,{"article":{"title":"nanobot-pre-train","byline":"About the Author \nStudyingLover","publishedAt":"2026-01-18T00:00:00.000Z","language":"en","blocks":[{"id":"b0","type":"paragraph","html":"

nanobot

","text":"nanobot"},{"id":"b1","type":"paragraph","html":"

pre-train

","text":"pre-train"},{"id":"b2","type":"paragraph","html":"

Published on 2026年1月18日

","text":"Published on 2026年1月18日"},{"id":"b3","type":"heading","html":"

数据构造

","text":"数据构造"},{"id":"b4","type":"list","html":"

每行都从 [<|bos|>]token开始
每条数据（row）100% token利用率(无padding)

","text":"每行都从 [<|bos|>]token开始每条数据（row）100% token利用率(无padding)"},{"id":"b5","type":"paragraph","html":"

具体做法：

","text":"具体做法："},{"id":"b6","type":"paragraph","html":"

先使用Tokenizer给每个文档前自动添加 <|bos|> token,然后初始化一个row = [],大小2049。假设buffer中有文档: 500 tokens, 800 tokens, 1200 tokens, 2500 tokens, remaining = 2049

","text":"先使用Tokenizer给每个文档前自动添加 <|bos|> token,然后初始化一个row = [],大小2049。假设buffer中有文档: 500 tokens, 800 tokens, 1200 tokens, 2500 tokens, remaining = 2049"},{"id":"b7","type":"list","html":"

选最大的能放入的: 1200 tokens\n
- row = [doc1200] 1200 tokens
- remaining = 849
\n
再选最大的: 800 tokens\n
- row = doc1200, doc800]
- remaining = 49
\n
没有文档 ≤49, 裁剪最短的文档(500 tokens)\n
- row = [doc1200, doc800, doc500[:49]]
- 结束
\n

","text":"选最大的能放入的: 1200 tokens row = [doc1200] 1200 tokens remaining = 849 再选最大的: 800 tokens row = doc1200, doc800] remaining = 49 没有文档 ≤49, 裁剪最短的文档(500 tokens) row = [doc1200, doc800, doc500[:49]] 结束"},{"id":"b8","type":"quote","html":"

\n
不难发现，单个row中会存在多个BOS，这能让AI学会处理文档边界和上下文切换
\n

","text":"不难发现，单个row中会存在多个BOS，这能让AI学会处理文档边界和上下文切换"},{"id":"b9","type":"heading","html":"

实践经验

","text":"实践经验"},{"id":"b10","type":"heading","html":"

Meta张量初始化（节省显存）

","text":"Meta张量初始化（节省显存）"},{"id":"b11","type":"paragraph","html":"

with torch.device(\"meta\"):\n    model = GPT(config)  # 只有shape没有数据\nmodel.to_empty(device=device)  # 分配存储但不初始化\nmodel.init_weights()  # 最后才初始化

","text":"with torch.device(\"meta\"): model = GPT(config) # 只有shape没有数据 model.to_empty(device=device) # 分配存储但不初始化 model.init_weights() # 最后才初始化"},{"id":"b12","type":"heading","html":"

数据预取（Pipeline优化）

","text":"数据预取（Pipeline优化）"},{"id":"b13","type":"paragraph","html":"

正常的非优化写法是在每个micro_step开始时取数据：

","text":"正常的非优化写法是在每个micro_step开始时取数据："},{"id":"b14","type":"paragraph","html":"

for micro_step in range(grad_accum_steps):\n    x, y, state = next(train_loader)  # ← 先取数据（GPU空闲等待）\n    loss = model(x, y)                 # 然后前向\n    loss.backward()                    # 然后反向

","text":"for micro_step in range(grad_accum_steps): x, y, state = next(train_loader) # ← 先取数据（GPU空闲等待） loss = model(x, y) # 然后前向 loss.backward() # 然后反向"},{"id":"b15","type":"paragraph","html":"

这就存在问题，取数据时GPU闲置，等待CPU的I/O操作

","text":"这就存在问题，取数据时GPU闲置，等待CPU的I/O操作"},{"id":"b16","type":"paragraph","html":"

这里的优化写法是

","text":"这里的优化写法是"},{"id":"b17","type":"paragraph","html":"

x, y, state = next(train_loader)  # 循环前取第一批\n\nfor micro_step in range(grad_accum_steps):\n    loss = model(x, y)                 # 前向（GPU忙）\n    loss.backward()                    # 反向（GPU忙）\n    x, y, state = next(train_loader)   # ← GPU计算时CPU异步准备下一批

","text":"x, y, state = next(train_loader) # 循环前取第一批 for micro_step in range(grad_accum_steps): loss = model(x, y) # 前向（GPU忙） loss.backward() # 反向（GPU忙） x, y, state = next(train_loader) # ← GPU计算时CPU异步准备下一批"},{"id":"b18","type":"paragraph","html":"

GPU计算当前batch的同时，CPU可以并行地tokenize/加载下一批数据，隐藏I/O延迟，提高硬件利用率。

","text":"GPU计算当前batch的同时，CPU可以并行地tokenize/加载下一批数据，隐藏I/O延迟，提高硬件利用率。"},{"id":"b19","type":"paragraph","html":"

有人会有问题

","text":"有人会有问题"},{"id":"b20","type":"list","html":"

loss.backward() 还没算完，x, y, state = next(train_loader) 会执行吗
那会不会loss.backward() 还没算完,代码已经跑到最后了

","text":"loss.backward() 还没算完，x, y, state = next(train_loader) 会执行吗那会不会loss.backward() 还没算完,代码已经跑到最后了"},{"id":"b21","type":"paragraph","html":"

在loss.backward() 会异步启动GPU上的反向传播，但不会阻塞CPU，所以说下一行会正常执行。但是，会有三部分同步cpu和gpu

","text":"在loss.backward() 会异步启动GPU上的反向传播，但不会阻塞CPU，所以说下一行会正常执行。但是，会有三部分同步cpu和gpu"},{"id":"b22","type":"list","html":"

loss = model(x, y)新的前向传播需要GPU资源，PyTorch会自动等待之前的kernel执行完毕
opt.step() 更新参数需要梯度，会等待所有backward完成
synchronize() 显式同步，这里的synchronize()是这样实现的synchronize = torch.cuda.synchronize if device_type == \"cuda\" else lambda: None

","text":"loss = model(x, y)新的前向传播需要GPU资源，PyTorch会自动等待之前的kernel执行完毕 opt.step() 更新参数需要梯度，会等待所有backward完成 synchronize() 显式同步，这里的synchronize()是这样实现的synchronize = torch.cuda.synchronize if device_type == \"cuda\" else lambda: None"},{"id":"b23","type":"heading","html":"

MFU（模型FLOPs利用率）监控

","text":"MFU（模型FLOPs利用率）监控"},{"id":"b24","type":"paragraph","html":"

MFU (Model FLOPs Utilization) 是衡量GPU硬件利用效率的关键指标。

","text":"MFU (Model FLOPs Utilization) 是衡量GPU硬件利用效率的关键指标。"},{"id":"b25","type":"list","html":"

MFU = 50%：你充分利用了GPU理论算力的一半，非常优秀（Transformer训练通常30-50%）
MFU = 20%：一般水平，有优化空间
MFU = 5%：很差，可能有严重瓶颈（数据加载慢、显存不足、kernel效率低）
为什么达不到100%？

","text":"MFU = 50%：你充分利用了GPU理论算力的一半，非常优秀（Transformer训练通常30-50%） MFU = 20%：一般水平，有优化空间 MFU = 5%：很差，可能有严重瓶颈（数据加载慢、显存不足、kernel效率低）为什么达不到100%？"},{"id":"b26","type":"list","html":"

内存带宽瓶颈：读写数据耗时
非计算操作：LayerNorm、激活函数等
通信开销：DDP的AllReduce
Kernel启动开销：小操作的固定成本
空闲等待：数据加载、同步等

","text":"内存带宽瓶颈：读写数据耗时非计算操作：LayerNorm、激活函数等通信开销：DDP的AllReduce Kernel启动开销：小操作的固定成本空闲等待：数据加载、同步等"},{"id":"b27","type":"paragraph","html":"

num_flops_per_token = model.estimate_flops()\nflops_per_sec = num_flops_per_token * batch_size / dt\ntotal_peak = gpu_peak_flops * ddp_world_size\nmfu = 100 * flops_per_sec / total_peak

","text":"num_flops_per_token = model.estimate_flops() flops_per_sec = num_flops_per_token * batch_size / dt total_peak = gpu_peak_flops * ddp_world_size mfu = 100 * flops_per_sec / total_peak"},{"id":"b28","type":"list","html":"

num_flops_per_token: 处理一个token需要的浮点运算数（前向+反向）
batch_size: 一次处理多少个token
dt: 实际耗时（秒）
gpu_peak_flops: 单张GPU的理论峰值（BF16）

","text":"num_flops_per_token: 处理一个token需要的浮点运算数（前向+反向） batch_size: 一次处理多少个token dt: 实际耗时（秒） gpu_peak_flops: 单张GPU的理论峰值（BF16）"},{"id":"b29","type":"paragraph","html":"

那怎么得到gpu_peak_flops呢？查表

","text":"那怎么得到gpu_peak_flops呢？查表"},{"id":"b30","type":"paragraph","html":"$1a","text":"$1b"}],"source":"$0:f:0:1:0:props:children:props:children:1:props:children:props:children:props:children:0:props:sources:397","originalUrl":"https://www.studyinglover.com/blog/nanobot/pre-train/","excerpt":"nanobot"},"shouldTranslate":true}],"$L1c"]}]