InternLM · franksfc · Jul 28, 2026
diff --git a/Megatron-LM/.github/workflows/repository-hygiene.yml b/Megatron-LM/.github/workflows/repository-hygiene.yml
@@ -0,0 +1,18 @@
+name: Repository hygiene
+
+on:
+  pull_request:
+  push:
+
+permissions:
+  contents: read
+
+jobs:
+  public-surface:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: false
+      - name: Scan public Git candidates
+        run: python scripts/check_repository_hygiene.py
diff --git a/Megatron-LM/.gitignore b/Megatron-LM/.gitignore
@@ -0,0 +1,33 @@
+__pycache__/
+*.py[cod]
+*.so
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.venv/
+.env
+*.env
+.netrc
+kubeconfig
+kubeconfig.*
+build/
+dist/
+*.egg-info/
+outputs/
+checkpoints/
+logs/
+wandb/
+tensorboard/
+eval_artifacts/
+kernel_meta/
+/cluster/
+runs/
+/data/
+*.log
+*.out
+*.err
+*.bin
+*.idx
+*.npy
+*.npz
+*.safetensors
diff --git a/Megatron-LM/.gitmodules b/Megatron-LM/.gitmodules
@@ -0,0 +1,12 @@
+[submodule "third_party/MindSpeed"]
+	path = third_party/MindSpeed
+	url = https://github.com/Ascend/MindSpeed.git
+[submodule "third_party/MindSpeed-LLM"]
+	path = third_party/MindSpeed-LLM
+	url = https://github.com/Ascend/MindSpeed-LLM.git
+[submodule "third_party/OLMo-core"]
+	path = third_party/OLMo-core
+	url = https://github.com/allenai/OLMo-core.git
+[submodule "third_party/olmes"]
+	path = third_party/olmes
+	url = https://github.com/allenai/olmes.git
diff --git a/Megatron-LM/ASCEND_RUNTIME_LOCK.json b/Megatron-LM/ASCEND_RUNTIME_LOCK.json
@@ -0,0 +1,48 @@
+{
+  "schema": "olmo3.mindspeed.ascend-runtime-lock/v1",
+  "recorded_at_utc": "2026-07-26T20:55:32Z",
+  "validation_scope": "Ascend 910C multi-node Stage 1 through Stage 4 validation",
+  "python": {
+    "version": "3.10.20"
+  },
+  "python_packages": {
+    "torch": {
+      "distribution": "torch",
+      "version": "2.6.0"
+    },
+    "torch_npu": {
+      "distribution": "torch-npu",
+      "version": "2.6.0"
+    }
+  },
+  "ascend_components": {
+    "runtime": {
+      "version_key": "runtime_running_version",
+      "build": "8.3.0.2.220",
+      "release": "8.3.RC2"
+    },
+    "hccl": {
+      "version_key": "hccl_running_version",
+      "build": "8.3.0.2.220",
+      "release": "8.3.RC2"
+    }
+  },
+  "upstream_declaration": {
+    "source": "UPSTREAM_LOCK.json upstream_declared_stack.pytorch",
+    "pytorch": "2.7.1"
+  },
+  "support_status": "project-validated-deviation",
+  "validated_deviation": {
+    "pytorch": "The pinned MindSpeed-LLM upstream branch declares PyTorch 2.7.1, while the deployment image validated end to end for this pipeline supplies the matched torch/torch_npu 2.6.0 pair. The executable runtime lock takes precedence over that upstream declaration for this deployment."
+  },
+  "validation_evidence": {
+    "kind": "isolated_end_to_end",
+    "scope": "isolated multi-node Stage 1 through Stage 4 pipeline",
+    "result": "passed",
+    "upstream_commits": {
+      "MindSpeed": "c94078bb90795dd8f3329a640eb5c0eac472f9d9",
+      "MindSpeed-LLM": "630fe3c4cd1a5416c8b90986cd04e212ad654f06",
+      "Megatron-LM": "a845aa7e12b3a117e24c2352b9e3e60bad2e3a17"
+    }
+  }
+}
diff --git a/Megatron-LM/README.md b/Megatron-LM/README.md
@@ -0,0 +1,170 @@
+# OLMo 3 MindSpeed Pipeline
+
+该仓库包含 OLMo 3 在 Ascend/MindSpeed 上的数据处理、Stage 1–4 训练、
+checkpoint 迁移与恢复、原生推理、PPL 验证和 OLMES 客观评测入口。
+
+主训练链：
+
+```text
+Dolma 3 Mix 6T
+  → Stage 1 pretraining, 8K
+  → Dolmino 100B Stage 2, 8K
+  → Longmino 50B Stage 3, 65K
+  → Think SFT, 32K
+  → Instruct SFT, 32K
+  → native inference
+  → PPL / OLMES objective evaluation
+```
+
+## 状态边界
+
+| 范围 | 状态 |
+|---|---|
+| Base 与 Siamese/Depth 两种模型定义 | 已实现，静态与单元测试覆盖 |
+| 1B/3B/7B 架构解析 | 已实现，来源固定到 OLMo-core 构造器 |
+| Stage 1–4 数据与训练命令生成 | 已实现 |
+| `torch_dist` checkpoint save/resume/transition | 已实现 |
+| 原生推理与 65K KV-cache smoke 入口 | 已实现；需要在目标 NPU 环境验证 |
+| PPL 与 OLMES 客观评测 | 已实现；需要使用目标 checkpoint 验证 |
+
+“已实现”不等同于“所有模型规模和拓扑均已在真实 NPU 上完成端到端训练”。
+
+## 目录
+
+```text
+configs/
+  models/         1B、3B、7B 架构
+  variants/       base、siamese_depth
+  stages/         Stage 1、2、3、Think SFT、Instruct SFT
+  data/           数据来源、tokenizer 和 packing 合同
+  topology/       Stage 1–4 与 TP2/SIO 参考拓扑
+  performance/    ordinary HSDP、TP2/SIO/MC2、CP single-halo
+  pipeline/       生产输入模板
+scripts/
+  data/           下载、tokenization、index/cache 构建
+  train/          通用 1B/3B/7B 训练入口
+  checkpoint/     checkpoint 检查与重分片
+  inference/      原生 checkpoint 推理
+  eval/           PPL 与 OLMES 冻结/评分
+src/              配置解析、运行时和模型实现
+megatron/         受控 Megatron 修改
+third_party/      固定 revision 的上游仓库
+```
+
+## 环境
+
+Conda 环境从空前缀创建。该过程不读取、不复制、不 clone 任何已有 Conda
+环境。
+
+```bash
+cd /path/to/olmo3_mindspeed_pipeline
+
+scripts/bootstrap_conda_env.sh \
+  --target-env olmo3-mindspeed-pipeline \
+  --ascend-wheelhouse /path/to/cp310-aarch64-wheels \
+  --strict-ascend
+```
+
+`--ascend-wheelhouse` 包含与
+[`ASCEND_RUNTIME_LOCK.json`](ASCEND_RUNTIME_LOCK.json) 匹配的
+`torch==2.6.0` 和 `torch-npu==2.6.0` AArch64/CPython 3.10 wheels。
+CANN/HCCL、驱动和固件属于宿主机二进制栈，不属于 Conda 环境。
+
+固定的 MindSpeed-LLM revision 上游声明 PyTorch 2.7.1；本部署使用经过项目
+验证的 PyTorch/TorchNPU 2.6.0 与 CANN/HCCL 8.3.RC2。
+该组合属于项目级集群验证偏差，不代表 MindSpeed 的完整上游支持矩阵。
+同样，Dolma 1.1.2 的包元数据限制 `tokenizers<=0.19.1`，而
+Transformers 4.46.1 要求 `tokenizers>=0.20,<0.21`；环境固定为实际验证过的
+0.20.3，并由 `scripts/verify_python_dependencies.py` 只允许这一项精确的
+元数据偏差。其他缺包、版本偏差或依赖冲突均使环境构建失败。
+
+控制节点无 NPU 时省略 `--strict-ascend`；所有训练 Pod 内仍执行：
+
+```bash
+conda run -n olmo3-mindspeed-pipeline \
+  python scripts/verify_ascend_env.py --strict
+```
+
+## 生产输入
+
+[`configs/pipeline/production.env.example`](configs/pipeline/production.env.example)
+集中列出路径、不可变 revision、模型、variant、LR、min LR、warmup tokens、
+训练 token 数、拓扑、run identity 和端口。模型架构值位于
+`configs/models/*.json`，训练超参数不写入模型架构配置。
+
+完整命令顺序位于
+[`docs/RUNBOOK.md`](docs/RUNBOOK.md)：
+
+```text
+环境
+→ 固定上游源码
+→ tokenizer
+→ 五类数据下载与处理
+→ Stage 1 fresh/resume
+→ Stage 1→2 transition
+→ Stage 2 resume/train
+→ Stage 2→3 transition
+→ Stage 3 resume/train
+→ Stage 3→Think transition
+→ Think resume/train
+→ Think→Instruct transition
+→ Instruct resume/train
+→ inference
+→ evaluation
+```
+
+每次真实提交对应新的 run ID、输出目录和 W&B run identity。多节点路径通过
+渲染后的 `launch.sh` 共享，不在脚本中固化集群绝对路径。
+
+非 editable 安装只安装 Python 包，不复制配置、脚本、Megatron overlay 或固定
+的第三方仓库。安装后的 `olmo3ctl`、`olmo3ckpt`、`olmo3eval` 以导出的
+`OLMO3_ROOT` 为仓库根目录；从仓库内运行时也可自动定位。无效的显式路径会
+立即报错，不会退回其他目录。安装包还会逐文件核对该 checkout 中的
+`olmo3_pipeline`、`modeling`、`runtime` 和 `llama_config`；源码更新后必须
+重新构建环境，旧 wheel 与新 checkout 混用会被拒绝。
+
+## 配置来源
+
+1B、3B、7B 的结构来自固定 OLMo-core revision 中的
+`TransformerConfig.olmo3_1B/3B/7B`。其中 7B/32B 存在官方发布的完整训练
+recipe；1B/3B 的 LR、min LR、warmup 和部署拓扑属于本项目显式实验输入，
+不标记为官方发布 recipe。
+
+两种 variant：
+
+| variant | 模型 | checkpoint keyspace |
+|---|---|---|
+| `base` | OLMo 3 | `olmo3_base_v1` |
+| `siamese_depth` | OLMo 3 + Siamese Norm + Depth Attention | `olmo3_siamese_depth_v1` |
+
+同一 checkpoint 链中的 model size 和 variant 保持不变。
+
+## 验证
+
+控制节点测试：
+
+```bash
+python scripts/check_repository_hygiene.py
+
+TORCH_DEVICE_BACKEND_AUTOLOAD=0 \
+PYTHONPATH=src:. \
+python -m pytest -q
+```
+
+公开提交前可通过重复的 `--deny-string` 或逗号分隔的
+`OLMO3_HYGIENE_DENY_STRINGS` 增加操作者特有的用户名、挂载点和集群标识。
+检查结果只报告文件、行号和规则，不回显疑似凭据内容。
+
+真实 NPU 验证分为严格环境检查、Stage 1–4 smoke、65K cache smoke
+和评测 smoke。
+
+## 详细文档
+
+- [`docs/RUNBOOK.md`](docs/RUNBOOK.md)：从空环境到评测的命令链
+- [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md)：模型与拓扑解析
+- [`docs/DATA.md`](docs/DATA.md)：数据合同
+- [`docs/CHECKPOINTS.md`](docs/CHECKPOINTS.md)：checkpoint 生命周期
+- [`docs/OPTIMIZATIONS.md`](docs/OPTIMIZATIONS.md)：TP/CP/HSDP 优化入口
+- [`docs/INFERENCE_EVALUATION.md`](docs/INFERENCE_EVALUATION.md)：推理与评测边界
+- [`UPSTREAM_LOCK.json`](UPSTREAM_LOCK.json)：上游源码 revision
+- [`ASCEND_RUNTIME_LOCK.json`](ASCEND_RUNTIME_LOCK.json)：Ascend 二进制合同
diff --git a/Megatron-LM/UPSTREAM_LOCK.json b/Megatron-LM/UPSTREAM_LOCK.json
@@ -0,0 +1,52 @@
+{
+  "generated_at_utc": "2026-07-26T14:00:00Z",
+  "upstream_declared_stack": {
+    "source": "third_party/MindSpeed-LLM/docs/zh/release_notes_llm.md",
+    "mindspeed_llm_branch": "master",
+    "mindspeed_branch": "master",
+    "megatron_version": "core_v0.12.1",
+    "python": "3.10",
+    "pytorch": "2.7.1"
+  },
+  "repositories": {
+    "MindSpeed": {
+      "url": "https://github.com/Ascend/MindSpeed.git",
+      "ref": "master",
+      "commit": "c94078bb90795dd8f3329a640eb5c0eac472f9d9",
+      "path": "third_party/MindSpeed",
+      "must_be_pristine": true,
+      "require_full_history": true
+    },
+    "MindSpeed-LLM": {
+      "url": "https://github.com/Ascend/MindSpeed-LLM.git",
+      "ref": "master",
+      "commit": "630fe3c4cd1a5416c8b90986cd04e212ad654f06",
+      "path": "third_party/MindSpeed-LLM",
+      "must_be_pristine": true,
+      "require_full_history": true
+    },
+    "Megatron-LM": {
+      "url": "https://github.com/NVIDIA/Megatron-LM.git",
+      "ref": "core_v0.12.1",
+      "commit": "a845aa7e12b3a117e24c2352b9e3e60bad2e3a17",
+      "path": "megatron",
+      "vendored_and_modified": true
+    },
+    "OLMo-core": {
+      "url": "https://github.com/allenai/OLMo-core.git",
+      "ref": "main",
+      "commit": "fa6c5014c9f6e9ee789da2d9c20d5126fee8df0d",
+      "path": "third_party/OLMo-core",
+      "must_be_pristine": true,
+      "role": "read-only architecture and recipe specification"
+    },
+    "OLMES": {
+      "url": "https://github.com/allenai/olmes.git",
+      "ref": "5a51f502d463b8cdc4a2dcad7d7096c41ff1197e",
+      "commit": "5a51f502d463b8cdc4a2dcad7d7096c41ff1197e",
+      "path": "third_party/olmes",
+      "must_be_pristine": true,
+      "role": "evaluation-only request construction and objective scoring; never imported by the Ascend training runtime"
+    }
+  }
+}
diff --git a/Megatron-LM/configs/data/dolci_instruct.json b/Megatron-LM/configs/data/dolci_instruct.json
@@ -0,0 +1,30 @@
+{
+  "data": {
+    "name": "dolci_instruct",
+    "stage": "sft_instruct",
+    "backend": "olmo3_sft_numpy",
+    "root": null,
+    "manifest": null,
+    "work_dir": null,
+    "expected_instances": null,
+    "expected_fingerprint": null,
+    "tokenizer": {
+      "kind": "huggingface_tokenizer_json",
+      "path": null,
+      "eos_token_id": 100257,
+      "pad_token_id": 100277,
+      "bos_token_id": null
+    },
+    "source": {
+      "repository": "allenai/Dolci-Instruct-SFT",
+      "revision": "bd3c8f3a9b2cc5a9682e44b96ddd0bb2ff027221"
+    },
+    "packing": {
+      "kind": "assistant_only_sft",
+      "sequence_length": 32768,
+      "assistant_only_loss": true,
+      "shift_labels_once": true,
+      "forbid_cross_document_attention": true
+    }
+  }
+}