[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"site-home-live-entry-setting":3,"site-header-contests":8,"site-header-camps":49,"study-material-detail-study-72":104},{"enabled":4,"title":5,"statusText":5,"buttonText":5,"campSlug":5,"stage":6,"classroomId":7,"hasClassroom":4},false,"",0,null,{"total":9,"list":10,"pageNum":44,"pageSize":45,"size":9,"startRow":44,"endRow":9,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":48,"navigateFirstPage":44,"navigateLastPage":44},3,[11,25,35],{"slug":12,"title":13,"logo":7,"liveUrl":14,"prize":15,"sponsor":16,"description":17,"organizer":18,"startTime":19,"endTime":20,"registrationDeadline":21,"status":22,"dashboardVisibility":23,"createdAt":24},"spring2026","2026春季启元人工智能大赛","https:\u002F\u002Fwww.bilibili.com\u002Fblackboard\u002Flive\u002Flive-activity-player.html?cid=1822703816&quality=0","200,000","麒麟软件有限公司","“2026春季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。\n各参赛选手需在规定的比赛周期内，依据赛事规则选定赛题并完成相应挑战，优胜者将共同角逐总额20万元的赛事奖金池，并且还将有机会获得实习和知名企业优质岗位内推资格。","启元实验室","2026-04-19 16:00:00","2026-07-13 16:00:00","2026-06-16 16:00:00","ongoing","private","2026-04-16 08:00:00",{"slug":26,"title":27,"logo":7,"liveUrl":14,"prize":28,"sponsor":7,"description":29,"organizer":18,"startTime":30,"endTime":31,"registrationDeadline":32,"status":33,"dashboardVisibility":23,"createdAt":34},"autumn2025","2025秋季启元人工智能大赛","150,000","“2025秋季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。","2025-11-08 08:00:00","2026-01-13 08:00:00","2025-12-30 08:00:00","ended","2025-11-03 08:00:00",{"slug":36,"title":37,"logo":7,"liveUrl":14,"prize":38,"sponsor":7,"description":39,"organizer":18,"startTime":40,"endTime":41,"registrationDeadline":42,"status":33,"dashboardVisibility":23,"createdAt":43},"summer2025","启元人工智能大赛(系统方向)","80,000","启元人工智能大赛由启元实验室主办，其中系统赛道基于启元实验室九源软件栈的相关开源项目，通过拉动开源社区力量，一方面促进国产智能计算生态的发展，另一方面也为人工智能系统方向的学生及从业者提供一个实践平台。","2025-07-08 08:00:00","2025-08-25 08:00:00","2025-07-22 08:00:00","2025-06-22 08:00:00",1,5,true,8,[44],{"total":45,"list":50,"pageNum":44,"pageSize":45,"size":45,"startRow":44,"endRow":45,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":103,"navigateFirstPage":44,"navigateLastPage":44},[51,62,73,84,94],{"id":52,"slug":53,"title":54,"logo":55,"publicLiveEmbedUrl":14,"description":56,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":16,"startTime":58,"endTime":59,"status":22,"dashboardVisibility":60,"createdAt":58,"updatedAt":61},"019eeeff-6b00-7d55-a629-efafded3e232","summer2026","2026年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2026-06\u002F019eef7f-7816-7828-9309-79c2c61dd568.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=uiV9kOffa9Xq8DT%2BbErZU0JyQ98%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。","...","2026-06-22 08:00:00","2026-10-19 08:00:00","public","2026-10-12 08:00:00",{"id":63,"slug":64,"title":65,"logo":66,"publicLiveEmbedUrl":7,"description":67,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":68,"startTime":69,"endTime":70,"status":33,"dashboardVisibility":23,"createdAt":71,"updatedAt":72},"019af3f3-e06a-7245-aa8d-37dc99a69c73","winter2025","2025年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-12\u002F019afc69-b7ca-7805-91fb-605429671c01.png?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=8u%2BaQaeliN4QzF6MZWiB4jHYDCE%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。 \n","新华三技术有限公司","2025-12-25 08:00:00","2026-03-27 08:00:00","2025-12-06 08:00:00","2026-10-22 20:24:30",{"id":74,"slug":36,"title":75,"logo":76,"publicLiveEmbedUrl":7,"description":77,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":78,"startTime":79,"endTime":80,"status":81,"dashboardVisibility":23,"createdAt":82,"updatedAt":83},"01980342-208c-7803-b3f3-19e1d31960ba","2025年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-07\u002F0198045e-28fe-78f0-95fe-23835ce58395.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=8pXXbnSlNEtVJCJ1e4RzUlqmRn8%3D","InfiniTensor大模型与人工智能系统训练营由启元实验室及清华大学 AI 系统方向团队发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营联合开源项目开发者、学术界大牛、工业界资深工程师参与授课与项目指导。课程采用「线上直播授课 + 在线答疑 + 项目实践辅导 」的方式进行，确保每位同学都能获得充分成长。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","北京积算科技有限公司","2025-07-14 08:00:00","2025-09-30 08:00:00","upcoming","2025-07-13 08:00:00","2026-10-22 20:24:58",{"id":85,"slug":86,"title":87,"logo":88,"publicLiveEmbedUrl":7,"description":89,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":90,"endTime":91,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":93},"01980cf9-a7d8-7796-ae86-0bbe05334add","winter2024","2024年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-ce23-7624-a287-9f07dd0d85f6.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。报名学员请阅读“导学”课程页的训练营介绍。","2025-01-05 08:00:00","2025-03-22 08:00:00","2025-07-15 08:00:00","2026-10-22 20:24:39",{"id":95,"slug":96,"title":97,"logo":98,"publicLiveEmbedUrl":7,"description":99,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":100,"endTime":101,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":102},"01980c20-363a-7dc3-8f5a-58a68e08dcc4","summer2024","2024年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-fbfa-788a-afcd-17fa9be74ff9.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区和高校，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过课程、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","2024-07-20 08:00:00","2024-09-29 08:00:00","2026-10-22 20:24:49",[44],{"currentItem":105,"processedData":125},{"img":106,"keywords":107,"treeType":44,"description":108,"showStatus":44,"updateTime":109,"type":110,"title":111,"content":112,"nextArticle":113,"createTime":109,"sortOrder":117,"canonicalPath":118,"id":119,"contentType":120,"prevArticle":121},"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_fm.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=y1ZBQdFyysEWB9emc2ZlD7FbhiM%3D","大模型并行,集合通信,Ring AllReduce,计算通信重叠,DualPipe","基于底层互联知识，讲解数据 \u002F 张量 \u002F 流水线 \u002F MoE 四大并行，详解 Ring AllReduce 等集合通信，介绍计算通信重叠、DualPipe 流水线优化方案。","2026-10-22 20:38:11",2,"2026 夏季 InfiniTensor 训练营通信与并行 2：《大模型并行策略与通信优化》","![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw01.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OAbFKBBIKpPqO75HF0eMgEcIHD4%3D)\r\n\r\n\r\n\u003Cspan style=\"font-size:12px;color:#555;\">7 月 21 日，2026 夏季 InfiniTensor 训练营通信与并行方向第二课《大模型并行策略与通信优化》开讲。\r\n上一节《互联、集群与通信》完整讲解 AI 集群底层通信，涵盖 PCIe、NVLink、RDMA 跨机通信、服务器架构与数据中心网络拓扑，建立高性能通信硬件认知。\u003C\u002Fspan>\r\n\r\n本节课基于 PCIe、NVLink、RDMA、集群网络等底层硬件通信基础，拆解大模型分布式训练各类并行方案、集合通信算子底层原理、通信性能优化核心手段，打通硬件互联到上层模型训练的完整链路。\r\n\r\n## 为什么需要分布式通信\r\n### 分布式训练两大核心诱因\r\n#### 1. 算力不足：数据并行加速训练\r\n- 小模型单卡即可完成训练，无需通信；\r\n- 大训练集下单卡计算速度慢，采用数据并行：多卡复制完整模型，每张卡加载独立数据分片，反向传播后通过通信完成全局梯度平均同步。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw02.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=FC7JKNKqfOGeiCufEKZBiCqq7DE%3D)\r\n\r\n#### 2. 显存不足：模型分片并行\r\n当前大模型参数规模高达 2.8T，单机多卡显存无法容纳完整权重，衍生多种模型分片并行方式：\r\n- 流水线并行：按模型层数切分设备，层间传输中间特征；\r\n- 张量并行：将单层计算中的权重矩阵或张量进行切分，由多个设备协同完成计算；\r\n- MoE 专家并行：路由分发、专家结果聚合，高频 All-to-All 通信；\r\n- 序列并行：超长上下文分片，降低激活显存占用。\r\n\r\n## 集合通信：打破点对点传输的性能天花板\r\n### 通信基础与痛点\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw03.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Dky%2B%2BEUIZgwL34TXZxgLrdo8kEs%3D)\r\n\r\n- **初始化过程**：创建 `Communicator` 时会隐式完成地址解析、路由建立等操作，开发者仅需获取 `Rank ID` 即可发起通信。\r\n- **点对点通信局限**：直接使用 `Send`\u002F`Recv` 操作时，开发者需手动编写同步逻辑，且多卡场景下存在大量冗余传输，无法利用底层网络拓扑特性进行全局优化。\r\n\r\n### 集合通信（以 AllReduce 为例）\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw04.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=tPSy5sUcFe3oFkqlzzoiyhorIVM%3D)\r\n\r\n1. **朴素两两同步**：每个节点向其余 N-1 个节点发送全量数据。通信次数随节点数呈平方级增长，带宽浪费极其严重，仅具理论参考价值。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw05.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NSD4HmQ5Wx9L9JNvLktsxLzKr%2BQ%3D)\r\n\r\n\r\n2. **Parameter Server 模式**：选定一个 Server 节点汇聚所有数据并广播结果。传输数据量降为线性，但 Server 节点的网卡带宽会成为单点瓶颈，不适用于大规模训练。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw06.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=x4xfCu2VwP5mhPP6iUA9ceEDBYI%3D)\r\n\r\n\r\n3. **Ring AllReduce（工业界标准）**：将数据切片后沿环形拓扑传递，在保证带宽均衡利用的同时，将传输时间复杂度有效降低，是当前集合通信的基石算法。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw07.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Whkj%2B%2F%2FbCKy7ON%2ByvUF198jyDPY%3D)\r\n\r\n\r\n### 其他常用集合通信算子\r\n- **AllGather**：每个节点提供局部数据，最终所有节点均获得完整的数据集合。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw08.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ftywqNws6xlZtFkt93cZZ4O0hqg%3D)\r\n\r\n- **AllToAll**：实现类似矩阵转置的数据重排，是 MoE 专家路由的核心通信原语。\r\n- **组合算子**：`ReduceScatter` 与 `AllGather` 的组合可等价实现 `AllReduce`，灵活适配不同并行策略。\r\n\r\n\r\n## 通信优化\r\n### 计算通信重叠\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw09.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=X1I%2FK5HfJpXt7UPfNvIcfdgH23Y%3D)\r\n\r\n1. **串行执行的浪费**：传统模式下，计算时通信链路闲置，通信时 GPU 计算单元停滞，资源利用率低下。\r\n\r\n2. **并行掩盖延迟**：通过异步通信接口和通信计算并行执行，使通信过程与 GPU 计算重叠，从而隐藏部分通信开销。\r\n\r\n3. 分布式矩阵乘优化实例\r\n   - **串行模式**：先完成完整的矩阵乘法计算，再统一执行 `ReduceScatter` 同步，总耗时为两者之和。\r\n   - **Overlap 模式**：将矩阵 A 切分为多个子块，计算完第一个子块后立即发起异步通信，同时在通信过程中继续计算后续子块。由于每次通信数据量减小且与计算并行，整体任务完成时间大幅缩短。\r\n\r\n### 拓扑感知\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw10.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=uGTvHDgQxNrLaZ4h9I5pOENzm04%3D)\r\n\r\n1. **机内总线网络**：带宽极高但规模受限，优先承载 `AllReduce`、`AllToAll` 等大流量集合通信算子。\r\n\r\n2. **机间 RDMA 网络**：带宽相对较低但支持万卡乃至十万卡级组网，承载流水线并行的 P2P 通信及跨机数据并行流量。\r\n\r\n3. **网络层级扩展**：随着集群规模扩大，交换机层级从 Leaf 逐步扩展至 Spine、Fabric，通信策略需动态适配拓扑变化。\r\n\r\n## 流水线并行 DualPipe\r\n\r\n### 1F1B 基础方案\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw11.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=1zEJBvtDeSVryhTkXMsTWSja8jU%3D)\r\n\r\n- 采用前向-后向交替执行的模式。\r\n- 由于反向传播耗时通常约为前向的两倍，会导致前向计算出现大量空跑气泡，资源利用率不高。\r\n\r\n\r\n### ZB1P 细粒度拆分方案\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw12.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=b7kX9F2htbxmE51QKqa%2FxFg%2FOkQ%3D)\r\n\r\n- 将反向计算进一步拆分为输入梯度计算和权重梯度计算等阶段，通过更细粒度调度减少流水线空泡。\r\n- 使各阶段耗时更加均衡，减少了部分气泡，但仍存在少量设备空闲时间。\r\n\r\n### DualPipe 和 DualPipe-V\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw13.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Whweq%2F9ShGu1PC2ai%2BfFEnpoLV8%3D)\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw14.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=zM1%2F9RwT7l3MOWU1nHpaQox8%2Fb0%3D)\r\n\r\n\r\n\r\n\r\n## 加入训练营\r\n#### 📺完整课程内容，请观看直播或查看课程回放：\r\n**InfiniTensor 官网**： https:\u002F\u002Fwww.infinitensor.com\r\n\r\n**B站直播**： InfiniTensor 官方直播间\r\n\r\n**视频号直播**： InfiniTensor 视频号预约\r\n\r\n**答疑交流**： 训练营官方社群⬇️\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002Fportal_news\u002F2026-xjq.png?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=oKlo6p8Ra0fcURk4F6PNTseTp%2BA%3D)\r\n\r\n## 关注与交流\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002Fportal_news\u002Fnews_ewm02.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=R03Psa4VwXKPuVbshfyqCCO33b4%3D)",{"publishTime":7,"createTime":109,"canonicalPath":114,"id":115,"title":116},"\u002Fstudy-material\u002Fstudy-71","study-71","2026 夏季 InfiniTensor 训练营 OpenCL 编程 1：《OpenCL 概述与运行时》",37,"\u002Fstudy-material\u002Fstudy-72","study-72","markdown",{"publishTime":7,"createTime":109,"canonicalPath":122,"id":123,"title":124},"\u002Fstudy-material\u002Fstudy-73","study-73","2026 夏季 InfiniTensor 训练营 AI 编程与智能体应用 2：《AI Agent 基础：从“会回答”到“能执行任务”的 AI 系统》",{"htmlContent":126,"headings":127},"\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw01.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OAbFKBBIKpPqO75HF0eMgEcIHD4%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cspan style=\"font-size:12px;color:#555;\">7 月 21 日，2026 夏季 InfiniTensor 训练营通信与并行方向第二课《大模型并行策略与通信优化》开讲。\u003Cbr>上一节《互联、集群与通信》完整讲解 AI 集群底层通信，涵盖 PCIe、NVLink、RDMA 跨机通信、服务器架构与数据中心网络拓扑，建立高性能通信硬件认知。\u003C\u002Fspan>\u003C\u002Fp>\n\u003Cp>本节课基于 PCIe、NVLink、RDMA、集群网络等底层硬件通信基础，拆解大模型分布式训练各类并行方案、集合通信算子底层原理、通信性能优化核心手段，打通硬件互联到上层模型训练的完整链路。\u003C\u002Fp>\n\u003Ch2 id=\"为什么需要分布式通信\">\u003Ca class=\"anchor\" href=\"#为什么需要分布式通信\" aria-label=\"跳转到标题：为什么需要分布式通信\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>为什么需要分布式通信\u003C\u002Fh2>\u003Ch3 id=\"分布式训练两大核心诱因\">\u003Ca class=\"anchor\" href=\"#分布式训练两大核心诱因\" aria-label=\"跳转到标题：分布式训练两大核心诱因\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>分布式训练两大核心诱因\u003C\u002Fh3>\u003Ch4 id=\"1-算力不足-数据并行加速训练\">\u003Ca class=\"anchor\" href=\"#1-算力不足-数据并行加速训练\" aria-label=\"跳转到标题：1. 算力不足：数据并行加速训练\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 算力不足：数据并行加速训练\u003C\u002Fh4>\u003Cul>\n\u003Cli>小模型单卡即可完成训练，无需通信；\u003C\u002Fli>\n\u003Cli>大训练集下单卡计算速度慢，采用数据并行：多卡复制完整模型，每张卡加载独立数据分片，反向传播后通过通信完成全局梯度平均同步。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw02.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=FC7JKNKqfOGeiCufEKZBiCqq7DE%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"2-显存不足-模型分片并行\">\u003Ca class=\"anchor\" href=\"#2-显存不足-模型分片并行\" aria-label=\"跳转到标题：2. 显存不足：模型分片并行\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 显存不足：模型分片并行\u003C\u002Fh4>\u003Cp>当前大模型参数规模高达 2.8T，单机多卡显存无法容纳完整权重，衍生多种模型分片并行方式：\u003C\u002Fp>\n\u003Cul>\n\u003Cli>流水线并行：按模型层数切分设备，层间传输中间特征；\u003C\u002Fli>\n\u003Cli>张量并行：将单层计算中的权重矩阵或张量进行切分，由多个设备协同完成计算；\u003C\u002Fli>\n\u003Cli>MoE 专家并行：路由分发、专家结果聚合，高频 All-to-All 通信；\u003C\u002Fli>\n\u003Cli>序列并行：超长上下文分片，降低激活显存占用。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch2 id=\"集合通信-打破点对点传输的性能天花板\">\u003Ca class=\"anchor\" href=\"#集合通信-打破点对点传输的性能天花板\" aria-label=\"跳转到标题：集合通信：打破点对点传输的性能天花板\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>集合通信：打破点对点传输的性能天花板\u003C\u002Fh2>\u003Ch3 id=\"通信基础与痛点\">\u003Ca class=\"anchor\" href=\"#通信基础与痛点\" aria-label=\"跳转到标题：通信基础与痛点\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>通信基础与痛点\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw03.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Dky%2B%2BEUIZgwL34TXZxgLrdo8kEs%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>初始化过程\u003C\u002Fstrong>：创建 \u003Ccode>Communicator\u003C\u002Fcode> 时会隐式完成地址解析、路由建立等操作，开发者仅需获取 \u003Ccode>Rank ID\u003C\u002Fcode> 即可发起通信。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>点对点通信局限\u003C\u002Fstrong>：直接使用 \u003Ccode>Send\u003C\u002Fcode>\u002F\u003Ccode>Recv\u003C\u002Fcode> 操作时，开发者需手动编写同步逻辑，且多卡场景下存在大量冗余传输，无法利用底层网络拓扑特性进行全局优化。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"集合通信-以-allreduce-为例\">\u003Ca class=\"anchor\" href=\"#集合通信-以-allreduce-为例\" aria-label=\"跳转到标题：集合通信（以 AllReduce 为例）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>集合通信（以 AllReduce 为例）\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw04.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=tPSy5sUcFe3oFkqlzzoiyhorIVM%3D\" alt=\"\">\u003C\u002Fp>\n\u003Col>\n\u003Cli>\u003Cstrong>朴素两两同步\u003C\u002Fstrong>：每个节点向其余 N-1 个节点发送全量数据。通信次数随节点数呈平方级增长，带宽浪费极其严重，仅具理论参考价值。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw05.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NSD4HmQ5Wx9L9JNvLktsxLzKr%2BQ%3D\" alt=\"\">\u003C\u002Fp>\n\u003Col start=\"2\">\n\u003Cli>\u003Cstrong>Parameter Server 模式\u003C\u002Fstrong>：选定一个 Server 节点汇聚所有数据并广播结果。传输数据量降为线性，但 Server 节点的网卡带宽会成为单点瓶颈，不适用于大规模训练。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw06.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=x4xfCu2VwP5mhPP6iUA9ceEDBYI%3D\" alt=\"\">\u003C\u002Fp>\n\u003Col start=\"3\">\n\u003Cli>\u003Cstrong>Ring AllReduce（工业界标准）\u003C\u002Fstrong>：将数据切片后沿环形拓扑传递，在保证带宽均衡利用的同时，将传输时间复杂度有效降低，是当前集合通信的基石算法。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw07.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Whkj%2B%2F%2FbCKy7ON%2ByvUF198jyDPY%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"其他常用集合通信算子\">\u003Ca class=\"anchor\" href=\"#其他常用集合通信算子\" aria-label=\"跳转到标题：其他常用集合通信算子\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>其他常用集合通信算子\u003C\u002Fh3>\u003Cul>\n\u003Cli>\u003Cstrong>AllGather\u003C\u002Fstrong>：每个节点提供局部数据，最终所有节点均获得完整的数据集合。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw08.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ftywqNws6xlZtFkt93cZZ4O0hqg%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>AllToAll\u003C\u002Fstrong>：实现类似矩阵转置的数据重排，是 MoE 专家路由的核心通信原语。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>组合算子\u003C\u002Fstrong>：\u003Ccode>ReduceScatter\u003C\u002Fcode> 与 \u003Ccode>AllGather\u003C\u002Fcode> 的组合可等价实现 \u003Ccode>AllReduce\u003C\u002Fcode>，灵活适配不同并行策略。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch2 id=\"通信优化\">\u003Ca class=\"anchor\" href=\"#通信优化\" aria-label=\"跳转到标题：通信优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>通信优化\u003C\u002Fh2>\u003Ch3 id=\"计算通信重叠\">\u003Ca class=\"anchor\" href=\"#计算通信重叠\" aria-label=\"跳转到标题：计算通信重叠\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>计算通信重叠\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw09.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=X1I%2FK5HfJpXt7UPfNvIcfdgH23Y%3D\" alt=\"\">\u003C\u002Fp>\n\u003Col>\n\u003Cli>\u003Cp>\u003Cstrong>串行执行的浪费\u003C\u002Fstrong>：传统模式下，计算时通信链路闲置，通信时 GPU 计算单元停滞，资源利用率低下。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003Cli>\u003Cp>\u003Cstrong>并行掩盖延迟\u003C\u002Fstrong>：通过异步通信接口和通信计算并行执行，使通信过程与 GPU 计算重叠，从而隐藏部分通信开销。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003Cli>\u003Cp>分布式矩阵乘优化实例\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>串行模式\u003C\u002Fstrong>：先完成完整的矩阵乘法计算，再统一执行 \u003Ccode>ReduceScatter\u003C\u002Fcode> 同步，总耗时为两者之和。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>Overlap 模式\u003C\u002Fstrong>：将矩阵 A 切分为多个子块，计算完第一个子块后立即发起异步通信，同时在通信过程中继续计算后续子块。由于每次通信数据量减小且与计算并行，整体任务完成时间大幅缩短。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Ch3 id=\"拓扑感知\">\u003Ca class=\"anchor\" href=\"#拓扑感知\" aria-label=\"跳转到标题：拓扑感知\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>拓扑感知\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw10.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=uGTvHDgQxNrLaZ4h9I5pOENzm04%3D\" alt=\"\">\u003C\u002Fp>\n\u003Col>\n\u003Cli>\u003Cp>\u003Cstrong>机内总线网络\u003C\u002Fstrong>：带宽极高但规模受限，优先承载 \u003Ccode>AllReduce\u003C\u002Fcode>、\u003Ccode>AllToAll\u003C\u002Fcode> 等大流量集合通信算子。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003Cli>\u003Cp>\u003Cstrong>机间 RDMA 网络\u003C\u002Fstrong>：带宽相对较低但支持万卡乃至十万卡级组网，承载流水线并行的 P2P 通信及跨机数据并行流量。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003Cli>\u003Cp>\u003Cstrong>网络层级扩展\u003C\u002Fstrong>：随着集群规模扩大，交换机层级从 Leaf 逐步扩展至 Spine、Fabric，通信策略需动态适配拓扑变化。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Ch2 id=\"流水线并行-dualpipe\">\u003Ca class=\"anchor\" href=\"#流水线并行-dualpipe\" aria-label=\"跳转到标题：流水线并行 DualPipe\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>流水线并行 DualPipe\u003C\u002Fh2>\u003Ch3 id=\"1f1b-基础方案\">\u003Ca class=\"anchor\" href=\"#1f1b-基础方案\" aria-label=\"跳转到标题：1F1B 基础方案\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1F1B 基础方案\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw11.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=1zEJBvtDeSVryhTkXMsTWSja8jU%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>采用前向-后向交替执行的模式。\u003C\u002Fli>\n\u003Cli>由于反向传播耗时通常约为前向的两倍，会导致前向计算出现大量空跑气泡，资源利用率不高。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"zb1p-细粒度拆分方案\">\u003Ca class=\"anchor\" href=\"#zb1p-细粒度拆分方案\" aria-label=\"跳转到标题：ZB1P 细粒度拆分方案\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>ZB1P 细粒度拆分方案\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw12.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=b7kX9F2htbxmE51QKqa%2FxFg%2FOkQ%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>将反向计算进一步拆分为输入梯度计算和权重梯度计算等阶段，通过更细粒度调度减少流水线空泡。\u003C\u002Fli>\n\u003Cli>使各阶段耗时更加均衡，减少了部分气泡，但仍存在少量设备空闲时间。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"dualpipe-和-dualpipe-v\">\u003Ca class=\"anchor\" href=\"#dualpipe-和-dualpipe-v\" aria-label=\"跳转到标题：DualPipe 和 DualPipe-V\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>DualPipe 和 DualPipe-V\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw13.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Whweq%2F9ShGu1PC2ai%2BfFEnpoLV8%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_72_zw14.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=zM1%2F9RwT7l3MOWU1nHpaQox8%2Fb0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"加入训练营\">\u003Ca class=\"anchor\" href=\"#加入训练营\" aria-label=\"跳转到标题：加入训练营\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>加入训练营\u003C\u002Fh2>\u003Ch4 id=\"完整课程内容-请观看直播或查看课程回放\">\u003Ca class=\"anchor\" href=\"#完整课程内容-请观看直播或查看课程回放\" aria-label=\"跳转到标题：📺完整课程内容，请观看直播或查看课程回放：\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>📺完整课程内容，请观看直播或查看课程回放：\u003C\u002Fh4>\u003Cp>\u003Cstrong>InfiniTensor 官网\u003C\u002Fstrong>： \u003Ca href=\"https:\u002F\u002Fwww.infinitensor.com\">https:\u002F\u002Fwww.infinitensor.com\u003C\u002Fa>\u003C\u002Fp>\n\u003Cp>\u003Cstrong>B站直播\u003C\u002Fstrong>： InfiniTensor 官方直播间\u003C\u002Fp>\n\u003Cp>\u003Cstrong>视频号直播\u003C\u002Fstrong>： InfiniTensor 视频号预约\u003C\u002Fp>\n\u003Cp>\u003Cstrong>答疑交流\u003C\u002Fstrong>： 训练营官方社群⬇️\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002Fportal_news\u002F2026-xjq.png?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=oKlo6p8Ra0fcURk4F6PNTseTp%2BA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"关注与交流\">\u003Ca class=\"anchor\" href=\"#关注与交流\" aria-label=\"跳转到标题：关注与交流\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>关注与交流\u003C\u002Fh2>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002Fportal_news\u002Fnews_ewm02.jpg?Expires=1792752557&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=R03Psa4VwXKPuVbshfyqCCO33b4%3D\" alt=\"\">\u003C\u002Fp>\n",[128,130,132,136,139,142,144,147,149,151,153,155,158,161,164,167,169,172],{"id":129,"text":129,"level":110},"为什么需要分布式通信",{"id":131,"text":131,"level":9},"分布式训练两大核心诱因",{"id":133,"text":134,"level":135},"1-算力不足-数据并行加速训练","1. 算力不足：数据并行加速训练",4,{"id":137,"text":138,"level":135},"2-显存不足-模型分片并行","2. 显存不足：模型分片并行",{"id":140,"text":141,"level":110},"集合通信-打破点对点传输的性能天花板","集合通信：打破点对点传输的性能天花板",{"id":143,"text":143,"level":9},"通信基础与痛点",{"id":145,"text":146,"level":9},"集合通信-以-allreduce-为例","集合通信（以 AllReduce 为例）",{"id":148,"text":148,"level":9},"其他常用集合通信算子",{"id":150,"text":150,"level":110},"通信优化",{"id":152,"text":152,"level":9},"计算通信重叠",{"id":154,"text":154,"level":9},"拓扑感知",{"id":156,"text":157,"level":110},"流水线并行-dualpipe","流水线并行 DualPipe",{"id":159,"text":160,"level":9},"1f1b-基础方案","1F1B 基础方案",{"id":162,"text":163,"level":9},"zb1p-细粒度拆分方案","ZB1P 细粒度拆分方案",{"id":165,"text":166,"level":9},"dualpipe-和-dualpipe-v","DualPipe 和 DualPipe-V",{"id":168,"text":168,"level":110},"加入训练营",{"id":170,"text":171,"level":135},"完整课程内容-请观看直播或查看课程回放","📺完整课程内容，请观看直播或查看课程回放：",{"id":173,"text":173,"level":110},"关注与交流"]