[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"site-home-live-entry-setting":3,"site-header-contests":8,"site-header-camps":49,"study-material-detail-study-22":104},{"enabled":4,"title":5,"statusText":5,"buttonText":5,"campSlug":5,"stage":6,"classroomId":7,"hasClassroom":4},false,"",0,null,{"total":9,"list":10,"pageNum":44,"pageSize":45,"size":9,"startRow":44,"endRow":9,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":48,"navigateFirstPage":44,"navigateLastPage":44},3,[11,25,35],{"slug":12,"title":13,"logo":7,"liveUrl":14,"prize":15,"sponsor":16,"description":17,"organizer":18,"startTime":19,"endTime":20,"registrationDeadline":21,"status":22,"dashboardVisibility":23,"createdAt":24},"spring2026","2026春季启元人工智能大赛","https:\u002F\u002Fwww.bilibili.com\u002Fblackboard\u002Flive\u002Flive-activity-player.html?cid=1822703816&quality=0","200,000","麒麟软件有限公司","“2026春季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。\n各参赛选手需在规定的比赛周期内，依据赛事规则选定赛题并完成相应挑战，优胜者将共同角逐总额20万元的赛事奖金池，并且还将有机会获得实习和知名企业优质岗位内推资格。","启元实验室","2026-04-19 16:00:00","2026-07-13 16:00:00","2026-06-16 16:00:00","ongoing","private","2026-04-16 08:00:00",{"slug":26,"title":27,"logo":7,"liveUrl":14,"prize":28,"sponsor":7,"description":29,"organizer":18,"startTime":30,"endTime":31,"registrationDeadline":32,"status":33,"dashboardVisibility":23,"createdAt":34},"autumn2025","2025秋季启元人工智能大赛","150,000","“2025秋季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。","2025-11-08 08:00:00","2026-01-13 08:00:00","2025-12-30 08:00:00","ended","2025-11-03 08:00:00",{"slug":36,"title":37,"logo":7,"liveUrl":14,"prize":38,"sponsor":7,"description":39,"organizer":18,"startTime":40,"endTime":41,"registrationDeadline":42,"status":33,"dashboardVisibility":23,"createdAt":43},"summer2025","启元人工智能大赛(系统方向)","80,000","启元人工智能大赛由启元实验室主办，其中系统赛道基于启元实验室九源软件栈的相关开源项目，通过拉动开源社区力量，一方面促进国产智能计算生态的发展，另一方面也为人工智能系统方向的学生及从业者提供一个实践平台。","2025-07-08 08:00:00","2025-08-25 08:00:00","2025-07-22 08:00:00","2025-06-22 08:00:00",1,5,true,8,[44],{"total":45,"list":50,"pageNum":44,"pageSize":45,"size":45,"startRow":44,"endRow":45,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":103,"navigateFirstPage":44,"navigateLastPage":44},[51,62,73,84,94],{"id":52,"slug":53,"title":54,"logo":55,"publicLiveEmbedUrl":14,"description":56,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":16,"startTime":58,"endTime":59,"status":22,"dashboardVisibility":60,"createdAt":58,"updatedAt":61},"019eeeff-6b00-7d55-a629-efafded3e232","summer2026","2026年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2026-06\u002F019eef7f-7816-7828-9309-79c2c61dd568.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=YuOgm%2FgmD4nuySjwNMl5nm0fmKI%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。","...","2026-06-22 08:00:00","2026-10-19 08:00:00","public","2026-10-12 08:00:00",{"id":63,"slug":64,"title":65,"logo":66,"publicLiveEmbedUrl":7,"description":67,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":68,"startTime":69,"endTime":70,"status":33,"dashboardVisibility":23,"createdAt":71,"updatedAt":72},"019af3f3-e06a-7245-aa8d-37dc99a69c73","winter2025","2025年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-12\u002F019afc69-b7ca-7805-91fb-605429671c01.png?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Kxv2UTqtU0%2FwvYQCDERqdg0W6E4%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。 \n","新华三技术有限公司","2025-12-25 08:00:00","2026-03-27 08:00:00","2025-12-06 08:00:00","2026-10-22 20:24:30",{"id":74,"slug":36,"title":75,"logo":76,"publicLiveEmbedUrl":7,"description":77,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":78,"startTime":79,"endTime":80,"status":81,"dashboardVisibility":23,"createdAt":82,"updatedAt":83},"01980342-208c-7803-b3f3-19e1d31960ba","2025年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-07\u002F0198045e-28fe-78f0-95fe-23835ce58395.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=nIij3MTYWjwdhBM00gMTqmyLnUM%3D","InfiniTensor大模型与人工智能系统训练营由启元实验室及清华大学 AI 系统方向团队发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营联合开源项目开发者、学术界大牛、工业界资深工程师参与授课与项目指导。课程采用「线上直播授课 + 在线答疑 + 项目实践辅导 」的方式进行，确保每位同学都能获得充分成长。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","北京积算科技有限公司","2025-07-14 08:00:00","2025-09-30 08:00:00","upcoming","2025-07-13 08:00:00","2026-10-22 20:24:58",{"id":85,"slug":86,"title":87,"logo":88,"publicLiveEmbedUrl":7,"description":89,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":90,"endTime":91,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":93},"01980cf9-a7d8-7796-ae86-0bbe05334add","winter2024","2024年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-ce23-7624-a287-9f07dd0d85f6.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。报名学员请阅读“导学”课程页的训练营介绍。","2025-01-05 08:00:00","2025-03-22 08:00:00","2025-07-15 08:00:00","2026-10-22 20:24:39",{"id":95,"slug":96,"title":97,"logo":98,"publicLiveEmbedUrl":7,"description":99,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":100,"endTime":101,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":102},"01980c20-363a-7dc3-8f5a-58a68e08dcc4","summer2024","2024年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-fbfa-788a-afcd-17fa9be74ff9.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区和高校，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过课程、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","2024-07-20 08:00:00","2024-09-29 08:00:00","2026-10-22 20:24:49",[44],{"currentItem":105,"processedData":125},{"img":106,"keywords":107,"treeType":44,"description":108,"showStatus":44,"updateTime":109,"type":110,"title":111,"content":112,"nextArticle":113,"createTime":109,"sortOrder":117,"canonicalPath":118,"id":119,"contentType":120,"prevArticle":121},"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_fm.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=yFCQ5kBS2v9OgfGnO%2BlE%2FI5H1ZU%3D","InfiniTensor,冬季训练营,并行策略,通信优化,大模型训练","介绍大模型并行策略与通信优化，说明多卡训练中的通信开销控制与效率提升。","2026-10-22 20:38:11",2,"InfiniTensor 冬季训练营通信与并行第二课《大模型并行策略与通信优化》","![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw01.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=LT77hQGEQ0KrF8BmFLloVQO8gL0%3D)\r\n\r\n2026 年 1 月 19 日，2025 冬季 InfiniTensor 大模型与人工智能系统训练营通信与并行系列第二课《大模型并行策略与通信优化》开讲。\r\n\r\n本节课将系统介绍大模型训练中常用的并行策略、通信基础原理及优化方法，帮助开发者理解并掌握大规模模型训练的关键技术。\r\n\r\n\r\n\r\n### 从单卡到多卡：训练需求的演变\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw02.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=jEzkx4I4KLQqSfDiM2ZOFP8S5vQ%3D)\r\n\r\n\r\n#### 1. 单卡训练的局限\r\n- 算力瓶颈：单张消费级显卡算力有限，训练大型模型需数月甚至数年\r\n- 显存限制：模型参数量超过显存容量\r\n- 训练效率：单卡训练速度慢，无法满足快速迭代需求\r\n\r\n#### 2. 多卡训练的必要性\r\n- 算力扩展：通过多卡并行，将计算任务分配到多张卡上\r\n- 显存扩展：通过模型切分，将模型参数分散到多张卡上\r\n- 训练加速：通过并行化，显著缩短训练时间\r\n\r\n### 大模型训练的三大并行策略\r\n\r\n#### 1. 数据并行（Data Parallelism）\r\n- 核心思想：每张卡上放置相同的模型，但处理不同的数据批次\r\n- 工作流程：\r\n   - 每张卡独立进行前向计算\r\n   - 计算梯度\r\n   - 梯度同步（如Allreduce）\r\n   - 更新模型参数\r\n- 优势：实现简单，适合大规模集群\r\n- 瓶颈：通信开销随卡数增加而增加\r\n\r\n#### 2. 模型并行（Model Parallelism）\r\n- 核心思想：将模型按层或参数切分，分配到不同卡上\r\n- 子类型：\r\n  - 流水线并行（Pipeline Parallelism）：按层切分，形成计算流水线\r\n  - 张量并行（Tensor Parallelism）：按参数维度切分，如将矩阵按列切分\r\n- 优势：解决单卡显存不足问题\r\n- 瓶颈：需要协调不同卡间的计算依赖\r\n\r\n#### 3. 混合并行（Hybrid Parallelism）\r\n- 核心思想：结合数据并行和模型并行，实现最优的资源利用\r\n- 典型实现：数据并行 + 流水线并行 + 张量并行\r\n- 优势：充分利用计算资源，适应不同规模的模型\r\n- 复杂性：实现难度较高，需要精细的调度策略\r\n\r\n#### 选择策略：\r\n- 根据模型规模、硬件资源和训练目标选择合适的并行策略。对于超大规模模型（>100B），通常需要采用混合并行策略。\r\n\r\n### 通信基础：点对点与集合通信\r\n\r\n#### 1. 点对点通信（Point-to-Point Communication）\r\n- 基本操作：发送端向接收端发送数据\r\n- 典型场景：在流水线并行中，将前一层的输出发送给下一层\r\n- 关键步骤：\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw03.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=bOQNIZCKTGPEbrIuJ%2Boglb42rg0%3D)\r\n\r\n#### 2. 集合通信（Collective Communication）\r\n- 核心功能：多卡间进行数据同步操作\r\n- 常用算子：\r\n  - ①Allreduce：求和\u002F平均所有节点数据。Allreduce 做规约（常见 sum），平均是 sum 后再除以 world size（有的框架提供 avg 但本质仍是规约+缩放）。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw04.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qJl%2BqqbBCVVRQlM7XHC15hfEd2U%3D)\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw05.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=npIEE66JFIsT75GQsFXoQMPLpl8%3D)\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw06.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=EqlqSFxtU7F5wfuYZiYabBMV4s0%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw07.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=GDJVVidV%2FFVcUYlfBrlEvu3%2Bx3s%3D)\r\n\r\n\r\n  - ②AllGather：收集所有节点数据\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw08.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=EunAaHjZZYb9g0ABZ3%2BbQT%2Fafx0%3D)\r\n\r\n  - ③AlltoAll：节点间的全交换通信。\r\n\r\n- 通信抽象：集合通信接口将复杂的通信操作抽象为简单的函数调用，使开发者无需了解底层实现细节。\r\n\r\n### 通信优化\r\n#### 1. 计算通信重叠\r\n- 问题：通信操作通常耗时较长，导致GPU空闲\r\n- 解决方案：在通信的同时，GPU继续执行计算任务\r\n- 分布式矩阵乘\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw09.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=cyA7VS1N21tfvXa44vPpLL9St3M%3D)\r\n\r\n#### 2. 拓扑感知\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw10.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Eg%2ByArWyjWrhtcOuwcoJW4tO2zI%3D)\r\n\r\n\r\n#### 3. DualPipe\r\n 1F1B 策略虽然让设备交替执行前向与后向计算，但受限于设备间的依赖关系，时间轴上留下了大量灰色的空闲“气泡”，造成算力浪费；ZB1P 在流水线调度中优先计算和传输输入梯度，并将权重梯度的计算延后，以填补流水线空闲时间。\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw11.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=bXT5pmNltY3qsCWypvxrq4kdTpk%3D)\r\n\r\nDeepSeek 提出的**DualPipe**通信优化技术的两种实施形态，旨在解决大规模模型训练中的通信瓶颈。上半部分的 **DualPipe** 描绘了一个**双向并行**的流水线系统（8个设备），它利用两组微批次数据从相反方向（Up to Down 和 Down to Up）穿过流水线，并结合 **ZB1P** 的细粒度切分策略（区分输入梯度和权重梯度），实现了**前向传播与后向传播的并行重叠**（图中橙绿混合色块），从而最大化地填补计算空隙并平衡网络带宽。下半部分的 **DualPipeV** 则是一种交错式（Interleaved）或虚拟化的变体，展示了如何在较少的物理设备（如4个）上，通过在同一设备上交替处理模型的“前半部分层”和“后半部分层”，在保持高吞吐量的同时灵活适配硬件资源，进一步减少流水线中的“气泡”。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw12.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NpQBSmsh0wi%2FllcU92m7hQ3iQSs%3D)\r\n\r\n\r\n通过具体的数学公式（Bubble 栏）证明：相比传统的 **1F1B** 和 **ZB1P**，**DualPipe** 系列方案能显著大幅减少流水线中的“气泡”（空闲等待时间）；但这同时也揭示了其代价——单设备的 **参数存储（Parameter Per Device）** 翻倍以及 **激活值（Activation）** 开销增加，体现了该算法“以显存空间换取计算时间”的核心优化哲学。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw13.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=5Ylpkj0BQY%2FiOQ0PBi2PtpHulLU%3D)\r\n\r\n\r\n\r\n### 总结与展望\r\n\r\n大模型训练的并行策略与通信优化是大规模AI训练的核心技术。通过合理选择并行策略、优化通信算法、实现通信与计算重叠，可以显著提升训练效率，降低训练成本。\r\n\r\n随着模型规模的持续扩大和硬件技术的不断进步，这些优化技术将变得越来越重要。对于AI从业者而言，深入理解并掌握这些技术，是高效训练大规模模型的关键。\r\n\r\n\r\n![2025冬季训练营课程表](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw6.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Ffj54EQtw2SzZEJdvnMd%2BBMhib0%3D)\r\n\r\n### 💡 学习贴士\r\n更多细节及完整课程内容，请观看直播或查看课程回放。\r\n\r\n链接：https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\r\n\r\n### 📚 训练营报名持续进行中\r\n扫码报名与加入群聊，或点击 **「阅读原文」** 直达官网报名\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw7.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ZU5%2F7VUywJvItSFp2MjcU%2FW%2FeuQ%3D)",{"publishTime":7,"createTime":109,"canonicalPath":114,"id":115,"title":116},"\u002Fstudy-material\u002Fstudy-21","study-21","InfiniTensor 冬季训练营 CPU 并行编程第二课《CPU 性能加速实战》",87,"\u002Fstudy-material\u002Fstudy-22","study-22","markdown",{"publishTime":7,"createTime":109,"canonicalPath":122,"id":123,"title":124},"\u002Fstudy-material\u002Fstudy-23","study-23","# InfiniTensor 冬季训练营 CUDA 编程第三课《内存模型与规约优化》",{"htmlContent":126,"headings":127},"\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw01.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=LT77hQGEQ0KrF8BmFLloVQO8gL0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>2026 年 1 月 19 日，2025 冬季 InfiniTensor 大模型与人工智能系统训练营通信与并行系列第二课《大模型并行策略与通信优化》开讲。\u003C\u002Fp>\n\u003Cp>本节课将系统介绍大模型训练中常用的并行策略、通信基础原理及优化方法，帮助开发者理解并掌握大规模模型训练的关键技术。\u003C\u002Fp>\n\u003Ch3 id=\"从单卡到多卡-训练需求的演变\">\u003Ca class=\"anchor\" href=\"#从单卡到多卡-训练需求的演变\" aria-label=\"跳转到标题：从单卡到多卡：训练需求的演变\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>从单卡到多卡：训练需求的演变\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw02.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=jEzkx4I4KLQqSfDiM2ZOFP8S5vQ%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"1-单卡训练的局限\">\u003Ca class=\"anchor\" href=\"#1-单卡训练的局限\" aria-label=\"跳转到标题：1. 单卡训练的局限\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 单卡训练的局限\u003C\u002Fh4>\u003Cul>\n\u003Cli>算力瓶颈：单张消费级显卡算力有限，训练大型模型需数月甚至数年\u003C\u002Fli>\n\u003Cli>显存限制：模型参数量超过显存容量\u003C\u002Fli>\n\u003Cli>训练效率：单卡训练速度慢，无法满足快速迭代需求\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-多卡训练的必要性\">\u003Ca class=\"anchor\" href=\"#2-多卡训练的必要性\" aria-label=\"跳转到标题：2. 多卡训练的必要性\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 多卡训练的必要性\u003C\u002Fh4>\u003Cul>\n\u003Cli>算力扩展：通过多卡并行，将计算任务分配到多张卡上\u003C\u002Fli>\n\u003Cli>显存扩展：通过模型切分，将模型参数分散到多张卡上\u003C\u002Fli>\n\u003Cli>训练加速：通过并行化，显著缩短训练时间\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"大模型训练的三大并行策略\">\u003Ca class=\"anchor\" href=\"#大模型训练的三大并行策略\" aria-label=\"跳转到标题：大模型训练的三大并行策略\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>大模型训练的三大并行策略\u003C\u002Fh3>\u003Ch4 id=\"1-数据并行-data-parallelism\">\u003Ca class=\"anchor\" href=\"#1-数据并行-data-parallelism\" aria-label=\"跳转到标题：1. 数据并行（Data Parallelism）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 数据并行（Data Parallelism）\u003C\u002Fh4>\u003Cul>\n\u003Cli>核心思想：每张卡上放置相同的模型，但处理不同的数据批次\u003C\u002Fli>\n\u003Cli>工作流程：\u003Cul>\n\u003Cli>每张卡独立进行前向计算\u003C\u002Fli>\n\u003Cli>计算梯度\u003C\u002Fli>\n\u003Cli>梯度同步（如Allreduce）\u003C\u002Fli>\n\u003Cli>更新模型参数\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003Cli>优势：实现简单，适合大规模集群\u003C\u002Fli>\n\u003Cli>瓶颈：通信开销随卡数增加而增加\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-模型并行-model-parallelism\">\u003Ca class=\"anchor\" href=\"#2-模型并行-model-parallelism\" aria-label=\"跳转到标题：2. 模型并行（Model Parallelism）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 模型并行（Model Parallelism）\u003C\u002Fh4>\u003Cul>\n\u003Cli>核心思想：将模型按层或参数切分，分配到不同卡上\u003C\u002Fli>\n\u003Cli>子类型：\u003Cul>\n\u003Cli>流水线并行（Pipeline Parallelism）：按层切分，形成计算流水线\u003C\u002Fli>\n\u003Cli>张量并行（Tensor Parallelism）：按参数维度切分，如将矩阵按列切分\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003Cli>优势：解决单卡显存不足问题\u003C\u002Fli>\n\u003Cli>瓶颈：需要协调不同卡间的计算依赖\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"3-混合并行-hybrid-parallelism\">\u003Ca class=\"anchor\" href=\"#3-混合并行-hybrid-parallelism\" aria-label=\"跳转到标题：3. 混合并行（Hybrid Parallelism）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 混合并行（Hybrid Parallelism）\u003C\u002Fh4>\u003Cul>\n\u003Cli>核心思想：结合数据并行和模型并行，实现最优的资源利用\u003C\u002Fli>\n\u003Cli>典型实现：数据并行 + 流水线并行 + 张量并行\u003C\u002Fli>\n\u003Cli>优势：充分利用计算资源，适应不同规模的模型\u003C\u002Fli>\n\u003Cli>复杂性：实现难度较高，需要精细的调度策略\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"选择策略\">\u003Ca class=\"anchor\" href=\"#选择策略\" aria-label=\"跳转到标题：选择策略：\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>选择策略：\u003C\u002Fh4>\u003Cul>\n\u003Cli>根据模型规模、硬件资源和训练目标选择合适的并行策略。对于超大规模模型（>100B），通常需要采用混合并行策略。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"通信基础-点对点与集合通信\">\u003Ca class=\"anchor\" href=\"#通信基础-点对点与集合通信\" aria-label=\"跳转到标题：通信基础：点对点与集合通信\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>通信基础：点对点与集合通信\u003C\u002Fh3>\u003Ch4 id=\"1-点对点通信-point-to-point-communication\">\u003Ca class=\"anchor\" href=\"#1-点对点通信-point-to-point-communication\" aria-label=\"跳转到标题：1. 点对点通信（Point-to-Point Communication）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 点对点通信（Point-to-Point Communication）\u003C\u002Fh4>\u003Cul>\n\u003Cli>基本操作：发送端向接收端发送数据\u003C\u002Fli>\n\u003Cli>典型场景：在流水线并行中，将前一层的输出发送给下一层\u003C\u002Fli>\n\u003Cli>关键步骤：\u003Cbr>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw03.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=bOQNIZCKTGPEbrIuJ%2Boglb42rg0%3D\" alt=\"\">\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-集合通信-collective-communication\">\u003Ca class=\"anchor\" href=\"#2-集合通信-collective-communication\" aria-label=\"跳转到标题：2. 集合通信（Collective Communication）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 集合通信（Collective Communication）\u003C\u002Fh4>\u003Cul>\n\u003Cli>核心功能：多卡间进行数据同步操作\u003C\u002Fli>\n\u003Cli>常用算子：\u003Cul>\n\u003Cli>①Allreduce：求和\u002F平均所有节点数据。Allreduce 做规约（常见 sum），平均是 sum 后再除以 world size（有的框架提供 avg 但本质仍是规约+缩放）。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw04.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qJl%2BqqbBCVVRQlM7XHC15hfEd2U%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw05.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=npIEE66JFIsT75GQsFXoQMPLpl8%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw06.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=EqlqSFxtU7F5wfuYZiYabBMV4s0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw07.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=GDJVVidV%2FFVcUYlfBrlEvu3%2Bx3s%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>②AllGather：收集所有节点数据\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw08.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=EunAaHjZZYb9g0ABZ3%2BbQT%2Fafx0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cp>③AlltoAll：节点间的全交换通信。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003Cli>\u003Cp>通信抽象：集合通信接口将复杂的通信操作抽象为简单的函数调用，使开发者无需了解底层实现细节。\u003C\u002Fp>\n\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"通信优化\">\u003Ca class=\"anchor\" href=\"#通信优化\" aria-label=\"跳转到标题：通信优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>通信优化\u003C\u002Fh3>\u003Ch4 id=\"1-计算通信重叠\">\u003Ca class=\"anchor\" href=\"#1-计算通信重叠\" aria-label=\"跳转到标题：1. 计算通信重叠\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 计算通信重叠\u003C\u002Fh4>\u003Cul>\n\u003Cli>问题：通信操作通常耗时较长，导致GPU空闲\u003C\u002Fli>\n\u003Cli>解决方案：在通信的同时，GPU继续执行计算任务\u003C\u002Fli>\n\u003Cli>分布式矩阵乘\u003Cbr>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw09.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=cyA7VS1N21tfvXa44vPpLL9St3M%3D\" alt=\"\">\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-拓扑感知\">\u003Ca class=\"anchor\" href=\"#2-拓扑感知\" aria-label=\"跳转到标题：2. 拓扑感知\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 拓扑感知\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw10.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Eg%2ByArWyjWrhtcOuwcoJW4tO2zI%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"3-dualpipe\">\u003Ca class=\"anchor\" href=\"#3-dualpipe\" aria-label=\"跳转到标题：3. DualPipe\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. DualPipe\u003C\u002Fh4>\u003Cp> 1F1B 策略虽然让设备交替执行前向与后向计算，但受限于设备间的依赖关系，时间轴上留下了大量灰色的空闲“气泡”，造成算力浪费；ZB1P 在流水线调度中优先计算和传输输入梯度，并将权重梯度的计算延后，以填补流水线空闲时间。\u003Cbr>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw11.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=bXT5pmNltY3qsCWypvxrq4kdTpk%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>DeepSeek 提出的\u003Cstrong>DualPipe\u003C\u002Fstrong>通信优化技术的两种实施形态，旨在解决大规模模型训练中的通信瓶颈。上半部分的 \u003Cstrong>DualPipe\u003C\u002Fstrong> 描绘了一个\u003Cstrong>双向并行\u003C\u002Fstrong>的流水线系统（8个设备），它利用两组微批次数据从相反方向（Up to Down 和 Down to Up）穿过流水线，并结合 \u003Cstrong>ZB1P\u003C\u002Fstrong> 的细粒度切分策略（区分输入梯度和权重梯度），实现了\u003Cstrong>前向传播与后向传播的并行重叠\u003C\u002Fstrong>（图中橙绿混合色块），从而最大化地填补计算空隙并平衡网络带宽。下半部分的 \u003Cstrong>DualPipeV\u003C\u002Fstrong> 则是一种交错式（Interleaved）或虚拟化的变体，展示了如何在较少的物理设备（如4个）上，通过在同一设备上交替处理模型的“前半部分层”和“后半部分层”，在保持高吞吐量的同时灵活适配硬件资源，进一步减少流水线中的“气泡”。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw12.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NpQBSmsh0wi%2FllcU92m7hQ3iQSs%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>通过具体的数学公式（Bubble 栏）证明：相比传统的 \u003Cstrong>1F1B\u003C\u002Fstrong> 和 \u003Cstrong>ZB1P\u003C\u002Fstrong>，\u003Cstrong>DualPipe\u003C\u002Fstrong> 系列方案能显著大幅减少流水线中的“气泡”（空闲等待时间）；但这同时也揭示了其代价——单设备的 \u003Cstrong>参数存储（Parameter Per Device）\u003C\u002Fstrong> 翻倍以及 \u003Cstrong>激活值（Activation）\u003C\u002Fstrong> 开销增加，体现了该算法“以显存空间换取计算时间”的核心优化哲学。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_22_zw13.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=5Ylpkj0BQY%2FiOQ0PBi2PtpHulLU%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"总结与展望\">\u003Ca class=\"anchor\" href=\"#总结与展望\" aria-label=\"跳转到标题：总结与展望\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>总结与展望\u003C\u002Fh3>\u003Cp>大模型训练的并行策略与通信优化是大规模AI训练的核心技术。通过合理选择并行策略、优化通信算法、实现通信与计算重叠，可以显著提升训练效率，降低训练成本。\u003C\u002Fp>\n\u003Cp>随着模型规模的持续扩大和硬件技术的不断进步，这些优化技术将变得越来越重要。对于AI从业者而言，深入理解并掌握这些技术，是高效训练大规模模型的关键。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw6.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Ffj54EQtw2SzZEJdvnMd%2BBMhib0%3D\" alt=\"2025冬季训练营课程表\">\u003C\u002Fp>\n\u003Ch3 id=\"学习贴士\">\u003Ca class=\"anchor\" href=\"#学习贴士\" aria-label=\"跳转到标题：💡 学习贴士\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>💡 学习贴士\u003C\u002Fh3>\u003Cp>更多细节及完整课程内容，请观看直播或查看课程回放。\u003C\u002Fp>\n\u003Cp>链接：\u003Ca href=\"https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\">https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\u003C\u002Fa>\u003C\u002Fp>\n\u003Ch3 id=\"训练营报名持续进行中\">\u003Ca class=\"anchor\" href=\"#训练营报名持续进行中\" aria-label=\"跳转到标题：📚 训练营报名持续进行中\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>📚 训练营报名持续进行中\u003C\u002Fh3>\u003Cp>扫码报名与加入群聊，或点击 \u003Cstrong>「阅读原文」\u003C\u002Fstrong> 直达官网报名\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw7.jpg?Expires=1792752821&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ZU5%2F7VUywJvItSFp2MjcU%2FW%2FeuQ%3D\" alt=\"\">\u003C\u002Fp>\n",[128,131,135,138,140,143,146,149,152,155,158,161,163,166,169,172,174,177],{"id":129,"text":130,"level":9},"从单卡到多卡-训练需求的演变","从单卡到多卡：训练需求的演变",{"id":132,"text":133,"level":134},"1-单卡训练的局限","1. 单卡训练的局限",4,{"id":136,"text":137,"level":134},"2-多卡训练的必要性","2. 多卡训练的必要性",{"id":139,"text":139,"level":9},"大模型训练的三大并行策略",{"id":141,"text":142,"level":134},"1-数据并行-data-parallelism","1. 数据并行（Data Parallelism）",{"id":144,"text":145,"level":134},"2-模型并行-model-parallelism","2. 模型并行（Model Parallelism）",{"id":147,"text":148,"level":134},"3-混合并行-hybrid-parallelism","3. 混合并行（Hybrid Parallelism）",{"id":150,"text":151,"level":134},"选择策略","选择策略：",{"id":153,"text":154,"level":9},"通信基础-点对点与集合通信","通信基础：点对点与集合通信",{"id":156,"text":157,"level":134},"1-点对点通信-point-to-point-communication","1. 点对点通信（Point-to-Point Communication）",{"id":159,"text":160,"level":134},"2-集合通信-collective-communication","2. 集合通信（Collective Communication）",{"id":162,"text":162,"level":9},"通信优化",{"id":164,"text":165,"level":134},"1-计算通信重叠","1. 计算通信重叠",{"id":167,"text":168,"level":134},"2-拓扑感知","2. 拓扑感知",{"id":170,"text":171,"level":134},"3-dualpipe","3. DualPipe",{"id":173,"text":173,"level":9},"总结与展望",{"id":175,"text":176,"level":9},"学习贴士","💡 学习贴士",{"id":178,"text":179,"level":9},"训练营报名持续进行中","📚 训练营报名持续进行中"]