[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"site-home-live-entry-setting":3,"site-header-contests":8,"site-header-camps":49,"study-material-detail-study-26":104},{"enabled":4,"title":5,"statusText":5,"buttonText":5,"campSlug":5,"stage":6,"classroomId":7,"hasClassroom":4},false,"",0,null,{"total":9,"list":10,"pageNum":44,"pageSize":45,"size":9,"startRow":44,"endRow":9,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":48,"navigateFirstPage":44,"navigateLastPage":44},3,[11,25,35],{"slug":12,"title":13,"logo":7,"liveUrl":14,"prize":15,"sponsor":16,"description":17,"organizer":18,"startTime":19,"endTime":20,"registrationDeadline":21,"status":22,"dashboardVisibility":23,"createdAt":24},"spring2026","2026春季启元人工智能大赛","https:\u002F\u002Fwww.bilibili.com\u002Fblackboard\u002Flive\u002Flive-activity-player.html?cid=1822703816&quality=0","200,000","麒麟软件有限公司","“2026春季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。\n各参赛选手需在规定的比赛周期内，依据赛事规则选定赛题并完成相应挑战，优胜者将共同角逐总额20万元的赛事奖金池，并且还将有机会获得实习和知名企业优质岗位内推资格。","启元实验室","2026-04-19 16:00:00","2026-07-13 16:00:00","2026-06-16 16:00:00","ongoing","private","2026-04-16 08:00:00",{"slug":26,"title":27,"logo":7,"liveUrl":14,"prize":28,"sponsor":7,"description":29,"organizer":18,"startTime":30,"endTime":31,"registrationDeadline":32,"status":33,"dashboardVisibility":23,"createdAt":34},"autumn2025","2025秋季启元人工智能大赛","150,000","“2025秋季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。","2025-11-08 08:00:00","2026-01-13 08:00:00","2025-12-30 08:00:00","ended","2025-11-03 08:00:00",{"slug":36,"title":37,"logo":7,"liveUrl":14,"prize":38,"sponsor":7,"description":39,"organizer":18,"startTime":40,"endTime":41,"registrationDeadline":42,"status":33,"dashboardVisibility":23,"createdAt":43},"summer2025","启元人工智能大赛(系统方向)","80,000","启元人工智能大赛由启元实验室主办，其中系统赛道基于启元实验室九源软件栈的相关开源项目，通过拉动开源社区力量，一方面促进国产智能计算生态的发展，另一方面也为人工智能系统方向的学生及从业者提供一个实践平台。","2025-07-08 08:00:00","2025-08-25 08:00:00","2025-07-22 08:00:00","2025-06-22 08:00:00",1,5,true,8,[44],{"total":45,"list":50,"pageNum":44,"pageSize":45,"size":45,"startRow":44,"endRow":45,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":103,"navigateFirstPage":44,"navigateLastPage":44},[51,62,73,84,94],{"id":52,"slug":53,"title":54,"logo":55,"publicLiveEmbedUrl":14,"description":56,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":16,"startTime":58,"endTime":59,"status":22,"dashboardVisibility":60,"createdAt":58,"updatedAt":61},"019eeeff-6b00-7d55-a629-efafded3e232","summer2026","2026年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2026-06\u002F019eef7f-7816-7828-9309-79c2c61dd568.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ihPLIAuCbx%2FbInTQLbtnjbL%2FgEc%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。","...","2026-06-22 08:00:00","2026-10-19 08:00:00","public","2026-10-12 08:00:00",{"id":63,"slug":64,"title":65,"logo":66,"publicLiveEmbedUrl":7,"description":67,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":68,"startTime":69,"endTime":70,"status":33,"dashboardVisibility":23,"createdAt":71,"updatedAt":72},"019af3f3-e06a-7245-aa8d-37dc99a69c73","winter2025","2025年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-12\u002F019afc69-b7ca-7805-91fb-605429671c01.png?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=RFJPol03COkEgvG5I%2FXurn1xrVs%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。 \n","新华三技术有限公司","2025-12-25 08:00:00","2026-03-27 08:00:00","2025-12-06 08:00:00","2026-10-22 20:24:30",{"id":74,"slug":36,"title":75,"logo":76,"publicLiveEmbedUrl":7,"description":77,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":78,"startTime":79,"endTime":80,"status":81,"dashboardVisibility":23,"createdAt":82,"updatedAt":83},"01980342-208c-7803-b3f3-19e1d31960ba","2025年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-07\u002F0198045e-28fe-78f0-95fe-23835ce58395.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=3EgCBk5kjXXxH6wOy5O8vDSNjzc%3D","InfiniTensor大模型与人工智能系统训练营由启元实验室及清华大学 AI 系统方向团队发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营联合开源项目开发者、学术界大牛、工业界资深工程师参与授课与项目指导。课程采用「线上直播授课 + 在线答疑 + 项目实践辅导 」的方式进行，确保每位同学都能获得充分成长。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","北京积算科技有限公司","2025-07-14 08:00:00","2025-09-30 08:00:00","upcoming","2025-07-13 08:00:00","2026-10-22 20:24:58",{"id":85,"slug":86,"title":87,"logo":88,"publicLiveEmbedUrl":7,"description":89,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":90,"endTime":91,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":93},"01980cf9-a7d8-7796-ae86-0bbe05334add","winter2024","2024年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-ce23-7624-a287-9f07dd0d85f6.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。报名学员请阅读“导学”课程页的训练营介绍。","2025-01-05 08:00:00","2025-03-22 08:00:00","2025-07-15 08:00:00","2026-10-22 20:24:39",{"id":95,"slug":96,"title":97,"logo":98,"publicLiveEmbedUrl":7,"description":99,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":100,"endTime":101,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":102},"01980c20-363a-7dc3-8f5a-58a68e08dcc4","summer2024","2024年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-fbfa-788a-afcd-17fa9be74ff9.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区和高校，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过课程、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","2024-07-20 08:00:00","2024-09-29 08:00:00","2026-10-22 20:24:49",[44],{"currentItem":105,"processedData":125},{"img":106,"keywords":107,"treeType":44,"description":108,"showStatus":44,"updateTime":109,"type":110,"title":111,"content":112,"nextArticle":113,"createTime":109,"sortOrder":117,"canonicalPath":118,"id":119,"contentType":120,"prevArticle":121},"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_fm.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=87tBYRO3emBVQpqqI8qtL1ITjrg%3D","InfiniTensor,冬季训练营,分布式训练,训练系统,并行训练","系统介绍分布式训练基础与实践，涵盖并行训练流程、资源调度与训练效率优化。","2026-10-22 20:38:11",2,"InfiniTensor 冬季训练营训练系统第二课《分布式训练》","![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw01.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=bb91qht32acvLjzBRCypxKq0X2E%3D)\r\n\r\n\r\n2026 年 1 月 20 日，2025 冬季 InfiniTensor 大模型与人工智能系统训练营训练系统方向第二课《分布式训练》开讲。\r\n\r\n本节课将讲解分布式训练的背景、基本概念、具体策略及其应用。\r\n\r\n### 分布式训练的必要性\r\n- **小模型时代**的训练方式：单张显卡即可完成训练，显存占用小，训练时间可接受。\r\n- **大模型时代**的挑战：模型参数激增，单张显卡显存不足，无法满足训练需求。\r\n- **SOTA 模型参数增长趋势**：展示近年来 SOTA 模型参数数量指数级增长，而 GPU 显存容量增长缓慢，导致显存容量与模型规模不匹配。\r\n- **算力需求增长**：Transformer 模型计算量需求显著增加，远超芯片算力增长速度。\r\n- **内存墙问题**：计算单元速度提升快于内存带宽和访问速度，导致计算单元空闲等待数据传输，影响训练效率。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw02.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=nfXFjJ65%2BU6Dn3GdDfsE78JF0Aw%3D)\r\n\r\n\r\n### 分布式训练的基本原理\r\n#### 1. 定义与目的：\r\n- 通过模型切分和数据切分，将训练任务分布到多台设备上协同执行，突破单卡显存限制，聚合带宽和算力，缓解内存墙问题。\r\n#### 2. 模型并行与数据并行的区别：\r\n- 模型并行：针对模型大的问题，将模型拆分成多个部分，分布到不同机器上训练。\r\n- 数据并行：针对数据大的问题，将数据拆分成多个小数据片，分布到不同机器上训练。\r\n\r\n### 数据并行\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw03.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Ycxtf0SQEwH1gOGhrzGcRtg3ORU%3D)\r\n\r\n#### 1. 分布式数据并行：\r\n- 数据切分：将数据 x 切分，每个设备上的参数 w 完整。\r\n- 输出拼接：各设备输出拼接得到完整 out。\r\n- 梯度聚合：反向传播时，各设备梯度聚合确保模型参数一致。\r\n#### 2. 梯度聚合算法：\r\n- 参数服务器版本\r\n- Allreduce 版本\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw04.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kwaNztvfXxFetxP6uxHDBXZdoiA%3D)\r\n\r\n#### 3. 数据并行的扩展瓶颈\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw05.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=SCIoFg5Z%2B9jdjO1sbmDOVy5UM1o%3D)\r\n\r\n\r\n### 模型并行\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw06.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NIs7TOG9Zh%2BhqlgJySm6a2XM7DE%3D)\r\n\r\n#### 1. 张量并行：分层内矩阵，跨设备计算\r\n- **优点**：\r\n  - 将单层中的权重张量按维度切分到多个设备上，降低单设备的权重内存占用\r\n  - 在计算密集、通信带宽充足的场景下，可提升矩阵计算的并行度和整体吞吐\r\n- **缺点**：\r\n    - 通信频繁（中间输出需 AllReduce）\r\n    - 将原本的大矩阵切分为小矩阵，可能降低 GPU 矩阵乘效率\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw07.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=pO%2BKNapYfPbMUbokJQw1Zmgfpzk%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw08.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=veaDzsd2nyFzyY224umuiLUHHzA%3D)\r\n\r\n\r\n#### 2. 流水并行\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw09.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=H%2FGCgNJq8OB6zWEejc56qx53oDA%3D)\r\n\r\n- **缺点**：存在空闲状态，计算资源可能浪费\r\n- **优化策略**：Gpipe、1F1B 等，通过细粒度切分和计算减少空闲时间。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw10.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Wv4jC1NU5vhJyEJAmsnbf9pf3cc%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw11.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ZY9oMXgKu15yxbnMRJAX3csiOdA%3D)\r\n\r\n\r\n\r\n#### 3. 专家并行\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw12.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=m7t%2B3%2BDyv%2BwtOLzhFlvTzrLoOPw%3D)\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw13.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qkv5N%2B5Tjs2p9MR7CTunrQI4E9E%3D)\r\n\r\n\r\n### 混合并行\r\n- 混合并行必要性：大规模模型训练需结合多种并行策略。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw14.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=d1RJZ4VrbAMae4gsRo6Uo2jqcqo%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw15.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=KhrRWmPbRcqjMBEMiMKk3pQPofc%3D)\r\n\r\n\r\n\r\n### 总结\r\n本节课全面讲述了分布式训练的背景、基本概念、具体策略及其应用。通过详细的知识点讲解和实际案例分析，帮助学员掌握分布式训练的核心技术和方法。\r\n\r\n\r\n\r\n\r\n![2025冬季训练营课程表](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw6.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=a8H1mmcFuHQlXPwOb9vIkgGHb3g%3D)\r\n\r\n### 💡 学习贴士\r\n更多细节及完整课程内容，请观看直播或查看课程回放。\r\n\r\n链接：https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\r\n\r\n### 📚 训练营报名持续进行中\r\n扫码报名与加入群聊，或点击 **「阅读原文」** 直达官网报名\r\n\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw7.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=1gVq94K1yhwDat7%2FIiB1Q44urNk%3D)",{"publishTime":7,"createTime":109,"canonicalPath":114,"id":115,"title":116},"\u002Fstudy-material\u002Fstudy-25","study-25","InfiniTensor 冬季训练营 CPU 并行编程第三课《鲲鹏 CPU 矩阵加速与特性介绍》",83,"\u002Fstudy-material\u002Fstudy-26","study-26","markdown",{"publishTime":7,"createTime":109,"canonicalPath":122,"id":123,"title":124},"\u002Fstudy-material\u002Fstudy-27","study-27","InfiniTensor 冬季训练营 AI 编译器第三课《PyTorch 中图优化》",{"htmlContent":126,"headings":127},"\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw01.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=bb91qht32acvLjzBRCypxKq0X2E%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>2026 年 1 月 20 日，2025 冬季 InfiniTensor 大模型与人工智能系统训练营训练系统方向第二课《分布式训练》开讲。\u003C\u002Fp>\n\u003Cp>本节课将讲解分布式训练的背景、基本概念、具体策略及其应用。\u003C\u002Fp>\n\u003Ch3 id=\"分布式训练的必要性\">\u003Ca class=\"anchor\" href=\"#分布式训练的必要性\" aria-label=\"跳转到标题：分布式训练的必要性\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>分布式训练的必要性\u003C\u002Fh3>\u003Cul>\n\u003Cli>\u003Cstrong>小模型时代\u003C\u002Fstrong>的训练方式：单张显卡即可完成训练，显存占用小，训练时间可接受。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>大模型时代\u003C\u002Fstrong>的挑战：模型参数激增，单张显卡显存不足，无法满足训练需求。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>SOTA 模型参数增长趋势\u003C\u002Fstrong>：展示近年来 SOTA 模型参数数量指数级增长，而 GPU 显存容量增长缓慢，导致显存容量与模型规模不匹配。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>算力需求增长\u003C\u002Fstrong>：Transformer 模型计算量需求显著增加，远超芯片算力增长速度。\u003C\u002Fli>\n\u003Cli>\u003Cstrong>内存墙问题\u003C\u002Fstrong>：计算单元速度提升快于内存带宽和访问速度，导致计算单元空闲等待数据传输，影响训练效率。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw02.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=nfXFjJ65%2BU6Dn3GdDfsE78JF0Aw%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"分布式训练的基本原理\">\u003Ca class=\"anchor\" href=\"#分布式训练的基本原理\" aria-label=\"跳转到标题：分布式训练的基本原理\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>分布式训练的基本原理\u003C\u002Fh3>\u003Ch4 id=\"1-定义与目的\">\u003Ca class=\"anchor\" href=\"#1-定义与目的\" aria-label=\"跳转到标题：1. 定义与目的：\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 定义与目的：\u003C\u002Fh4>\u003Cul>\n\u003Cli>通过模型切分和数据切分，将训练任务分布到多台设备上协同执行，突破单卡显存限制，聚合带宽和算力，缓解内存墙问题。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-模型并行与数据并行的区别\">\u003Ca class=\"anchor\" href=\"#2-模型并行与数据并行的区别\" aria-label=\"跳转到标题：2. 模型并行与数据并行的区别：\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 模型并行与数据并行的区别：\u003C\u002Fh4>\u003Cul>\n\u003Cli>模型并行：针对模型大的问题，将模型拆分成多个部分，分布到不同机器上训练。\u003C\u002Fli>\n\u003Cli>数据并行：针对数据大的问题，将数据拆分成多个小数据片，分布到不同机器上训练。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"数据并行\">\u003Ca class=\"anchor\" href=\"#数据并行\" aria-label=\"跳转到标题：数据并行\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>数据并行\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw03.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Ycxtf0SQEwH1gOGhrzGcRtg3ORU%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"1-分布式数据并行\">\u003Ca class=\"anchor\" href=\"#1-分布式数据并行\" aria-label=\"跳转到标题：1. 分布式数据并行：\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 分布式数据并行：\u003C\u002Fh4>\u003Cul>\n\u003Cli>数据切分：将数据 x 切分，每个设备上的参数 w 完整。\u003C\u002Fli>\n\u003Cli>输出拼接：各设备输出拼接得到完整 out。\u003C\u002Fli>\n\u003Cli>梯度聚合：反向传播时，各设备梯度聚合确保模型参数一致。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-梯度聚合算法\">\u003Ca class=\"anchor\" href=\"#2-梯度聚合算法\" aria-label=\"跳转到标题：2. 梯度聚合算法：\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 梯度聚合算法：\u003C\u002Fh4>\u003Cul>\n\u003Cli>参数服务器版本\u003C\u002Fli>\n\u003Cli>Allreduce 版本\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw04.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kwaNztvfXxFetxP6uxHDBXZdoiA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"3-数据并行的扩展瓶颈\">\u003Ca class=\"anchor\" href=\"#3-数据并行的扩展瓶颈\" aria-label=\"跳转到标题：3. 数据并行的扩展瓶颈\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 数据并行的扩展瓶颈\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw05.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=SCIoFg5Z%2B9jdjO1sbmDOVy5UM1o%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"模型并行\">\u003Ca class=\"anchor\" href=\"#模型并行\" aria-label=\"跳转到标题：模型并行\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>模型并行\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw06.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NIs7TOG9Zh%2BhqlgJySm6a2XM7DE%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"1-张量并行-分层内矩阵-跨设备计算\">\u003Ca class=\"anchor\" href=\"#1-张量并行-分层内矩阵-跨设备计算\" aria-label=\"跳转到标题：1. 张量并行：分层内矩阵，跨设备计算\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 张量并行：分层内矩阵，跨设备计算\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>优点\u003C\u002Fstrong>：\u003Cul>\n\u003Cli>将单层中的权重张量按维度切分到多个设备上，降低单设备的权重内存占用\u003C\u002Fli>\n\u003Cli>在计算密集、通信带宽充足的场景下，可提升矩阵计算的并行度和整体吞吐\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003Cli>\u003Cstrong>缺点\u003C\u002Fstrong>：\u003Cul>\n\u003Cli>通信频繁（中间输出需 AllReduce）\u003C\u002Fli>\n\u003Cli>将原本的大矩阵切分为小矩阵，可能降低 GPU 矩阵乘效率\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw07.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=pO%2BKNapYfPbMUbokJQw1Zmgfpzk%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw08.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=veaDzsd2nyFzyY224umuiLUHHzA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"2-流水并行\">\u003Ca class=\"anchor\" href=\"#2-流水并行\" aria-label=\"跳转到标题：2. 流水并行\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 流水并行\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw09.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=H%2FGCgNJq8OB6zWEejc56qx53oDA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>缺点\u003C\u002Fstrong>：存在空闲状态，计算资源可能浪费\u003C\u002Fli>\n\u003Cli>\u003Cstrong>优化策略\u003C\u002Fstrong>：Gpipe、1F1B 等，通过细粒度切分和计算减少空闲时间。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw10.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Wv4jC1NU5vhJyEJAmsnbf9pf3cc%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw11.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ZY9oMXgKu15yxbnMRJAX3csiOdA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"3-专家并行\">\u003Ca class=\"anchor\" href=\"#3-专家并行\" aria-label=\"跳转到标题：3. 专家并行\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 专家并行\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw12.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=m7t%2B3%2BDyv%2BwtOLzhFlvTzrLoOPw%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw13.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qkv5N%2B5Tjs2p9MR7CTunrQI4E9E%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"混合并行\">\u003Ca class=\"anchor\" href=\"#混合并行\" aria-label=\"跳转到标题：混合并行\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>混合并行\u003C\u002Fh3>\u003Cul>\n\u003Cli>混合并行必要性：大规模模型训练需结合多种并行策略。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw14.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=d1RJZ4VrbAMae4gsRo6Uo2jqcqo%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_26_zw15.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=KhrRWmPbRcqjMBEMiMKk3pQPofc%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"总结\">\u003Ca class=\"anchor\" href=\"#总结\" aria-label=\"跳转到标题：总结\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>总结\u003C\u002Fh3>\u003Cp>本节课全面讲述了分布式训练的背景、基本概念、具体策略及其应用。通过详细的知识点讲解和实际案例分析，帮助学员掌握分布式训练的核心技术和方法。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw6.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=a8H1mmcFuHQlXPwOb9vIkgGHb3g%3D\" alt=\"2025冬季训练营课程表\">\u003C\u002Fp>\n\u003Ch3 id=\"学习贴士\">\u003Ca class=\"anchor\" href=\"#学习贴士\" aria-label=\"跳转到标题：💡 学习贴士\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>💡 学习贴士\u003C\u002Fh3>\u003Cp>更多细节及完整课程内容，请观看直播或查看课程回放。\u003C\u002Fp>\n\u003Cp>链接：\u003Ca href=\"https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\">https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\u003C\u002Fa>\u003C\u002Fp>\n\u003Ch3 id=\"训练营报名持续进行中\">\u003Ca class=\"anchor\" href=\"#训练营报名持续进行中\" aria-label=\"跳转到标题：📚 训练营报名持续进行中\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>📚 训练营报名持续进行中\u003C\u002Fh3>\u003Cp>扫码报名与加入群聊，或点击 \u003Cstrong>「阅读原文」\u003C\u002Fstrong> 直达官网报名\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw7.jpg?Expires=1792752556&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=1gVq94K1yhwDat7%2FIiB1Q44urNk%3D\" alt=\"\">\u003C\u002Fp>\n",[128,130,132,136,139,141,144,147,150,152,155,158,161,163,165,168],{"id":129,"text":129,"level":9},"分布式训练的必要性",{"id":131,"text":131,"level":9},"分布式训练的基本原理",{"id":133,"text":134,"level":135},"1-定义与目的","1. 定义与目的：",4,{"id":137,"text":138,"level":135},"2-模型并行与数据并行的区别","2. 模型并行与数据并行的区别：",{"id":140,"text":140,"level":9},"数据并行",{"id":142,"text":143,"level":135},"1-分布式数据并行","1. 分布式数据并行：",{"id":145,"text":146,"level":135},"2-梯度聚合算法","2. 梯度聚合算法：",{"id":148,"text":149,"level":135},"3-数据并行的扩展瓶颈","3. 数据并行的扩展瓶颈",{"id":151,"text":151,"level":9},"模型并行",{"id":153,"text":154,"level":135},"1-张量并行-分层内矩阵-跨设备计算","1. 张量并行：分层内矩阵，跨设备计算",{"id":156,"text":157,"level":135},"2-流水并行","2. 流水并行",{"id":159,"text":160,"level":135},"3-专家并行","3. 专家并行",{"id":162,"text":162,"level":9},"混合并行",{"id":164,"text":164,"level":9},"总结",{"id":166,"text":167,"level":9},"学习贴士","💡 学习贴士",{"id":169,"text":170,"level":9},"训练营报名持续进行中","📚 训练营报名持续进行中"]