[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"site-home-live-entry-setting":3,"site-header-contests":8,"site-header-camps":49,"study-material-detail-study-33":104},{"enabled":4,"title":5,"statusText":5,"buttonText":5,"campSlug":5,"stage":6,"classroomId":7,"hasClassroom":4},false,"",0,null,{"total":9,"list":10,"pageNum":44,"pageSize":45,"size":9,"startRow":44,"endRow":9,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":48,"navigateFirstPage":44,"navigateLastPage":44},3,[11,25,35],{"slug":12,"title":13,"logo":7,"liveUrl":14,"prize":15,"sponsor":16,"description":17,"organizer":18,"startTime":19,"endTime":20,"registrationDeadline":21,"status":22,"dashboardVisibility":23,"createdAt":24},"spring2026","2026春季启元人工智能大赛","https:\u002F\u002Fwww.bilibili.com\u002Fblackboard\u002Flive\u002Flive-activity-player.html?cid=1822703816&quality=0","200,000","麒麟软件有限公司","“2026春季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。\n各参赛选手需在规定的比赛周期内，依据赛事规则选定赛题并完成相应挑战，优胜者将共同角逐总额20万元的赛事奖金池，并且还将有机会获得实习和知名企业优质岗位内推资格。","启元实验室","2026-04-19 16:00:00","2026-07-13 16:00:00","2026-06-16 16:00:00","ongoing","private","2026-04-16 08:00:00",{"slug":26,"title":27,"logo":7,"liveUrl":14,"prize":28,"sponsor":7,"description":29,"organizer":18,"startTime":30,"endTime":31,"registrationDeadline":32,"status":33,"dashboardVisibility":23,"createdAt":34},"autumn2025","2025秋季启元人工智能大赛","150,000","“2025秋季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。","2025-11-08 08:00:00","2026-01-13 08:00:00","2025-12-30 08:00:00","ended","2025-11-03 08:00:00",{"slug":36,"title":37,"logo":7,"liveUrl":14,"prize":38,"sponsor":7,"description":39,"organizer":18,"startTime":40,"endTime":41,"registrationDeadline":42,"status":33,"dashboardVisibility":23,"createdAt":43},"summer2025","启元人工智能大赛(系统方向)","80,000","启元人工智能大赛由启元实验室主办，其中系统赛道基于启元实验室九源软件栈的相关开源项目，通过拉动开源社区力量，一方面促进国产智能计算生态的发展，另一方面也为人工智能系统方向的学生及从业者提供一个实践平台。","2025-07-08 08:00:00","2025-08-25 08:00:00","2025-07-22 08:00:00","2025-06-22 08:00:00",1,5,true,8,[44],{"total":45,"list":50,"pageNum":44,"pageSize":45,"size":45,"startRow":44,"endRow":45,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":103,"navigateFirstPage":44,"navigateLastPage":44},[51,62,73,84,94],{"id":52,"slug":53,"title":54,"logo":55,"publicLiveEmbedUrl":14,"description":56,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":16,"startTime":58,"endTime":59,"status":22,"dashboardVisibility":60,"createdAt":58,"updatedAt":61},"019eeeff-6b00-7d55-a629-efafded3e232","summer2026","2026年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2026-06\u002F019eef7f-7816-7828-9309-79c2c61dd568.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=nCcM9ECgK5PlmraUwj%2BH%2F1S%2BELk%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。","...","2026-06-22 08:00:00","2026-10-19 08:00:00","public","2026-10-12 08:00:00",{"id":63,"slug":64,"title":65,"logo":66,"publicLiveEmbedUrl":7,"description":67,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":68,"startTime":69,"endTime":70,"status":33,"dashboardVisibility":23,"createdAt":71,"updatedAt":72},"019af3f3-e06a-7245-aa8d-37dc99a69c73","winter2025","2025年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-12\u002F019afc69-b7ca-7805-91fb-605429671c01.png?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kRi1yuIH5NG2docsx57MXWlzm1k%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。 \n","新华三技术有限公司","2025-12-25 08:00:00","2026-03-27 08:00:00","2025-12-06 08:00:00","2026-10-22 20:24:30",{"id":74,"slug":36,"title":75,"logo":76,"publicLiveEmbedUrl":7,"description":77,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":78,"startTime":79,"endTime":80,"status":81,"dashboardVisibility":23,"createdAt":82,"updatedAt":83},"01980342-208c-7803-b3f3-19e1d31960ba","2025年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-07\u002F0198045e-28fe-78f0-95fe-23835ce58395.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=eT9ZAOmPWlOldw%2FMHXc0X0x2%2Bhg%3D","InfiniTensor大模型与人工智能系统训练营由启元实验室及清华大学 AI 系统方向团队发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营联合开源项目开发者、学术界大牛、工业界资深工程师参与授课与项目指导。课程采用「线上直播授课 + 在线答疑 + 项目实践辅导 」的方式进行，确保每位同学都能获得充分成长。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","北京积算科技有限公司","2025-07-14 08:00:00","2025-09-30 08:00:00","upcoming","2025-07-13 08:00:00","2026-10-22 20:24:58",{"id":85,"slug":86,"title":87,"logo":88,"publicLiveEmbedUrl":7,"description":89,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":90,"endTime":91,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":93},"01980cf9-a7d8-7796-ae86-0bbe05334add","winter2024","2024年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-ce23-7624-a287-9f07dd0d85f6.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。报名学员请阅读“导学”课程页的训练营介绍。","2025-01-05 08:00:00","2025-03-22 08:00:00","2025-07-15 08:00:00","2026-10-22 20:24:39",{"id":95,"slug":96,"title":97,"logo":98,"publicLiveEmbedUrl":7,"description":99,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":100,"endTime":101,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":102},"01980c20-363a-7dc3-8f5a-58a68e08dcc4","summer2024","2024年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-fbfa-788a-afcd-17fa9be74ff9.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区和高校，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过课程、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","2024-07-20 08:00:00","2024-09-29 08:00:00","2026-10-22 20:24:49",[44],{"currentItem":105,"processedData":125},{"img":106,"keywords":107,"treeType":44,"description":108,"showStatus":44,"updateTime":109,"type":110,"title":111,"content":112,"nextArticle":113,"createTime":109,"sortOrder":117,"canonicalPath":118,"id":119,"contentType":120,"prevArticle":121},"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_fm.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=BrDOuoT9cBw%2F4aJ%2BITypcrfxhrI%3D","InfiniTensor,冬季训练营,大模型架构,前沿模型,技术趋势","梳理大模型前沿架构演进方向，介绍主流结构特点、能力边界与应用趋势。","2026-10-22 20:38:11",2,"InfiniTensor 冬季训练营前沿模型第四课《大模型前沿架构》","![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw01.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=JJ8mHzkKKeuJ83p9vQeyZy6tNjI%3D)\r\n\r\n2026 年 1 月 23 日，2025 冬季 InfiniTensor 大模型与人工智能系统训练营前沿模型系列第四课《大模型前沿架构》开讲。\r\n\r\n本节课深入解析大模型架构的最新进展，揭示稀疏激活、MoE 架构，再到极致量化技术，全面展现大模型效率优化的技术路径。\r\n\r\n## 通用智能时代的三大支柱\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw02.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=wLXm0YFw5AgTbcENqAciWo03a%2Bw%3D)\r\n\r\n### 架构统一\r\n\r\nTransformer 架构的出现实现了 AI 模型架构的统一：\r\n\r\n- 文本处理：将文本转换为 token 序列\r\n- 图像处理：将图像分割为 patch 序列\r\n- 图结构：对节点和连接关系进行编码\r\n\r\n这种统一架构使得单一模型能够处理多模态任务，奠定了通用智能的基础。\r\n\r\n### 学习方法统一\r\n\r\n通过\"预测下一个 token\"的自回归方式，各类 NLP 任务均可转化为序列生成问题：\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw03.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=MM%2FRn6%2FwRDLKF0tN3TeTlbpoXHU%3D)\r\n\r\n这种统一的学习范式使得模型能够进行多任务联合训练，获得通用能力。\r\n\r\n### 通用模型能力\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw04.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=FJK%2BxblCkxykAwgWq0rWdKzpM0k%3D)\r\n\r\n## 大模型结构高效\r\n\r\n### 人脑 vs 大模型\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw05.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ayjF12NxWtw%2BWlWPbqu0EdVapiE%3D)\r\n\r\n- 神经元规模：人脑约数百亿级别神经元，GPT-3 约 1750 亿参数，规模相当\r\n- 能耗对比：GPT-3 单次推理 400 瓦，人脑不到 15 瓦\r\n- 响应时间：GPT-3 约 2000 毫秒，人脑不到 100 毫秒\r\n- 关键差异：人脑神经元激活比例不到 5%，而大模型接近 100%\r\n\r\n### 稀疏激活现象\r\n\r\n研究发现大模型具有自发的稀疏激活特性：\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw06.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OGgHG8KrOfajj999zcsjqbp5x30%3D)\r\n\r\n### 混合专家（MoE）架构\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw07.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=XlBHO%2FEX4b0GHN3Ud7lNM537aH8%3D)\r\n\r\n#### 1. 核心思想\r\n\r\n将稠密前馈网络（FFN）转换为混合专家网络：\r\n\r\n- **专家划分**：将经常同时激活的神经元分组为专家\r\n- **路由机制**：根据输入选择最相关的专家进行计算\r\n- **稀疏激活**：每次只激活部分专家，大幅减少计算量\r\n\r\n#### 2. 性能表现\r\n\r\n- **3B 模型**：20% 的计算量可还原 98% 的性能\r\n- **模型规模效应**：更大的模型具有更明显的功能模块分区\r\n\r\n#### 3. MoE vs 稠密模型\r\n\r\n实验表明，MoE 架构能更好地刻画模型的模块化性质，功能神经元在专家中的比例更高。\r\n\r\n### 功能神经元\r\n\r\n#### 1. Skill Neuron（能力神经元）\r\n\r\n特定神经元对任务标签具有高度预测性：\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw08.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=WHzrX10IUjwe2mXFkYrYSwVV228%3D)\r\n\r\n#### 2. 语言神经元\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw09.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=zpQZnTUqthvZ0u%2F%2BMDT%2BLSqZvdQ%3D)\r\n\r\n#### 3. 情感神经元\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw10.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=mHrhhkHKHWXEMLgreJmEUKdyG3o%3D)\r\n\r\n### 稀疏激活函数\r\n\r\n#### 1. ReLU vs Swish\r\n\r\n- **ReLU**：将负数直接置零，产生稀疏激活\r\n- **Swish**：平滑处理所有输入值，保持参数吸收性但降低稀疏性\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw11.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=SLvedn421stwzqN3n7mahhvpOlM%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw12.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=psgTfWJkNh20%2BasOWymZooiWkvo%3D)\r\n\r\n## 大模型计算高效\r\n\r\n### 低位宽优化\r\n\r\n- 通过降低参数精度提升效率：\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw13.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=im2RmuCDf%2F73Y%2FNx8I7hMJ4OdzU%3D)\r\n\r\n- **效果**：减少带宽需求，提升训练\u002F推理速度\r\n\r\n### 标量与向量量化的统一\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw14.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OFF5ZRxGRh9F2NTu4zBBppce%2FAo%3D)\r\n\r\n### 稀疏优化\r\n\r\n#### 1. PowerInfer 混合推理\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw15.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=GmWsNgUy6%2Bng0HdPi%2B0S3vg7XMs%3D)\r\n\r\n#### 2. PowerInfer2 进阶优化\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw16.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qyUb2mrCTo46Zivb%2BWZAbci1qsc%3D)\r\n\r\n### 高效解码技术\r\n\r\n#### 1. 投机采样\r\n\r\n大小模型协同解码：\r\n\r\n- **草稿模型**：小模型为大模型打草稿\r\n- **并行验证**：大模型同时验证多个 token\r\n- **加速效果**：推理速度提升 1.6-2 倍\r\n\r\n#### 2. Medusa 解码\r\n\r\n- **轻量解码头**：可并行训练的预测头\r\n- **树状注意力**：同时验证多条候选路径\r\n- **加速效果**：约 2 倍推理加速\r\n\r\n#### 3. 高速树状投机（EAGLE）\r\n\r\n- **极简起草**：仅用一层全连接层起草\r\n- **特征复用**：利用倒数第二层特征消除随机性\r\n- **加速效果**：比 Medusa 再快 1-1.6 倍\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw17.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=WmBw8Yll%2BTz%2FLzTmCJBLKSKs7kY%3D)\r\n\r\n### 大词表优化\r\n\r\n针对 10 万+词表的起草瓶颈：\r\n\r\n- **长尾分布**：75% 的词出现频率总和小于 5%\r\n- **高频词起草**：仅预测前 32K 高频词\r\n- **加速效果**：A100 上额外带来 1.1-1.8 倍加速，3090 上达 1.2-5.2 倍\r\n\r\n### 注意力机制优化\r\n\r\n#### 1. 分块计算\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw18.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qg%2BoYScT7RHNeg9wlGOX2ceK5p0%3D)\r\n\r\n#### 2. 丢弃词元\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw19.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ljVk3zJpU04AXkE5A7D%2FyZQLa%2BQ%3D)\r\n\r\n#### 3. 算子优化\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw20.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ygT89FZuP3faF7hPeh5W8It3QiM%3D)\r\n\r\n## 混合专家模型与大模型稀疏化\r\n\r\n### 混合专家模型 MoE\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw21.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=FE96oMuHOoBxmOqweUKvpMKDFEc%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw22.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=exW80OG%2BhmUXPA3vqSeJTumLf6U%3D)\r\n\r\n### 核心挑战\r\n\r\n#### 1. Token 负载均衡\r\n\r\n- **问题**：Token 倾向于发送到少数受欢迎的专家\r\n- **解决方案**：\r\n  - 辅助损失：确保每个专家有相同重要性\r\n  - 随机路由：TOP2 中第二个专家按权重随机选择\r\n\r\n#### 2. 计算负载均衡\r\n\r\n- **问题**：不同专家处理的 token 数量不均，导致计算不平衡\r\n- **解决方案**：Megablocks 方法支持块稀疏矩阵乘法，灵活适应不均衡分配\r\n\r\n#### 3. 专家并行通信\r\n\r\n- **架构**：路由网络复制到每个计算单元，专家网络独立部署\r\n- **通信模式**：Token 智能路由到对应专家节点，结果返回原节点\r\n- **扩展性**：专家数量与 GPU 数量正相关\r\n\r\n### 创新方向\r\n\r\n#### ReMoE\r\n\r\n- **可微路由**：用 ReLU 激活替代 TOPK + Softmax\r\n- **优势**：更好的专家数量可扩展性\r\n\r\n#### Block FFN\r\n\r\n- **LoRA启发**：将 FFN 分解为类似 LoRA 的模块\r\n- **动态路由**：激活数量完全由稀疏激活函数决定\r\n- **优势**：兼顾稀疏动态性和结构性，访存模式更集中\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw23.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Ifdlz%2FXtJHy8TrTtU9VKjz7Jng0%3D)\r\n\r\n## 总结\r\n\r\n大模型架构正从单纯的规模扩张转向效率优先的精细化设计。通过**稀疏激活**、**MoE架构**、**量化技术**和**推理优化**的协同创新，业界正在构建更加高效、可持续的大模型生态系统。\r\n\r\n这些技术不仅解决了当前的计算和能耗瓶颈，更为未来更大规模、更复杂任务的 AI 系统奠定了基础。在通用智能的道路上，效率与能力的平衡将成为持续探索的重要方向。\r\n\r\n![2025冬季训练营课程表](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw6.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Qx4clxyNRHWc1WkZcCdvQcg69a4%3D)\r\n\r\n### 💡 学习贴士\r\n\r\n更多细节及完整课程内容，请观看直播或查看课程回放。\r\n\r\n链接：https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\r\n\r\n### 📚 训练营报名持续进行中\r\n\r\n扫码报名与加入群聊，或点击 **「阅读原文」** 直达官网报名\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw7.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=k66yp4XzAuO1%2FrXmGmXbKwbUKas%3D)",{"publishTime":7,"createTime":109,"canonicalPath":114,"id":115,"title":116},"\u002Fstudy-material\u002Fstudy-32","study-32","InfiniTensor 冬季训练营 OpenCL 编程第二课《OpenCL 编程抽象与语法》",76,"\u002Fstudy-material\u002Fstudy-33","study-33","markdown",{"publishTime":7,"createTime":109,"canonicalPath":122,"id":123,"title":124},"\u002Fstudy-material\u002Fstudy-34","study-34","InfiniTensor 冬季训练营 Triton & 九齿第三课《九齿二重：渐悟》",{"htmlContent":126,"headings":127},"\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw01.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=JJ8mHzkKKeuJ83p9vQeyZy6tNjI%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>2026 年 1 月 23 日，2025 冬季 InfiniTensor 大模型与人工智能系统训练营前沿模型系列第四课《大模型前沿架构》开讲。\u003C\u002Fp>\n\u003Cp>本节课深入解析大模型架构的最新进展，揭示稀疏激活、MoE 架构，再到极致量化技术，全面展现大模型效率优化的技术路径。\u003C\u002Fp>\n\u003Ch2 id=\"通用智能时代的三大支柱\">\u003Ca class=\"anchor\" href=\"#通用智能时代的三大支柱\" aria-label=\"跳转到标题：通用智能时代的三大支柱\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>通用智能时代的三大支柱\u003C\u002Fh2>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw02.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=wLXm0YFw5AgTbcENqAciWo03a%2Bw%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"架构统一\">\u003Ca class=\"anchor\" href=\"#架构统一\" aria-label=\"跳转到标题：架构统一\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>架构统一\u003C\u002Fh3>\u003Cp>Transformer 架构的出现实现了 AI 模型架构的统一：\u003C\u002Fp>\n\u003Cul>\n\u003Cli>文本处理：将文本转换为 token 序列\u003C\u002Fli>\n\u003Cli>图像处理：将图像分割为 patch 序列\u003C\u002Fli>\n\u003Cli>图结构：对节点和连接关系进行编码\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>这种统一架构使得单一模型能够处理多模态任务，奠定了通用智能的基础。\u003C\u002Fp>\n\u003Ch3 id=\"学习方法统一\">\u003Ca class=\"anchor\" href=\"#学习方法统一\" aria-label=\"跳转到标题：学习方法统一\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>学习方法统一\u003C\u002Fh3>\u003Cp>通过"预测下一个 token"的自回归方式，各类 NLP 任务均可转化为序列生成问题：\u003Cbr>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw03.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=MM%2FRn6%2FwRDLKF0tN3TeTlbpoXHU%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>这种统一的学习范式使得模型能够进行多任务联合训练，获得通用能力。\u003C\u002Fp>\n\u003Ch3 id=\"通用模型能力\">\u003Ca class=\"anchor\" href=\"#通用模型能力\" aria-label=\"跳转到标题：通用模型能力\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>通用模型能力\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw04.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=FJK%2BxblCkxykAwgWq0rWdKzpM0k%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"大模型结构高效\">\u003Ca class=\"anchor\" href=\"#大模型结构高效\" aria-label=\"跳转到标题：大模型结构高效\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>大模型结构高效\u003C\u002Fh2>\u003Ch3 id=\"人脑-vs-大模型\">\u003Ca class=\"anchor\" href=\"#人脑-vs-大模型\" aria-label=\"跳转到标题：人脑 vs 大模型\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>人脑 vs 大模型\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw05.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ayjF12NxWtw%2BWlWPbqu0EdVapiE%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>神经元规模：人脑约数百亿级别神经元，GPT-3 约 1750 亿参数，规模相当\u003C\u002Fli>\n\u003Cli>能耗对比：GPT-3 单次推理 400 瓦，人脑不到 15 瓦\u003C\u002Fli>\n\u003Cli>响应时间：GPT-3 约 2000 毫秒，人脑不到 100 毫秒\u003C\u002Fli>\n\u003Cli>关键差异：人脑神经元激活比例不到 5%，而大模型接近 100%\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"稀疏激活现象\">\u003Ca class=\"anchor\" href=\"#稀疏激活现象\" aria-label=\"跳转到标题：稀疏激活现象\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>稀疏激活现象\u003C\u002Fh3>\u003Cp>研究发现大模型具有自发的稀疏激活特性：\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw06.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OGgHG8KrOfajj999zcsjqbp5x30%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"混合专家-moe-架构\">\u003Ca class=\"anchor\" href=\"#混合专家-moe-架构\" aria-label=\"跳转到标题：混合专家（MoE）架构\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>混合专家（MoE）架构\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw07.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=XlBHO%2FEX4b0GHN3Ud7lNM537aH8%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"1-核心思想\">\u003Ca class=\"anchor\" href=\"#1-核心思想\" aria-label=\"跳转到标题：1. 核心思想\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 核心思想\u003C\u002Fh4>\u003Cp>将稠密前馈网络（FFN）转换为混合专家网络：\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>专家划分\u003C\u002Fstrong>：将经常同时激活的神经元分组为专家\u003C\u002Fli>\n\u003Cli>\u003Cstrong>路由机制\u003C\u002Fstrong>：根据输入选择最相关的专家进行计算\u003C\u002Fli>\n\u003Cli>\u003Cstrong>稀疏激活\u003C\u002Fstrong>：每次只激活部分专家，大幅减少计算量\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-性能表现\">\u003Ca class=\"anchor\" href=\"#2-性能表现\" aria-label=\"跳转到标题：2. 性能表现\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 性能表现\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>3B 模型\u003C\u002Fstrong>：20% 的计算量可还原 98% 的性能\u003C\u002Fli>\n\u003Cli>\u003Cstrong>模型规模效应\u003C\u002Fstrong>：更大的模型具有更明显的功能模块分区\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"3-moe-vs-稠密模型\">\u003Ca class=\"anchor\" href=\"#3-moe-vs-稠密模型\" aria-label=\"跳转到标题：3. MoE vs 稠密模型\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. MoE vs 稠密模型\u003C\u002Fh4>\u003Cp>实验表明，MoE 架构能更好地刻画模型的模块化性质，功能神经元在专家中的比例更高。\u003C\u002Fp>\n\u003Ch3 id=\"功能神经元\">\u003Ca class=\"anchor\" href=\"#功能神经元\" aria-label=\"跳转到标题：功能神经元\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>功能神经元\u003C\u002Fh3>\u003Ch4 id=\"1-skill-neuron-能力神经元\">\u003Ca class=\"anchor\" href=\"#1-skill-neuron-能力神经元\" aria-label=\"跳转到标题：1. Skill Neuron（能力神经元）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. Skill Neuron（能力神经元）\u003C\u002Fh4>\u003Cp>特定神经元对任务标签具有高度预测性：\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw08.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=WHzrX10IUjwe2mXFkYrYSwVV228%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"2-语言神经元\">\u003Ca class=\"anchor\" href=\"#2-语言神经元\" aria-label=\"跳转到标题：2. 语言神经元\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 语言神经元\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw09.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=zpQZnTUqthvZ0u%2F%2BMDT%2BLSqZvdQ%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"3-情感神经元\">\u003Ca class=\"anchor\" href=\"#3-情感神经元\" aria-label=\"跳转到标题：3. 情感神经元\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 情感神经元\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw10.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=mHrhhkHKHWXEMLgreJmEUKdyG3o%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"稀疏激活函数\">\u003Ca class=\"anchor\" href=\"#稀疏激活函数\" aria-label=\"跳转到标题：稀疏激活函数\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>稀疏激活函数\u003C\u002Fh3>\u003Ch4 id=\"1-relu-vs-swish\">\u003Ca class=\"anchor\" href=\"#1-relu-vs-swish\" aria-label=\"跳转到标题：1. ReLU vs Swish\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. ReLU vs Swish\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>ReLU\u003C\u002Fstrong>：将负数直接置零，产生稀疏激活\u003C\u002Fli>\n\u003Cli>\u003Cstrong>Swish\u003C\u002Fstrong>：平滑处理所有输入值，保持参数吸收性但降低稀疏性\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw11.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=SLvedn421stwzqN3n7mahhvpOlM%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw12.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=psgTfWJkNh20%2BasOWymZooiWkvo%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"大模型计算高效\">\u003Ca class=\"anchor\" href=\"#大模型计算高效\" aria-label=\"跳转到标题：大模型计算高效\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>大模型计算高效\u003C\u002Fh2>\u003Ch3 id=\"低位宽优化\">\u003Ca class=\"anchor\" href=\"#低位宽优化\" aria-label=\"跳转到标题：低位宽优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>低位宽优化\u003C\u002Fh3>\u003Cul>\n\u003Cli>通过降低参数精度提升效率：\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw13.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=im2RmuCDf%2F73Y%2FNx8I7hMJ4OdzU%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>效果\u003C\u002Fstrong>：减少带宽需求，提升训练\u002F推理速度\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"标量与向量量化的统一\">\u003Ca class=\"anchor\" href=\"#标量与向量量化的统一\" aria-label=\"跳转到标题：标量与向量量化的统一\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>标量与向量量化的统一\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw14.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OFF5ZRxGRh9F2NTu4zBBppce%2FAo%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"稀疏优化\">\u003Ca class=\"anchor\" href=\"#稀疏优化\" aria-label=\"跳转到标题：稀疏优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>稀疏优化\u003C\u002Fh3>\u003Ch4 id=\"1-powerinfer-混合推理\">\u003Ca class=\"anchor\" href=\"#1-powerinfer-混合推理\" aria-label=\"跳转到标题：1. PowerInfer 混合推理\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. PowerInfer 混合推理\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw15.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=GmWsNgUy6%2Bng0HdPi%2B0S3vg7XMs%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"2-powerinfer2-进阶优化\">\u003Ca class=\"anchor\" href=\"#2-powerinfer2-进阶优化\" aria-label=\"跳转到标题：2. PowerInfer2 进阶优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. PowerInfer2 进阶优化\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw16.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qyUb2mrCTo46Zivb%2BWZAbci1qsc%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"高效解码技术\">\u003Ca class=\"anchor\" href=\"#高效解码技术\" aria-label=\"跳转到标题：高效解码技术\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>高效解码技术\u003C\u002Fh3>\u003Ch4 id=\"1-投机采样\">\u003Ca class=\"anchor\" href=\"#1-投机采样\" aria-label=\"跳转到标题：1. 投机采样\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 投机采样\u003C\u002Fh4>\u003Cp>大小模型协同解码：\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>草稿模型\u003C\u002Fstrong>：小模型为大模型打草稿\u003C\u002Fli>\n\u003Cli>\u003Cstrong>并行验证\u003C\u002Fstrong>：大模型同时验证多个 token\u003C\u002Fli>\n\u003Cli>\u003Cstrong>加速效果\u003C\u002Fstrong>：推理速度提升 1.6-2 倍\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-medusa-解码\">\u003Ca class=\"anchor\" href=\"#2-medusa-解码\" aria-label=\"跳转到标题：2. Medusa 解码\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. Medusa 解码\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>轻量解码头\u003C\u002Fstrong>：可并行训练的预测头\u003C\u002Fli>\n\u003Cli>\u003Cstrong>树状注意力\u003C\u002Fstrong>：同时验证多条候选路径\u003C\u002Fli>\n\u003Cli>\u003Cstrong>加速效果\u003C\u002Fstrong>：约 2 倍推理加速\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"3-高速树状投机-eagle\">\u003Ca class=\"anchor\" href=\"#3-高速树状投机-eagle\" aria-label=\"跳转到标题：3. 高速树状投机（EAGLE）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 高速树状投机（EAGLE）\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>极简起草\u003C\u002Fstrong>：仅用一层全连接层起草\u003C\u002Fli>\n\u003Cli>\u003Cstrong>特征复用\u003C\u002Fstrong>：利用倒数第二层特征消除随机性\u003C\u002Fli>\n\u003Cli>\u003Cstrong>加速效果\u003C\u002Fstrong>：比 Medusa 再快 1-1.6 倍\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw17.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=WmBw8Yll%2BTz%2FLzTmCJBLKSKs7kY%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"大词表优化\">\u003Ca class=\"anchor\" href=\"#大词表优化\" aria-label=\"跳转到标题：大词表优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>大词表优化\u003C\u002Fh3>\u003Cp>针对 10 万+词表的起草瓶颈：\u003C\u002Fp>\n\u003Cul>\n\u003Cli>\u003Cstrong>长尾分布\u003C\u002Fstrong>：75% 的词出现频率总和小于 5%\u003C\u002Fli>\n\u003Cli>\u003Cstrong>高频词起草\u003C\u002Fstrong>：仅预测前 32K 高频词\u003C\u002Fli>\n\u003Cli>\u003Cstrong>加速效果\u003C\u002Fstrong>：A100 上额外带来 1.1-1.8 倍加速，3090 上达 1.2-5.2 倍\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"注意力机制优化\">\u003Ca class=\"anchor\" href=\"#注意力机制优化\" aria-label=\"跳转到标题：注意力机制优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>注意力机制优化\u003C\u002Fh3>\u003Ch4 id=\"1-分块计算\">\u003Ca class=\"anchor\" href=\"#1-分块计算\" aria-label=\"跳转到标题：1. 分块计算\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 分块计算\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw18.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=qg%2BoYScT7RHNeg9wlGOX2ceK5p0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"2-丢弃词元\">\u003Ca class=\"anchor\" href=\"#2-丢弃词元\" aria-label=\"跳转到标题：2. 丢弃词元\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 丢弃词元\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw19.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ljVk3zJpU04AXkE5A7D%2FyZQLa%2BQ%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"3-算子优化\">\u003Ca class=\"anchor\" href=\"#3-算子优化\" aria-label=\"跳转到标题：3. 算子优化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 算子优化\u003C\u002Fh4>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw20.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=ygT89FZuP3faF7hPeh5W8It3QiM%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"混合专家模型与大模型稀疏化\">\u003Ca class=\"anchor\" href=\"#混合专家模型与大模型稀疏化\" aria-label=\"跳转到标题：混合专家模型与大模型稀疏化\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>混合专家模型与大模型稀疏化\u003C\u002Fh2>\u003Ch3 id=\"混合专家模型-moe\">\u003Ca class=\"anchor\" href=\"#混合专家模型-moe\" aria-label=\"跳转到标题：混合专家模型 MoE\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>混合专家模型 MoE\u003C\u002Fh3>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw21.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=FE96oMuHOoBxmOqweUKvpMKDFEc%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw22.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=exW80OG%2BhmUXPA3vqSeJTumLf6U%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"核心挑战\">\u003Ca class=\"anchor\" href=\"#核心挑战\" aria-label=\"跳转到标题：核心挑战\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>核心挑战\u003C\u002Fh3>\u003Ch4 id=\"1-token-负载均衡\">\u003Ca class=\"anchor\" href=\"#1-token-负载均衡\" aria-label=\"跳转到标题：1. Token 负载均衡\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. Token 负载均衡\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>问题\u003C\u002Fstrong>：Token 倾向于发送到少数受欢迎的专家\u003C\u002Fli>\n\u003Cli>\u003Cstrong>解决方案\u003C\u002Fstrong>：\u003Cul>\n\u003Cli>辅助损失：确保每个专家有相同重要性\u003C\u002Fli>\n\u003Cli>随机路由：TOP2 中第二个专家按权重随机选择\u003C\u002Fli>\n\u003C\u002Ful>\n\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-计算负载均衡\">\u003Ca class=\"anchor\" href=\"#2-计算负载均衡\" aria-label=\"跳转到标题：2. 计算负载均衡\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 计算负载均衡\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>问题\u003C\u002Fstrong>：不同专家处理的 token 数量不均，导致计算不平衡\u003C\u002Fli>\n\u003Cli>\u003Cstrong>解决方案\u003C\u002Fstrong>：Megablocks 方法支持块稀疏矩阵乘法，灵活适应不均衡分配\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"3-专家并行通信\">\u003Ca class=\"anchor\" href=\"#3-专家并行通信\" aria-label=\"跳转到标题：3. 专家并行通信\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 专家并行通信\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>架构\u003C\u002Fstrong>：路由网络复制到每个计算单元，专家网络独立部署\u003C\u002Fli>\n\u003Cli>\u003Cstrong>通信模式\u003C\u002Fstrong>：Token 智能路由到对应专家节点，结果返回原节点\u003C\u002Fli>\n\u003Cli>\u003Cstrong>扩展性\u003C\u002Fstrong>：专家数量与 GPU 数量正相关\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch3 id=\"创新方向\">\u003Ca class=\"anchor\" href=\"#创新方向\" aria-label=\"跳转到标题：创新方向\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>创新方向\u003C\u002Fh3>\u003Ch4 id=\"remoe\">\u003Ca class=\"anchor\" href=\"#remoe\" aria-label=\"跳转到标题：ReMoE\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>ReMoE\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>可微路由\u003C\u002Fstrong>：用 ReLU 激活替代 TOPK + Softmax\u003C\u002Fli>\n\u003Cli>\u003Cstrong>优势\u003C\u002Fstrong>：更好的专家数量可扩展性\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"block-ffn\">\u003Ca class=\"anchor\" href=\"#block-ffn\" aria-label=\"跳转到标题：Block FFN\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>Block FFN\u003C\u002Fh4>\u003Cul>\n\u003Cli>\u003Cstrong>LoRA启发\u003C\u002Fstrong>：将 FFN 分解为类似 LoRA 的模块\u003C\u002Fli>\n\u003Cli>\u003Cstrong>动态路由\u003C\u002Fstrong>：激活数量完全由稀疏激活函数决定\u003C\u002Fli>\n\u003Cli>\u003Cstrong>优势\u003C\u002Fstrong>：兼顾稀疏动态性和结构性，访存模式更集中\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_33_zw23.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Ifdlz%2FXtJHy8TrTtU9VKjz7Jng0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"总结\">\u003Ca class=\"anchor\" href=\"#总结\" aria-label=\"跳转到标题：总结\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>总结\u003C\u002Fh2>\u003Cp>大模型架构正从单纯的规模扩张转向效率优先的精细化设计。通过\u003Cstrong>稀疏激活\u003C\u002Fstrong>、\u003Cstrong>MoE架构\u003C\u002Fstrong>、\u003Cstrong>量化技术\u003C\u002Fstrong>和\u003Cstrong>推理优化\u003C\u002Fstrong>的协同创新，业界正在构建更加高效、可持续的大模型生态系统。\u003C\u002Fp>\n\u003Cp>这些技术不仅解决了当前的计算和能耗瓶颈，更为未来更大规模、更复杂任务的 AI 系统奠定了基础。在通用智能的道路上，效率与能力的平衡将成为持续探索的重要方向。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw6.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Qx4clxyNRHWc1WkZcCdvQcg69a4%3D\" alt=\"2025冬季训练营课程表\">\u003C\u002Fp>\n\u003Ch3 id=\"学习贴士\">\u003Ca class=\"anchor\" href=\"#学习贴士\" aria-label=\"跳转到标题：💡 学习贴士\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>💡 学习贴士\u003C\u002Fh3>\u003Cp>更多细节及完整课程内容，请观看直播或查看课程回放。\u003C\u002Fp>\n\u003Cp>链接：\u003Ca href=\"https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\">https:\u002F\u002Fwww.infinitensor.com\u002Fcamp\u002Fwinter2025\u003C\u002Fa>\u003C\u002Fp>\n\u003Ch3 id=\"训练营报名持续进行中\">\u003Ca class=\"anchor\" href=\"#训练营报名持续进行中\" aria-label=\"跳转到标题：📚 训练营报名持续进行中\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>📚 训练营报名持续进行中\u003C\u002Fh3>\u003Cp>扫码报名与加入群聊，或点击 \u003Cstrong>「阅读原文」\u003C\u002Fstrong> 直达官网报名\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_01_zw7.jpg?Expires=1792752589&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=k66yp4XzAuO1%2FrXmGmXbKwbUKas%3D\" alt=\"\">\u003C\u002Fp>\n",[128,130,132,134,136,138,141,143,146,150,153,156,158,161,164,167,169,172,174,176,178,180,183,186,188,191,194,197,199,201,204,207,210,212,215,217,220,223,226,228,231,234,236,239],{"id":129,"text":129,"level":110},"通用智能时代的三大支柱",{"id":131,"text":131,"level":9},"架构统一",{"id":133,"text":133,"level":9},"学习方法统一",{"id":135,"text":135,"level":9},"通用模型能力",{"id":137,"text":137,"level":110},"大模型结构高效",{"id":139,"text":140,"level":9},"人脑-vs-大模型","人脑 vs 大模型",{"id":142,"text":142,"level":9},"稀疏激活现象",{"id":144,"text":145,"level":9},"混合专家-moe-架构","混合专家（MoE）架构",{"id":147,"text":148,"level":149},"1-核心思想","1. 核心思想",4,{"id":151,"text":152,"level":149},"2-性能表现","2. 性能表现",{"id":154,"text":155,"level":149},"3-moe-vs-稠密模型","3. MoE vs 稠密模型",{"id":157,"text":157,"level":9},"功能神经元",{"id":159,"text":160,"level":149},"1-skill-neuron-能力神经元","1. Skill Neuron（能力神经元）",{"id":162,"text":163,"level":149},"2-语言神经元","2. 语言神经元",{"id":165,"text":166,"level":149},"3-情感神经元","3. 情感神经元",{"id":168,"text":168,"level":9},"稀疏激活函数",{"id":170,"text":171,"level":149},"1-relu-vs-swish","1. ReLU vs Swish",{"id":173,"text":173,"level":110},"大模型计算高效",{"id":175,"text":175,"level":9},"低位宽优化",{"id":177,"text":177,"level":9},"标量与向量量化的统一",{"id":179,"text":179,"level":9},"稀疏优化",{"id":181,"text":182,"level":149},"1-powerinfer-混合推理","1. PowerInfer 混合推理",{"id":184,"text":185,"level":149},"2-powerinfer2-进阶优化","2. PowerInfer2 进阶优化",{"id":187,"text":187,"level":9},"高效解码技术",{"id":189,"text":190,"level":149},"1-投机采样","1. 投机采样",{"id":192,"text":193,"level":149},"2-medusa-解码","2. Medusa 解码",{"id":195,"text":196,"level":149},"3-高速树状投机-eagle","3. 高速树状投机（EAGLE）",{"id":198,"text":198,"level":9},"大词表优化",{"id":200,"text":200,"level":9},"注意力机制优化",{"id":202,"text":203,"level":149},"1-分块计算","1. 分块计算",{"id":205,"text":206,"level":149},"2-丢弃词元","2. 丢弃词元",{"id":208,"text":209,"level":149},"3-算子优化","3. 算子优化",{"id":211,"text":211,"level":110},"混合专家模型与大模型稀疏化",{"id":213,"text":214,"level":9},"混合专家模型-moe","混合专家模型 MoE",{"id":216,"text":216,"level":9},"核心挑战",{"id":218,"text":219,"level":149},"1-token-负载均衡","1. Token 负载均衡",{"id":221,"text":222,"level":149},"2-计算负载均衡","2. 计算负载均衡",{"id":224,"text":225,"level":149},"3-专家并行通信","3. 专家并行通信",{"id":227,"text":227,"level":9},"创新方向",{"id":229,"text":230,"level":149},"remoe","ReMoE",{"id":232,"text":233,"level":149},"block-ffn","Block FFN",{"id":235,"text":235,"level":110},"总结",{"id":237,"text":238,"level":9},"学习贴士","💡 学习贴士",{"id":240,"text":241,"level":9},"训练营报名持续进行中","📚 训练营报名持续进行中"]