[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"site-home-live-entry-setting":3,"site-header-contests":8,"site-header-camps":49,"study-material-detail-study-95":104},{"enabled":4,"title":5,"statusText":5,"buttonText":5,"campSlug":5,"stage":6,"classroomId":7,"hasClassroom":4},false,"",0,null,{"total":9,"list":10,"pageNum":44,"pageSize":45,"size":9,"startRow":44,"endRow":9,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":48,"navigateFirstPage":44,"navigateLastPage":44},3,[11,25,35],{"slug":12,"title":13,"logo":7,"liveUrl":14,"prize":15,"sponsor":16,"description":17,"organizer":18,"startTime":19,"endTime":20,"registrationDeadline":21,"status":22,"dashboardVisibility":23,"createdAt":24},"spring2026","2026春季启元人工智能大赛","https:\u002F\u002Fwww.bilibili.com\u002Fblackboard\u002Flive\u002Flive-activity-player.html?cid=1822703816&quality=0","200,000","麒麟软件有限公司","“2026春季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。\n各参赛选手需在规定的比赛周期内，依据赛事规则选定赛题并完成相应挑战，优胜者将共同角逐总额20万元的赛事奖金池，并且还将有机会获得实习和知名企业优质岗位内推资格。","启元实验室","2026-04-19 16:00:00","2026-07-13 16:00:00","2026-06-16 16:00:00","ongoing","private","2026-04-16 08:00:00",{"slug":26,"title":27,"logo":7,"liveUrl":14,"prize":28,"sponsor":7,"description":29,"organizer":18,"startTime":30,"endTime":31,"registrationDeadline":32,"status":33,"dashboardVisibility":23,"createdAt":34},"autumn2025","2025秋季启元人工智能大赛","150,000","“2025秋季启元人工智能大赛”是由启元实验室主办的面向全国人工智能领域开发者、科研人员及高校学生的权威赛事。大赛旨在促进与培养具备工程实践能力和创新能力的优秀人工智能领域开发者、推广国产软硬件技术与生态、助力国家人工智能产业高质量发展。","2025-11-08 08:00:00","2026-01-13 08:00:00","2025-12-30 08:00:00","ended","2025-11-03 08:00:00",{"slug":36,"title":37,"logo":7,"liveUrl":14,"prize":38,"sponsor":7,"description":39,"organizer":18,"startTime":40,"endTime":41,"registrationDeadline":42,"status":33,"dashboardVisibility":23,"createdAt":43},"summer2025","启元人工智能大赛(系统方向)","80,000","启元人工智能大赛由启元实验室主办，其中系统赛道基于启元实验室九源软件栈的相关开源项目，通过拉动开源社区力量，一方面促进国产智能计算生态的发展，另一方面也为人工智能系统方向的学生及从业者提供一个实践平台。","2025-07-08 08:00:00","2025-08-25 08:00:00","2025-07-22 08:00:00","2025-06-22 08:00:00",1,5,true,8,[44],{"total":45,"list":50,"pageNum":44,"pageSize":45,"size":45,"startRow":44,"endRow":45,"pages":44,"prePage":6,"nextPage":6,"isFirstPage":46,"isLastPage":46,"hasPreviousPage":4,"hasNextPage":4,"navigatePages":47,"navigatepageNums":103,"navigateFirstPage":44,"navigateLastPage":44},[51,62,73,84,94],{"id":52,"slug":53,"title":54,"logo":55,"publicLiveEmbedUrl":14,"description":56,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":16,"startTime":58,"endTime":59,"status":22,"dashboardVisibility":60,"createdAt":58,"updatedAt":61},"019eeeff-6b00-7d55-a629-efafded3e232","summer2026","2026年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2026-06\u002F019eef7f-7816-7828-9309-79c2c61dd568.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Loo9mNcAiqrEJeoxKJGChNjsIP0%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。","...","2026-06-22 08:00:00","2026-10-19 08:00:00","public","2026-10-12 08:00:00",{"id":63,"slug":64,"title":65,"logo":66,"publicLiveEmbedUrl":7,"description":67,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":68,"startTime":69,"endTime":70,"status":33,"dashboardVisibility":23,"createdAt":71,"updatedAt":72},"019af3f3-e06a-7245-aa8d-37dc99a69c73","winter2025","2025年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-12\u002F019afc69-b7ca-7805-91fb-605429671c01.png?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=aQ7DEIPmeZ6G2GUnnO42fZDzkoY%3D","由启元实验室发起的InfiniTensor大模型与人工智能系统训练营，面向开源社区、高校及产业界伙伴开放。活动以「普及AI全栈知识技术、培养工程实践与创新型人才、推广国产软硬件技术生态」为核心目标，助力国家人工智能产业高质量发展。 \n","新华三技术有限公司","2025-12-25 08:00:00","2026-03-27 08:00:00","2025-12-06 08:00:00","2026-10-22 20:24:30",{"id":74,"slug":36,"title":75,"logo":76,"publicLiveEmbedUrl":7,"description":77,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":78,"startTime":79,"endTime":80,"status":81,"dashboardVisibility":23,"createdAt":82,"updatedAt":83},"01980342-208c-7803-b3f3-19e1d31960ba","2025年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fuploads\u002F2025-07\u002F0198045e-28fe-78f0-95fe-23835ce58395.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=rRUH7HCePjJ0x1Z1esoWYs0tjmE%3D","InfiniTensor大模型与人工智能系统训练营由启元实验室及清华大学 AI 系统方向团队发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营联合开源项目开发者、学术界大牛、工业界资深工程师参与授课与项目指导。课程采用「线上直播授课 + 在线答疑 + 项目实践辅导 」的方式进行，确保每位同学都能获得充分成长。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","北京积算科技有限公司","2025-07-14 08:00:00","2025-09-30 08:00:00","upcoming","2025-07-13 08:00:00","2026-10-22 20:24:58",{"id":85,"slug":86,"title":87,"logo":88,"publicLiveEmbedUrl":7,"description":89,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":90,"endTime":91,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":93},"01980cf9-a7d8-7796-ae86-0bbe05334add","winter2024","2024年冬季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-ce23-7624-a287-9f07dd0d85f6.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区、高校和产业界，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过视频、直播课、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。报名学员请阅读“导学”课程页的训练营介绍。","2025-01-05 08:00:00","2025-03-22 08:00:00","2025-07-15 08:00:00","2026-10-22 20:24:39",{"id":95,"slug":96,"title":97,"logo":98,"publicLiveEmbedUrl":7,"description":99,"introduction":57,"organizer":18,"coOrganizer":5,"sponsor":5,"startTime":100,"endTime":101,"status":81,"dashboardVisibility":23,"createdAt":92,"updatedAt":102},"01980c20-363a-7dc3-8f5a-58a68e08dcc4","summer2024","2024年夏季InfiniTensor大模型与人工智能系统训练营","https:\u002F\u002Ftesting-cdn.infinitensor.com\u002Fuploads\u002F2025-07\u002F01981269-fbfa-788a-afcd-17fa9be74ff9.jpg","InfiniTensor大模型与人工智能系统训练营由清华大学发起，面向开源社区和高校，普及人工智能各方面知识和技术，培养相关人才。训练营全程免费，通过课程、习题和项目实践，在人工智能飞速发展的背景下，为学员提供一窥深层技术的机会。","2024-07-20 08:00:00","2024-09-29 08:00:00","2026-10-22 20:24:49",[44],{"currentItem":105,"processedData":125},{"img":106,"keywords":107,"treeType":44,"description":108,"showStatus":44,"updateTime":109,"type":110,"title":111,"content":112,"nextArticle":113,"createTime":109,"sortOrder":117,"canonicalPath":118,"id":119,"contentType":120,"prevArticle":121},"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_fm.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=QKmBfHyOTzhQh3Y3hQZOI0N8pZE%3D","大模型服务,KV Cache,Paged Attention,连续批处理,流式推理","讲解大模型线上服务完整架构，涵盖 OpenAI 流式 API、Paged Attention 分页 KV Cache、连续批次调度，拆解三层服务模块与并发优化方案。","2026-10-22 20:38:11",2,"2026 夏季 InfiniTensor 训练营推理引擎 5：《大模型服务系统》","![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw01.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=9Q2JAKNB3dkNsEG3uBsT5hj1Ieo%3D)\r\n\r\n\u003Cspan style=\"font-size:12px;color:#555;\">8 月 6 日，2026 夏季 InfiniTensor 训练营推理引擎方向第五课《大模型服务系统》开讲。\r\n上节课聚焦推理底层算子与分布式并行，解决单卡显存不足、单 Token 推理延迟高问题。本节课转向线上生产服务完整工程链路，覆盖对话业务功能、流式接口、KV Cache 内存池、多请求调度整套落地方案。\u003C\u002Fspan>\r\n\r\n本节课从线上服务落地视角，讲解对话前端、OpenAI 标准接口、KV Cache 内存管理、批量调度、连续批次等完整服务架构。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw02.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kBV%2BmbrbqQUMJQsoQGU0pIqHzNk%3D)\r\n\r\n## 大模型对话产品业务功能梳理\r\n### 基础交互能力\r\n1. 逐 Token 流式输出，实时展示生成内容；\r\n2. 推理中断：支持随时终止当前回答；\r\n3. 多轮上下文记忆，支持历史对话编辑、回溯分支；\r\n4. 多会话隔离：新建、切换、删除不同话题会话。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw03.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=uiexvauW%2FMrLXOBPUmNKtNS7VKU%3D)\r\n\r\n### 底层推理循环基础逻辑\r\n单次推理循环流程：输入上下文 → Prefill 填充 KV Cache → 循环 Decode 生成 Token → 触发终止符\u002F长度上限\u002F用户断连后退出推理。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw04.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=aw2M%2B20m4cRrwtFV2XzHRRQhkjM%3D)\r\n\r\n## 推理服务标准接口：OpenAI API 规范\r\n### 同步非流式接口\r\n请求路径：`\u002Fv1\u002Fchat\u002Fcompletions`\r\n- 请求字段：model、messages（system\u002Fuser\u002Fassistant 角色数组）、超参数；\r\n- 返回：完整一次性对话结果，`finish_reason` 标记停止原因（stop \u002F length）。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw05.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=3W5DYPCHA9m5XdRBgSraeHWYgu4%3D)\r\n\r\n### 流式推理服务\r\n请求增加 `stream: true` 标识：\r\n1. 按生成过程持续返回增量内容；\r\n2. 用户断连后后端自动终止推理，节省算力；\r\n3. 解决长等待无反馈、中途放弃浪费资源问题。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw06.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=V0dAqkaC1v5wW4kqKU3bkckAKAQ%3D)\r\n\r\n### 服务端 Handler 实现\r\n基于 FastAPI 等 Web 框架搭建 POST 接口，解析 JSON 请求，转发至底层推理引擎，同步\u002F流式封装返回数据包。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw07.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=10zqesQtyzEi4hG7blFYryFj7Jw%3D)\r\n\r\n## 多会话核心：KV Cache 内存管理机制\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw08.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NkwOdbEc4M8dfsPCm4%2FFiStglk4%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw09.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=32YSnsCGLCc%2FECznt8PV4Nt4omA%3D)\r\n\r\n### 方案一：会话绑定完整 KV Cache\r\n每个会话独立分配一整块最大长度 KV Cache；\r\n- 优点：上下文前缀 100% 复用，无需重算；\r\n- 缺点：会话数量受显存严格限制，分支场景内存膨胀严重。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw10.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=%2FL5GKzbYp9RF18wz59rRrG6vGhM%3D)\r\n\r\n### 方案二：KV Cache 全局池 + 前缀匹配\r\n共用固定 Cache 内存池，新请求匹配最长历史前缀复用缓存块；\r\n- 优点：相比固定 Cache 分配方式，可以显著提高显存利用率，支持更多并发会话;\r\n- 缺点：无匹配时覆盖旧缓存，部分场景需重复计算。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw11.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=9VlEif85FF540zWRsFTQSFbOgBA%3D)\r\n\r\n### 分页式 Paged Attention（工业主流方案）\r\n#### 1. 核心思想\r\n借鉴操作系统虚拟内存分页，不分配整块 Cache，拆分为固定大小 Page 小块，通过映射表拼接逻辑上下文。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw12.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=15OWKZGG6v0Ynxe9cL1A6Jy4s00%3D)\r\n\r\n#### 2. 复用与分支逻辑\r\n多会话共享相同前缀 Page，采用引用计数管理：\r\n1. 共享块引用计数 +1；\r\n2. 当多个会话共享同一 Page，后续请求需要修改该 Page 内容时，通过 Copy-on-Write 创建新的 Page，避免影响其他会话。\r\n3. 会话销毁时引用计数 -1，计数为 0 回收 Page 内存。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw13.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Cb7MiWW2LQGLbrhzh3U3FCZDTxc%3D)\r\n\r\n#### 3. 内存扩容兜底策略\r\nPage 池耗尽时，可将闲置 Page Offload 至 CPU \u002F NVME 磁盘释放 GPU 显存。\r\n\r\n#### 4. 显存占用现状\r\n7 B \u002F 13 B 模型推理场景，KV Cache 通常占用超过 30% 单卡显存，分页机制大幅提升并发会话上限。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw14.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OQVNByvODPbSLEw4DYVggaNfje0%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw15.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=1d8bfIUhHeV4yJVRV%2FS%2FJ%2Fz54g8%3D)\r\n\r\n## 多请求批量调度方案演进\r\n### 方案1：串行单请求处理\r\n请求排队依次推理；\r\n- 缺陷：GPU 算力空置，吞吐极低，无法支撑线上并发。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw16.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=tK%2B9ZRwIRXxTZgJN1ynXKwnDDrM%3D)\r\n\r\n### 方案2：固定批次 Padding 批处理\r\n同一轮合并多条请求，短请求补零对齐张量；\r\n- 缺陷：短请求等待长请求，Padding 带来无效算力开销；Prefill \u002F Decode 请求难以进行高效混合调度。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw17.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=oHNkfrq8lsyhy6Z5G5jGkNxhpr8%3D)\r\n\r\n### 方案3：逐轮动态重批（基础优化）\r\n每轮 Decode 结束后重新从请求池组合 Batch，完成请求直接退出；\r\n- 改善短请求等待问题，但仍存在大量 Padding 无效计算。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw18.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=2wZURCL2vCSTRk7h03kCac4yOXk%3D)\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw19.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=hTPJpinjAF3VHr7zsusVOoxi7Hg%3D)\r\n\r\n### 方案4：连续批次 Continuous Batching（工业最优）\r\n#### 1.分层计算拆分\r\n- 无依赖算子（Linear \u002F RMSNorm \u002F RoPE）：可通过连续批处理方式减少 Padding 开销.\r\n- Attention 层根据各请求的 block table 访问对应 KV Cache，通过批量化 Kernel 完成不同请求 Attention 计算。\r\n\r\n#### 2.整体收益\r\n消除无效 Padding，GPU 硬件利用率大幅提升；支持 Prefill、Decode 请求混合调度，适配线上随机流量。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw20.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=YvUMEnz%2BkWnmEALXvLhiiQcRBRE%3D)\r\n\r\n## 标准推理服务三层分离架构\r\n### 前端接入模块\r\nWeb 接口服务，接收 HTTP 请求、解析 OpenAI JSON、流式数据包推送。\r\n\r\n### 调度管理模块\r\n1. 全局请求池维护；\r\n2. 每轮构造连续批次；\r\n3. Paged Attention KV Cache 块分配、引用计数、内存回收；\r\n4. 绑定 IO 管线，管理每路用户流式输出。\r\n\r\n### 底层推理引擎模块\r\n执行 Prefill \u002F Decode 推理循环，提供算子、分布式推理底层能力。\r\n\r\n### 模块隔离优势\r\n三模块通过异步队列进行解耦，使请求接收、调度管理和模型计算能够尽可能并行执行，提高系统吞吐。\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw21.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kWtyUYjtHMCfhzUrsjlQ4347dEg%3D)\r\n\r\n\r\n\r\n\r\n## 加入训练营\r\n📺**完整课程内容，请观看直播或查看课程回放：**\r\n**InfiniTensor 官网**： https:\u002F\u002Fwww.infinitensor.com\r\n\r\n**B站直播**： InfiniTensor 官方直播间\r\n\r\n**视频号直播**： InfiniTensor 视频号预约\r\n\r\n**答疑交流**： 训练营官方社群⬇️\r\n\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002Fqun_1072.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=GIJwh433B3Yl1X2vYlwRpO%2B9Mps%3D)\r\n\r\n## 关注与交流\r\n![](https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002Fportal_news\u002Fnews_ewm02.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=9Am%2FQfw7y%2BpppXGy6g1VG29SISM%3D)",{"publishTime":7,"createTime":109,"canonicalPath":114,"id":115,"title":116},"\u002Fstudy-material\u002Fstudy-94","study-94","2026 夏季 InfiniTensor 训练营 Triton & 九齿 5：《九齿三重：天通》",15,"\u002Fstudy-material\u002Fstudy-95","study-95","markdown",{"publishTime":7,"createTime":109,"canonicalPath":122,"id":123,"title":124},"\u002Fstudy-material\u002Fstudy-96","study-96","2026 夏季 InfiniTensor 训练营 AI 编程与智能体应用 7：《AI Agent 应用：本地模型接入与能力调用》",{"htmlContent":126,"headings":127},"\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw01.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=9Q2JAKNB3dkNsEG3uBsT5hj1Ieo%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cspan style=\"font-size:12px;color:#555;\">8 月 6 日，2026 夏季 InfiniTensor 训练营推理引擎方向第五课《大模型服务系统》开讲。\u003Cbr>上节课聚焦推理底层算子与分布式并行，解决单卡显存不足、单 Token 推理延迟高问题。本节课转向线上生产服务完整工程链路，覆盖对话业务功能、流式接口、KV Cache 内存池、多请求调度整套落地方案。\u003C\u002Fspan>\u003C\u002Fp>\n\u003Cp>本节课从线上服务落地视角，讲解对话前端、OpenAI 标准接口、KV Cache 内存管理、批量调度、连续批次等完整服务架构。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw02.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kBV%2BmbrbqQUMJQsoQGU0pIqHzNk%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"大模型对话产品业务功能梳理\">\u003Ca class=\"anchor\" href=\"#大模型对话产品业务功能梳理\" aria-label=\"跳转到标题：大模型对话产品业务功能梳理\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>大模型对话产品业务功能梳理\u003C\u002Fh2>\u003Ch3 id=\"基础交互能力\">\u003Ca class=\"anchor\" href=\"#基础交互能力\" aria-label=\"跳转到标题：基础交互能力\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>基础交互能力\u003C\u002Fh3>\u003Col>\n\u003Cli>逐 Token 流式输出，实时展示生成内容；\u003C\u002Fli>\n\u003Cli>推理中断：支持随时终止当前回答；\u003C\u002Fli>\n\u003Cli>多轮上下文记忆，支持历史对话编辑、回溯分支；\u003C\u002Fli>\n\u003Cli>多会话隔离：新建、切换、删除不同话题会话。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw03.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=uiexvauW%2FMrLXOBPUmNKtNS7VKU%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"底层推理循环基础逻辑\">\u003Ca class=\"anchor\" href=\"#底层推理循环基础逻辑\" aria-label=\"跳转到标题：底层推理循环基础逻辑\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>底层推理循环基础逻辑\u003C\u002Fh3>\u003Cp>单次推理循环流程：输入上下文 → Prefill 填充 KV Cache → 循环 Decode 生成 Token → 触发终止符\u002F长度上限\u002F用户断连后退出推理。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw04.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=aw2M%2B20m4cRrwtFV2XzHRRQhkjM%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"推理服务标准接口-openai-api-规范\">\u003Ca class=\"anchor\" href=\"#推理服务标准接口-openai-api-规范\" aria-label=\"跳转到标题：推理服务标准接口：OpenAI API 规范\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>推理服务标准接口：OpenAI API 规范\u003C\u002Fh2>\u003Ch3 id=\"同步非流式接口\">\u003Ca class=\"anchor\" href=\"#同步非流式接口\" aria-label=\"跳转到标题：同步非流式接口\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>同步非流式接口\u003C\u002Fh3>\u003Cp>请求路径：\u003Ccode>\u002Fv1\u002Fchat\u002Fcompletions\u003C\u002Fcode>\u003C\u002Fp>\n\u003Cul>\n\u003Cli>请求字段：model、messages（system\u002Fuser\u002Fassistant 角色数组）、超参数；\u003C\u002Fli>\n\u003Cli>返回：完整一次性对话结果，\u003Ccode>finish_reason\u003C\u002Fcode> 标记停止原因（stop \u002F length）。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw05.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=3W5DYPCHA9m5XdRBgSraeHWYgu4%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"流式推理服务\">\u003Ca class=\"anchor\" href=\"#流式推理服务\" aria-label=\"跳转到标题：流式推理服务\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>流式推理服务\u003C\u002Fh3>\u003Cp>请求增加 \u003Ccode>stream: true\u003C\u002Fcode> 标识：\u003C\u002Fp>\n\u003Col>\n\u003Cli>按生成过程持续返回增量内容；\u003C\u002Fli>\n\u003Cli>用户断连后后端自动终止推理，节省算力；\u003C\u002Fli>\n\u003Cli>解决长等待无反馈、中途放弃浪费资源问题。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw06.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=V0dAqkaC1v5wW4kqKU3bkckAKAQ%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"服务端-handler-实现\">\u003Ca class=\"anchor\" href=\"#服务端-handler-实现\" aria-label=\"跳转到标题：服务端 Handler 实现\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>服务端 Handler 实现\u003C\u002Fh3>\u003Cp>基于 FastAPI 等 Web 框架搭建 POST 接口，解析 JSON 请求，转发至底层推理引擎，同步\u002F流式封装返回数据包。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw07.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=10zqesQtyzEi4hG7blFYryFj7Jw%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"多会话核心-kv-cache-内存管理机制\">\u003Ca class=\"anchor\" href=\"#多会话核心-kv-cache-内存管理机制\" aria-label=\"跳转到标题：多会话核心：KV Cache 内存管理机制\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>多会话核心：KV Cache 内存管理机制\u003C\u002Fh2>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw08.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=NkwOdbEc4M8dfsPCm4%2FFiStglk4%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw09.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=32YSnsCGLCc%2FECznt8PV4Nt4omA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"方案一-会话绑定完整-kv-cache\">\u003Ca class=\"anchor\" href=\"#方案一-会话绑定完整-kv-cache\" aria-label=\"跳转到标题：方案一：会话绑定完整 KV Cache\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>方案一：会话绑定完整 KV Cache\u003C\u002Fh3>\u003Cp>每个会话独立分配一整块最大长度 KV Cache；\u003C\u002Fp>\n\u003Cul>\n\u003Cli>优点：上下文前缀 100% 复用，无需重算；\u003C\u002Fli>\n\u003Cli>缺点：会话数量受显存严格限制，分支场景内存膨胀严重。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw10.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=%2FL5GKzbYp9RF18wz59rRrG6vGhM%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"方案二-kv-cache-全局池-前缀匹配\">\u003Ca class=\"anchor\" href=\"#方案二-kv-cache-全局池-前缀匹配\" aria-label=\"跳转到标题：方案二：KV Cache 全局池 + 前缀匹配\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>方案二：KV Cache 全局池 + 前缀匹配\u003C\u002Fh3>\u003Cp>共用固定 Cache 内存池，新请求匹配最长历史前缀复用缓存块；\u003C\u002Fp>\n\u003Cul>\n\u003Cli>优点：相比固定 Cache 分配方式，可以显著提高显存利用率，支持更多并发会话;\u003C\u002Fli>\n\u003Cli>缺点：无匹配时覆盖旧缓存，部分场景需重复计算。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw11.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=9VlEif85FF540zWRsFTQSFbOgBA%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"分页式-paged-attention-工业主流方案\">\u003Ca class=\"anchor\" href=\"#分页式-paged-attention-工业主流方案\" aria-label=\"跳转到标题：分页式 Paged Attention（工业主流方案）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>分页式 Paged Attention（工业主流方案）\u003C\u002Fh3>\u003Ch4 id=\"1-核心思想\">\u003Ca class=\"anchor\" href=\"#1-核心思想\" aria-label=\"跳转到标题：1. 核心思想\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1. 核心思想\u003C\u002Fh4>\u003Cp>借鉴操作系统虚拟内存分页，不分配整块 Cache，拆分为固定大小 Page 小块，通过映射表拼接逻辑上下文。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw12.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=15OWKZGG6v0Ynxe9cL1A6Jy4s00%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"2-复用与分支逻辑\">\u003Ca class=\"anchor\" href=\"#2-复用与分支逻辑\" aria-label=\"跳转到标题：2. 复用与分支逻辑\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2. 复用与分支逻辑\u003C\u002Fh4>\u003Cp>多会话共享相同前缀 Page，采用引用计数管理：\u003C\u002Fp>\n\u003Col>\n\u003Cli>共享块引用计数 +1；\u003C\u002Fli>\n\u003Cli>当多个会话共享同一 Page，后续请求需要修改该 Page 内容时，通过 Copy-on-Write 创建新的 Page，避免影响其他会话。\u003C\u002Fli>\n\u003Cli>会话销毁时引用计数 -1，计数为 0 回收 Page 内存。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw13.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=Cb7MiWW2LQGLbrhzh3U3FCZDTxc%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch4 id=\"3-内存扩容兜底策略\">\u003Ca class=\"anchor\" href=\"#3-内存扩容兜底策略\" aria-label=\"跳转到标题：3. 内存扩容兜底策略\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>3. 内存扩容兜底策略\u003C\u002Fh4>\u003Cp>Page 池耗尽时，可将闲置 Page Offload 至 CPU \u002F NVME 磁盘释放 GPU 显存。\u003C\u002Fp>\n\u003Ch4 id=\"4-显存占用现状\">\u003Ca class=\"anchor\" href=\"#4-显存占用现状\" aria-label=\"跳转到标题：4. 显存占用现状\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>4. 显存占用现状\u003C\u002Fh4>\u003Cp>7 B \u002F 13 B 模型推理场景，KV Cache 通常占用超过 30% 单卡显存，分页机制大幅提升并发会话上限。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw14.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=OQVNByvODPbSLEw4DYVggaNfje0%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw15.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=1d8bfIUhHeV4yJVRV%2FS%2FJ%2Fz54g8%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"多请求批量调度方案演进\">\u003Ca class=\"anchor\" href=\"#多请求批量调度方案演进\" aria-label=\"跳转到标题：多请求批量调度方案演进\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>多请求批量调度方案演进\u003C\u002Fh2>\u003Ch3 id=\"方案1-串行单请求处理\">\u003Ca class=\"anchor\" href=\"#方案1-串行单请求处理\" aria-label=\"跳转到标题：方案1：串行单请求处理\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>方案1：串行单请求处理\u003C\u002Fh3>\u003Cp>请求排队依次推理；\u003C\u002Fp>\n\u003Cul>\n\u003Cli>缺陷：GPU 算力空置，吞吐极低，无法支撑线上并发。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw16.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=tK%2B9ZRwIRXxTZgJN1ynXKwnDDrM%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"方案2-固定批次-padding-批处理\">\u003Ca class=\"anchor\" href=\"#方案2-固定批次-padding-批处理\" aria-label=\"跳转到标题：方案2：固定批次 Padding 批处理\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>方案2：固定批次 Padding 批处理\u003C\u002Fh3>\u003Cp>同一轮合并多条请求，短请求补零对齐张量；\u003C\u002Fp>\n\u003Cul>\n\u003Cli>缺陷：短请求等待长请求，Padding 带来无效算力开销；Prefill \u002F Decode 请求难以进行高效混合调度。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw17.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=oHNkfrq8lsyhy6Z5G5jGkNxhpr8%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"方案3-逐轮动态重批-基础优化\">\u003Ca class=\"anchor\" href=\"#方案3-逐轮动态重批-基础优化\" aria-label=\"跳转到标题：方案3：逐轮动态重批（基础优化）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>方案3：逐轮动态重批（基础优化）\u003C\u002Fh3>\u003Cp>每轮 Decode 结束后重新从请求池组合 Batch，完成请求直接退出；\u003C\u002Fp>\n\u003Cul>\n\u003Cli>改善短请求等待问题，但仍存在大量 Padding 无效计算。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw18.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=2wZURCL2vCSTRk7h03kCac4yOXk%3D\" alt=\"\">\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw19.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=hTPJpinjAF3VHr7zsusVOoxi7Hg%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch3 id=\"方案4-连续批次-continuous-batching-工业最优\">\u003Ca class=\"anchor\" href=\"#方案4-连续批次-continuous-batching-工业最优\" aria-label=\"跳转到标题：方案4：连续批次 Continuous Batching（工业最优）\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>方案4：连续批次 Continuous Batching（工业最优）\u003C\u002Fh3>\u003Ch4 id=\"1-分层计算拆分\">\u003Ca class=\"anchor\" href=\"#1-分层计算拆分\" aria-label=\"跳转到标题：1.分层计算拆分\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>1.分层计算拆分\u003C\u002Fh4>\u003Cul>\n\u003Cli>无依赖算子（Linear \u002F RMSNorm \u002F RoPE）：可通过连续批处理方式减少 Padding 开销.\u003C\u002Fli>\n\u003Cli>Attention 层根据各请求的 block table 访问对应 KV Cache，通过批量化 Kernel 完成不同请求 Attention 计算。\u003C\u002Fli>\n\u003C\u002Ful>\n\u003Ch4 id=\"2-整体收益\">\u003Ca class=\"anchor\" href=\"#2-整体收益\" aria-label=\"跳转到标题：2.整体收益\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>2.整体收益\u003C\u002Fh4>\u003Cp>消除无效 Padding，GPU 硬件利用率大幅提升；支持 Prefill、Decode 请求混合调度，适配线上随机流量。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw20.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=YvUMEnz%2BkWnmEALXvLhiiQcRBRE%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"标准推理服务三层分离架构\">\u003Ca class=\"anchor\" href=\"#标准推理服务三层分离架构\" aria-label=\"跳转到标题：标准推理服务三层分离架构\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>标准推理服务三层分离架构\u003C\u002Fh2>\u003Ch3 id=\"前端接入模块\">\u003Ca class=\"anchor\" href=\"#前端接入模块\" aria-label=\"跳转到标题：前端接入模块\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>前端接入模块\u003C\u002Fh3>\u003Cp>Web 接口服务，接收 HTTP 请求、解析 OpenAI JSON、流式数据包推送。\u003C\u002Fp>\n\u003Ch3 id=\"调度管理模块\">\u003Ca class=\"anchor\" href=\"#调度管理模块\" aria-label=\"跳转到标题：调度管理模块\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>调度管理模块\u003C\u002Fh3>\u003Col>\n\u003Cli>全局请求池维护；\u003C\u002Fli>\n\u003Cli>每轮构造连续批次；\u003C\u002Fli>\n\u003Cli>Paged Attention KV Cache 块分配、引用计数、内存回收；\u003C\u002Fli>\n\u003Cli>绑定 IO 管线，管理每路用户流式输出。\u003C\u002Fli>\n\u003C\u002Fol>\n\u003Ch3 id=\"底层推理引擎模块\">\u003Ca class=\"anchor\" href=\"#底层推理引擎模块\" aria-label=\"跳转到标题：底层推理引擎模块\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>底层推理引擎模块\u003C\u002Fh3>\u003Cp>执行 Prefill \u002F Decode 推理循环，提供算子、分布式推理底层能力。\u003C\u002Fp>\n\u003Ch3 id=\"模块隔离优势\">\u003Ca class=\"anchor\" href=\"#模块隔离优势\" aria-label=\"跳转到标题：模块隔离优势\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>模块隔离优势\u003C\u002Fh3>\u003Cp>三模块通过异步队列进行解耦，使请求接收、调度管理和模型计算能够尽可能并行执行，提高系统吞吐。\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002FKB_95_zw21.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=kWtyUYjtHMCfhzUrsjlQ4347dEg%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"加入训练营\">\u003Ca class=\"anchor\" href=\"#加入训练营\" aria-label=\"跳转到标题：加入训练营\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>加入训练营\u003C\u002Fh2>\u003Cp>📺\u003Cstrong>完整课程内容，请观看直播或查看课程回放：\u003C\u002Fstrong>\u003Cbr>\u003Cstrong>InfiniTensor 官网\u003C\u002Fstrong>： \u003Ca href=\"https:\u002F\u002Fwww.infinitensor.com\">https:\u002F\u002Fwww.infinitensor.com\u003C\u002Fa>\u003C\u002Fp>\n\u003Cp>\u003Cstrong>B站直播\u003C\u002Fstrong>： InfiniTensor 官方直播间\u003C\u002Fp>\n\u003Cp>\u003Cstrong>视频号直播\u003C\u002Fstrong>： InfiniTensor 视频号预约\u003C\u002Fp>\n\u003Cp>\u003Cstrong>答疑交流\u003C\u002Fstrong>： 训练营官方社群⬇️\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002FKnowledge_Base\u002Fqun_1072.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=GIJwh433B3Yl1X2vYlwRpO%2B9Mps%3D\" alt=\"\">\u003C\u002Fp>\n\u003Ch2 id=\"关注与交流\">\u003Ca class=\"anchor\" href=\"#关注与交流\" aria-label=\"跳转到标题：关注与交流\">\u003Cspan class=\"octicon octicon-link\">\u003C\u002Fspan>\u003C\u002Fa>关注与交流\u003C\u002Fh2>\u003Cp>\u003Cimg src=\"https:\u002F\u002Foss.infinitensor.com\u002Fprod\u002Fimages\u002Fportal_news\u002Fnews_ewm02.jpg?Expires=1792752563&OSSAccessKeyId=LTAI5t9R6uzvTa69CT92cFvV&Signature=9Am%2FQfw7y%2BpppXGy6g1VG29SISM%3D\" alt=\"\">\u003C\u002Fp>\n",[128,130,132,134,137,139,141,144,147,150,153,156,160,163,166,169,171,174,177,180,183,186,189,191,193,195,197,199,201],{"id":129,"text":129,"level":110},"大模型对话产品业务功能梳理",{"id":131,"text":131,"level":9},"基础交互能力",{"id":133,"text":133,"level":9},"底层推理循环基础逻辑",{"id":135,"text":136,"level":110},"推理服务标准接口-openai-api-规范","推理服务标准接口：OpenAI API 规范",{"id":138,"text":138,"level":9},"同步非流式接口",{"id":140,"text":140,"level":9},"流式推理服务",{"id":142,"text":143,"level":9},"服务端-handler-实现","服务端 Handler 实现",{"id":145,"text":146,"level":110},"多会话核心-kv-cache-内存管理机制","多会话核心：KV Cache 内存管理机制",{"id":148,"text":149,"level":9},"方案一-会话绑定完整-kv-cache","方案一：会话绑定完整 KV Cache",{"id":151,"text":152,"level":9},"方案二-kv-cache-全局池-前缀匹配","方案二：KV Cache 全局池 + 前缀匹配",{"id":154,"text":155,"level":9},"分页式-paged-attention-工业主流方案","分页式 Paged Attention（工业主流方案）",{"id":157,"text":158,"level":159},"1-核心思想","1. 核心思想",4,{"id":161,"text":162,"level":159},"2-复用与分支逻辑","2. 复用与分支逻辑",{"id":164,"text":165,"level":159},"3-内存扩容兜底策略","3. 内存扩容兜底策略",{"id":167,"text":168,"level":159},"4-显存占用现状","4. 显存占用现状",{"id":170,"text":170,"level":110},"多请求批量调度方案演进",{"id":172,"text":173,"level":9},"方案1-串行单请求处理","方案1：串行单请求处理",{"id":175,"text":176,"level":9},"方案2-固定批次-padding-批处理","方案2：固定批次 Padding 批处理",{"id":178,"text":179,"level":9},"方案3-逐轮动态重批-基础优化","方案3：逐轮动态重批（基础优化）",{"id":181,"text":182,"level":9},"方案4-连续批次-continuous-batching-工业最优","方案4：连续批次 Continuous Batching（工业最优）",{"id":184,"text":185,"level":159},"1-分层计算拆分","1.分层计算拆分",{"id":187,"text":188,"level":159},"2-整体收益","2.整体收益",{"id":190,"text":190,"level":110},"标准推理服务三层分离架构",{"id":192,"text":192,"level":9},"前端接入模块",{"id":194,"text":194,"level":9},"调度管理模块",{"id":196,"text":196,"level":9},"底层推理引擎模块",{"id":198,"text":198,"level":9},"模块隔离优势",{"id":200,"text":200,"level":110},"加入训练营",{"id":202,"text":202,"level":110},"关注与交流"]