{"id":15749,"date":"2026-11-12T16:33:37","date_gmt":"2026-11-12T08:33:37","guid":{"rendered":"https:\/\/ai-stack.ai\/?p=15749"},"modified":"2026-11-12T16:36:43","modified_gmt":"2026-11-12T08:36:43","slug":"ai-training-at-scale-slurm-on-k8s","status":"publish","type":"post","link":"https:\/\/ai-stack.ai\/zh-hant\/ai-training-at-scale-slurm-on-k8s","title":{"rendered":"AI \u8a13\u7df4\u898f\u6a21\u5316\u7684\u4e09\u5927\u6311\u6230\uff1aAI-Stack Slurm on K8s \u5982\u4f55\u56e0\u61c9\u00a0"},"content":{"rendered":"\n<p class=\"wp-block-paragraph\">\u96a8\u8457\u5927\u578b\u8a9e\u8a00\u6a21\u578b\u6301\u7e8c\u64f4\u5927\uff0cAI \u8a13\u7df4\u6b63\u5f9e\u55ae\u6a5f\u8d70\u5411\u591a\u7bc0\u9ede\u3001\u5927\u898f\u6a21\u7684\u5206\u6563\u5f0f\u67b6\u69cb\u3002\u898f\u6a21\u5e36\u4f86\u7b97\u529b\uff0c\u4e5f\u5e36\u4f86\u4e09\u5927\u6311\u6230\uff1a\u8cc7\u6e90\u6392\u7a0b\u3001\u7dad\u904b\u7ba1\u7406\u8207\u6545\u969c\u97cc\u6027\u3002AI-Stack Slurm on Kubernetes\u5c07 HPC \u4e3b\u6d41\u6392\u7a0b\u7cfb\u7d71 Slurm \u90e8\u7f72\u65bc Kubernetes \u5bb9\u5668\u5e73\u53f0\uff0c\u7531 Slurm \u8ca0\u8cac\u4f5c\u696d\u6392\u7a0b\u3001Kubernetes \u8ca0\u8cac\u5bb9\u5668\u5316\u7dad\u904b\uff0c\u662f\u4f01\u696d\u56e0\u61c9\u9019\u4e9b\u6311\u6230\u7684\u95dc\u9375\u67b6\u69cb\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u8da8\u52e2\uff1aAI \u8a13\u7df4\u5168\u9762\u8d70\u5411\u8de8\u7bc0\u9ede\u3001\u5927\u898f\u6a21\u5316<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u5927\u578b\u8a9e\u8a00\u6a21\u578b\u7684\u53c3\u6578\u91cf\u8207\u8a13\u7df4\u8cc7\u6599\u6301\u7e8c\u6210\u9577\u3002\u4ee5 Meta \u7684 Llama 3 405B \u70ba\u4f8b\uff0c<a href=\"https:\/\/arxiv.org\/html\/2407.21783\" target=\"_blank\" rel=\"noopener\">\u8a13\u7df4\u6642\u6700\u591a\u540c\u6642\u4f7f\u7528 16,000 \u5f35 H100 GPU<\/a>\u3002\u7576\u6a21\u578b\u6216\u8cc7\u6599\u8d85\u51fa\u55ae\u53f0\u4f3a\u670d\u5668\u7684\u8ca0\u8377\uff0c\u5c31\u9700\u8981\u6574\u5408\u591a\u53f0\u4f3a\u670d\u5668\u7684 GPU \u5171\u540c\u57f7\u884c\u540c\u4e00\u500b\u8a13\u7df4\u4f5c\u696d\uff0c\u5373\u8de8\u7bc0\u9ede\u5206\u6563\u5f0f\u8a13\u7df4\u3002\u7531\u65bc\u5404\u7bc0\u9ede\u9808\u983b\u7e41\u4ea4\u63db\u8cc7\u6599\u3001\u9032\u5ea6\u9ad8\u5ea6\u76f8\u4f9d\uff0c\u5f9e\u55ae\u6a5f\u64f4\u5c55\u5230\u591a\u6a5f\uff0c\u589e\u52a0\u7684\u4e0d\u53ea\u662f GPU \u6578\u91cf\uff0c\u9084\u6709\u5354\u8abf\u4e0a\u7684\u8907\u96dc\u5ea6\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u5e73\u53f0\u5c64\uff0cSlurm \u8207 Kubernetes \u662f\u5169\u5927\u4e3b\u6d41\u3002<a href=\"https:\/\/slurm.schedmd.com\/quickstart.html\" target=\"_blank\" rel=\"noopener\">Slurm<\/a> \u662f\u958b\u6e90\u7684\u53e2\u96c6\u7ba1\u7406\u8207\u4f5c\u696d\u6392\u7a0b\u7cfb\u7d71\uff0c\u5c08\u70ba\u5927\u578b\u6279\u6b21\u4f5c\u696d\u8a2d\u8a08\u3002\u6839\u64da <a href=\"https:\/\/blogs.nvidia.com\/blog\/nvidia-acquires-schedmd\" target=\"_blank\" rel=\"noopener\">NVIDIA \u7684\u8aaa\u6cd5<\/a>\uff0c\u5168\u7403 TOP500 \u8d85\u7d1a\u96fb\u8166\u7684\u524d 10 \u540d\u8207\u524d 100 \u540d\u4e2d\uff0c\u5747\u6709\u8d85\u904e\u534a\u6578\u63a1\u7528 Slurm\u3002Kubernetes\u5247\u662f\u4f01\u696d\u7ba1\u7406\u5bb9\u5668\u5316\u670d\u52d9\u7684\u4e3b\u6d41\u5e73\u53f0\uff0c\u793e\u7fa4\u4e5f\u6301\u7e8c\u5f37\u5316\u5206\u6563\u5f0f\u4f5c\u696d\u7684\u652f\u63f4\uff0c\u539f\u751f\u7684 <a href=\"https:\/\/kubernetes.io\/docs\/concepts\/scheduling-eviction\/gang-scheduling\/\" target=\"_blank\" rel=\"noopener\">Gang \u6392\u7a0b<\/a>\u5df2\u65bc v1.37 \u9032\u5165 Beta \u968e\u6bb5\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u7136\u800c\uff0c\u5e73\u53f0\u6210\u719f\u4e26\u4e0d\u4ee3\u8868\u898f\u6a21\u5316\u8a13\u7df4\u5df2\u7121\u969c\u7919\u3002\u7576\u8a13\u7df4\u64f4\u5c55\u5230\u6578\u5341\u3001\u6578\u767e\u5f35 GPU\uff0c\u4f01\u696d\u5c07\u9762\u81e8\u4e09\u5927\u6311\u6230\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u6311\u6230\uff1a\u898f\u6a21\u5316 AI \u8a13\u7df4\u7684\u4e09\u5927\u6311\u6230<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">\u6311\u6230\u4e00\uff1a\u8cc7\u6e90\u6392\u7a0b\u2014\u2014\u5206\u6563\u5f0f\u4f5c\u696d\u53ef\u80fd\u4e92\u76f8\u9396\u6b7b<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u55ae\u4e00\u8a13\u7df4\u4f5c\u696d\u9700\u8981\u591a\u500b\u7bc0\u9ede\u540c\u6642\u5c31\u7dd2\uff0c\u4f46 Kubernetes \u7684<a href=\"https:\/\/kubernetes.io\/docs\/concepts\/scheduling-eviction\/kube-scheduler\/\" target=\"_blank\" rel=\"noopener\">\u9810\u8a2d\u6392\u7a0b\u5668<\/a>\u6703\u9010\u4e00\u70ba\u6bcf\u500b Pod \u5c0b\u627e\u7bc0\u9ede\uff0c\u800c\u975e\u5c07\u540c\u4e00\u4f5c\u696d\u7684\u6240\u6709 Pod \u4e00\u4f75\u6392\u7a0b\u3002\u5047\u8a2d\u53e2\u96c6\u5171\u6709 4 \u5f35 GPU\uff0c\u4efb\u52d9 A \u8207\u4efb\u52d9 B \u5404\u9700 4 \u5f35\uff1b\u82e5\u96d9\u65b9\u5404\u53d6\u5f97 2 \u5f35\uff0c\u5169\u500b\u4efb\u52d9\u90fd\u7121\u6cd5\u555f\u52d5\uff0c\u5df2\u53d6\u5f97\u7684 GPU \u4e5f\u4e0d\u6703\u91cb\u653e\uff08\u898b\u5716 1\uff09\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img data-recalc-dims=\"1\" fetchpriority=\"high\" decoding=\"async\" width=\"1024\" height=\"417\" data-attachment-id=\"15754\" data-permalink=\"https:\/\/ai-stack.ai\/zh-hant\/ai-training-at-scale-slurm-on-k8s\/image-98\" data-orig-file=\"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/e7842c80-1.png?fit=1600%2C652&amp;quality=100&amp;ct=202603031250&amp;ssl=1\" data-orig-size=\"1600,652\" data-comments-opened=\"0\" data-image-title=\"image\" data-image-description=\"\" data-image-caption=\"\" data-large-file=\"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/e7842c80-1.png?fit=1024%2C417&amp;quality=100&amp;ct=202603031250&amp;ssl=1\" src=\"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/25ad36d9-1.png?resize=1024%2C417&#038;quality=100&#038;ct=202603031250&#038;ssl=1\" alt=\"\" class=\"wp-image-15754\" style=\"aspect-ratio:2.459016393442623\" title=\"slurm-on-k8s-gang-scheduling-deadlock.png\"\/><figcaption class=\"wp-element-caption\">\u5716 1\u3000Kubernetes \u9010\u4e00\u6392\u7a0b\u53ef\u80fd\u9020\u6210\u6b7b\u9396\uff0c\u6574\u7d44\u6392\u7a0b\uff08Gang Scheduling\uff09\u53ef\u907f\u514d\u6b64\u554f\u984c<\/figcaption><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u89e3\u6c7a\u65b9\u6cd5\u662f\u6574\u7d44\u6392\u7a0b\uff08Gang Scheduling\uff09\uff1a\u540c\u4e00\u4f5c\u696d\u7684\u6240\u6709 Pod \u90fd\u53d6\u5f97\u8cc7\u6e90\u5f8c\u624d\u4e00\u8d77\u555f\u52d5\u3002Kubernetes \u539f\u751f\u7684 Gang \u6392\u7a0b\u96d6\u5df2\u9032\u5165 Beta\uff0c\u4f46\u9810\u8a2d\u4e26\u672a\u555f\u7528\uff0c\u8a55\u4f30\u5e73\u53f0\u6642\u61c9\u78ba\u8a8d\u6240\u4f7f\u7528\u7684\u7248\u672c\u8207\u8a2d\u5b9a\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u6311\u6230\u4e8c\uff1a\u7dad\u904b\u7ba1\u7406\u2014\u2014\u50b3\u7d71 Slurm \u53e2\u96c6\u4ef0\u8cf4\u4eba\u529b<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">Slurm \u64c5\u9577\u6392\u7a0b\uff0c\u4f46\u50b3\u7d71 Slurm \u53e2\u96c6\u591a\u76f4\u63a5\u90e8\u7f72\u65bc\u88f8\u6a5f\uff0c\u7dad\u904b\u4e0a\u6709\u4e09\u9805\u75db\u9ede\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u74b0\u5883\u4ef0\u8cf4\u4eba\u5de5\u7dad\u8b77<\/strong>\uff1a\u9a45\u52d5\u7a0b\u5f0f\u3001\u51fd\u5f0f\u5eab\u8207\u8edf\u9ad4\u7248\u672c\u9808\u624b\u52d5\u7ba1\u7406\uff0c\u5bb9\u6613\u767c\u751f\u76f8\u4f9d\u6027\u885d\u7a81\u3002<\/li>\n\n\n\n<li><strong>\u7f3a\u4e4f\u81ea\u6211\u4fee\u5fa9\u6a5f\u5236<\/strong>\uff1a\u5143\u4ef6\u6216\u7bc0\u9ede\u7570\u5e38\u6642\uff0c\u591a\u534a\u9700\u8981\u7dad\u904b\u4eba\u54e1\u4ecb\u5165\u3002<\/li>\n\n\n\n<li><strong>\u8cc7\u6e90\u7121\u6cd5\u5f48\u6027\u5171\u7528<\/strong>\uff1a\u8a13\u7df4\u53e2\u96c6\u96e3\u4ee5\u8207\u63a8\u8ad6\u670d\u52d9\u3001\u958b\u767c\u74b0\u5883\u5171\u7528\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u82e5\u6539\u7528 Kubernetes\uff0c\u96d6\u53ef\u89e3\u6c7a\u74b0\u5883\u4e00\u81f4\u6027\u8207\u81ea\u6211\u4fee\u5fa9\u554f\u984c\uff0c\u4f46\u65e2\u6709\u7684 Slurm \u6307\u4ee4\u8207\u4f5c\u696d\u8173\u672c\u7121\u6cd5\u76f4\u63a5\u79fb\u8f49\uff1b\u5c0d\u5df2\u6709 Slurm \u4f7f\u7528\u8005\u7684\u7d44\u7e54\u800c\u8a00\uff0c\u8f49\u63db\u6210\u672c\u4f9d\u7136\u5b58\u5728\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u6311\u6230\u4e09\uff1a\u6545\u969c\u97cc\u6027\u2014\u2014\u7bc0\u9ede\u8d8a\u591a\uff0c\u4e2d\u65b7\u8d8a\u983b\u7e41<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><a href=\"https:\/\/arxiv.org\/html\/2407.21783\" target=\"_blank\" rel=\"noopener\">Meta \u5728 Llama 3 \u6280\u8853\u5831\u544a\u4e2d\u63ed\u9732<\/a>\uff0c\u8a13\u7df4 Llama 3 405B \u7684 54 \u5929\u5167\u5171\u767c\u751f 419 \u6b21\u975e\u9810\u671f\u4e2d\u65b7\uff0c\u5e73\u5747\u7d04\u6bcf 3 \u5c0f\u6642\u4e00\u6b21\uff0c\u5176\u4e2d GPU \u554f\u984c\u5360 58.7%\uff0c\u70ba\u6700\u4e3b\u8981\u7684\u4e2d\u65b7\u539f\u56e0\u3002\u4f01\u696d\u53e2\u96c6\u7684\u898f\u6a21\u96d6\u5c0f\u5f97\u591a\uff0c\u539f\u7406\u537b\u76f8\u540c\uff1a\u5047\u8a2d\u55ae\u4e00\u7bc0\u9ede\u6bcf\u65e5\u6545\u969c\u6a5f\u7387\u50c5 0.1%\uff0c100 \u53f0\u6a5f\u5668\u9023\u7e8c\u904b\u884c 30 \u5929\uff0c\u81f3\u5c11\u767c\u751f\u4e00\u6b21\u6545\u969c\u7684\u6a5f\u7387\u5c31\u63a5\u8fd1 95%\uff08\u793a\u7bc4\u7528\u5047\u8a2d\uff0c\u4e26\u5047\u8a2d\u5404\u7bc0\u9ede\u6545\u969c\u5f7c\u6b64\u7368\u7acb\uff09\u3002<a href=\"https:\/\/arxiv.org\/abs\/2410.21680\" target=\"_blank\" rel=\"noopener\">Meta \u7814\u7a76\u5718\u968a\u5c0d\u5169\u500b\u5927\u578b\u6a5f\u5668\u5b78\u7fd2\u53e2\u96c6\u7684\u5206\u6790<\/a>\u4e5f\u986f\u793a\uff0c\u5927\u578b\u4f5c\u696d\u53d7\u6545\u969c\u7684\u885d\u64ca\u660e\u986f\u66f4\u5927\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u70ba\u4ec0\u9ebc Checkpoint \u8207 Kubernetes \u81ea\u6211\u4fee\u5fa9\u4ecd\u4e0d\u8db3\u5920\uff1f<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u6700\u5e38\u898b\u7684\u9632\u8b77\u63aa\u65bd\u662f Checkpoint\uff08\u6aa2\u67e5\u9ede\uff09\uff1a\u5b9a\u671f\u4fdd\u5b58\u8a13\u7df4\u9032\u5ea6\u3001\u6a21\u578b\u53c3\u6578\u8207\u512a\u5316\u5668\u72c0\u614b\uff0c\u6545\u969c\u6642\u5f9e\u6700\u8fd1\u4e00\u6b21\u7684\u5b58\u6a94\u63a5\u7e8c\u3002\u5b58\u6a94\u983b\u7387\u9700\u8981\u6b0a\u8861\uff0c\u592a\u983b\u7e41\u6703\u58d3\u7e2e\u8a13\u7df4\u6642\u9593\uff0c\u592a\u7a00\u758f\u5247\u6bcf\u6b21\u6545\u969c\u640d\u5931\u66f4\u591a\u9032\u5ea6\u3002\u6700\u4f73\u5b58\u6a94\u9593\u9694\u53d6\u6c7a\u65bc\u5beb\u5165\u6210\u672c\u8207\u6545\u969c\u7387\uff1a\u7bc0\u9ede\u8d8a\u591a\u3001\u6545\u969c\u8d8a\u983b\u7e41\uff0c\u9593\u9694\u5c31\u61c9\u8d8a\u77ed\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u7136\u800c\uff0cCheckpoint \u53ea\u80fd\u907f\u514d\u9032\u5ea6\u6b78\u96f6\uff0c\u7121\u6cd5\u7e2e\u77ed\u6062\u5fa9\u6642\u9593\uff1a\u6545\u969c\u5f8c\u6574\u500b\u4f5c\u696d\u4ecd\u9808\u91cd\u65b0\u6392\u968a\u3001\u5206\u914d\u8cc7\u6e90\u4e26\u521d\u59cb\u5316\u74b0\u5883\uff0c\u5f80\u5f80\u8017\u6642\u6578\u5c0f\u6642\u3002Kubernetes \u7684\u81ea\u6211\u4fee\u5fa9\u4e5f\u7121\u6cd5\u88dc\u4e0a\u9019\u500b\u7f3a\u53e3\uff0c\u56e0\u70ba\u8a13\u7df4\u6062\u5fa9\u9084\u9808\u6eff\u8db3\u4e09\u9805\u689d\u4ef6\uff1a\u6240\u6709\u7bc0\u9ede\u91cd\u65b0\u9023\u7dda\u4e26\u8f09\u5165\u540c\u4e00\u4efd\u5b58\u6a94\u3001\u5373\u6642\u53d6\u5f97\u53ef\u7528\u7684\u66ff\u88dc\u7bc0\u9ede\uff0c\u4ee5\u53ca\u5148\u9694\u96e2\u6545\u969c\u786c\u9ad4\u4ee5\u514d\u5de5\u4f5c\u518d\u6b21\u843d\u5230\u540c\u4e00\u7bc0\u9ede\u3002\u9019\u4e9b\u90fd\u9700\u8981\u5c08\u70ba\u8a13\u7df4\u8a2d\u8a08\u7684\u6062\u5fa9\u6a5f\u5236\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">INFINITIX AI-Stack\uff1a\u4f01\u696d\u7d1a Slurm on K8s \u89e3\u6c7a\u65b9\u6848<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\"><a href=\"https:\/\/ai-stack.ai\/\">AI-Stack<\/a> \u662f\u4ee5 Kubernetes \u5bb9\u5668\u6280\u8853\u70ba\u6838\u5fc3\u7684 AI \u57fa\u790e\u8a2d\u65bd\u7ba1\u7406\u5e73\u53f0\uff0c Slurm on K8s \u662f\u5e73\u53f0\u8de8\u7bc0\u9ede\u8a13\u7df4\u529f\u80fd\u7684\u4e00\u74b0\uff0c\u91dd\u5c0d\u524d\u8ff0\u4e09\u5927\u6311\u6230\uff0cAI-Stack \u63d0\u4f9b\u5c0d\u61c9\u7684\u89e3\u6cd5\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u8cc7\u6e90\u6392\u7a0b<\/strong>\uff1a\u4ee5 Slurm \u6392\u7a0b\u8a13\u7df4\u4f5c\u696d\uff0c\u5e73\u53f0\u4e0a\u5176\u4ed6\u5206\u6563\u5f0f\u5de5\u4f5c\u8ca0\u8f09\u5247\u9810\u8a2d\u63a1\u7528\u6574\u7d44\u6392\u7a0b\uff0c\u907f\u514d\u8cc7\u6e90\u9396\u6b7b\u3002<\/li>\n\n\n\n<li><strong>\u7dad\u904b\u7ba1\u7406<\/strong>\uff1aSlurm \u5143\u4ef6\u4ee5\u5bb9\u5668\u5f62\u5f0f\u81ea\u52d5\u90e8\u7f72\uff0c\u4e26\u63d0\u4f9b\u5716\u5f62\u5316\u7ba1\u7406\u4ecb\u9762\u8207\u5b8c\u6574\u76e3\u63a7\u3002<\/li>\n\n\n\n<li><strong>\u6545\u969c\u97cc\u6027<\/strong>\uff1a\u652f\u63f4 Checkpoint \u5132\u5b58\u639b\u8f09\uff0c\u4e26\u900f\u904e\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\uff0c\u4ee5\u6eab\u5099\u7bc0\u9ede\u81ea\u52d5\u66ff\u63db\u6545\u969c\u7bc0\u9ede\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u5c0d\u7814\u7a76\u4eba\u54e1\u800c\u8a00\uff0c\u65e5\u5e38\u64cd\u4f5c\u5e7e\u4e4e\u4e0d\u8b8a\uff0c\u540c\u6a23\u900f\u904e SSH \u9023\u7dda\u4e26\u63d0\u4ea4\u4f5c\u696d\uff1b\u5c0d\u7dad\u904b\u4eba\u54e1\u800c\u8a00\uff0cSlurm \u5143\u4ef6\u6539\u4ee5\u5bb9\u5668\u5f62\u5f0f\u904b\u884c\uff0c\u53ef\u6cbf\u7528 K8s \u7684\u65b9\u5f0f\u90e8\u7f72\u8207\u7dad\u8b77\uff1b\u5c0d\u7ba1\u7406\u8005\u800c\u8a00\uff0c\u8a13\u7df4\u4e0d\u5fc5\u518d\u7368\u5360\u5c08\u5c6c\u53e2\u96c6\uff0c\u53ef\u8207\u63a8\u8ad6\u3001\u958b\u767c\u5171\u7528\u540c\u4e00\u500b\u5e95\u5c64\u8cc7\u6e90\u6c60\u3002Slurm on K8s \u5f9e\u67b6\u69cb\u4e0a\u89e3\u6c7a\u4e86\u8cc7\u6e90\u6392\u7a0b\u8207\u7dad\u904b\u7ba1\u7406\u7684\u554f\u984c\uff1b\u6545\u969c\u97cc\u6027\u5247\u642d\u914d\u5e73\u53f0\u5c64\u7684\u8a13\u7df4\u4e2d\u65b7\u81ea\u6062\u5fa9\u6a5f\u5236\u3002<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th><strong>\u6bd4\u8f03\u9805\u76ee<\/strong><\/th><th><strong>\u50b3\u7d71 Slurm \u53e2\u96c6<\/strong><\/th><th><strong>\u7d14 Kubernetes<\/strong><\/th><th><strong>Slurm on K8s<\/strong><\/th><\/tr><\/thead><tbody><tr><td><strong>\u4f5c\u696d\u6392\u7a0b<\/strong><\/td><td>\u5c08\u70ba\u5927\u898f\u6a21\u6279\u6b21\u4f5c\u696d\u8a2d\u8a08<\/td><td>\u64c5\u9577\u7dda\u4e0a\u670d\u52d9\uff0c\u6279\u6b21\u6392\u7a0b\u9700\u53e6\u884c\u88dc\u5f37<\/td><td>\u4ee5 Slurm \u6392\u7a0b\u8a13\u7df4\u4f5c\u696d<\/td><\/tr><tr><td><strong>MPI\uff0fHPC \u4f7f\u7528\u7fd2\u6163<\/strong><\/td><td>\u539f\u751f\u652f\u63f4<\/td><td>\u9700\u8981\u984d\u5916\u9069\u61c9<\/td><td>\u7dad\u6301\u539f\u6709\u7684\u4f5c\u696d\u63d0\u4ea4\u7fd2\u6163<\/td><\/tr><tr><td><strong>\u74b0\u5883\u4e00\u81f4\u6027<\/strong><\/td><td>\u9760\u4eba\u5de5\u7dad\u8b77\uff0c\u7248\u672c\u6613\u885d\u7a81<\/td><td>\u5bb9\u5668\u5316\uff0c\u74b0\u5883\u4e00\u81f4\u53ef\u651c<\/td><td>\u4ee5\u5bb9\u5668\u90e8\u7f72 Slurm \u5143\u4ef6<\/td><\/tr><tr><td><strong>\u7dad\u904b<\/strong><\/td><td>\u591a\u9760\u4eba\u5de5\u4ecb\u5165<\/td><td>\u670d\u52d9\u81ea\u7652\u3001\u81ea\u52d5\u5316\u7dad\u904b<\/td><td>\u7531 K8s \u8ca0\u8cac\u5143\u4ef6\u90e8\u7f72\u8207\u7dad\u904b<\/td><\/tr><tr><td><strong>\u8cc7\u6e90\u5171\u7528<\/strong><\/td><td>\u901a\u5e38\u5c08\u6a5f\u5c08\u7528<\/td><td>\u7d71\u4e00\u62bd\u8c61\u7db2\u8def\u8207\u5132\u5b58<\/td><td>\u8a13\u7df4\u3001\u63a8\u8ad6\u3001\u958b\u767c\u5171\u7528\u8cc7\u6e90\u6c60<\/td><\/tr><\/tbody><\/table><figcaption class=\"wp-element-caption\">\u8868 1\u3000\u50b3\u7d71 Slurm\u3001\u7d14 Kubernetes \u8207 Slurm on K8s \u7684\u6bd4\u8f03<\/figcaption><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u5167\u5efa\u6574\u7d44\u6392\u7a0b\uff0c\u907f\u514d\u8cc7\u6e90\u9396\u6b7b<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u9664 Slurm \u53e2\u96c6\u5916\uff0c\u5e73\u53f0\u4e0a\u7684\u5176\u4ed6\u5206\u6563\u5f0f\u5de5\u4f5c\u8ca0\u8f09\u540c\u6a23\u53d7\u5230\u6392\u7a0b\u4fdd\u8b77\uff1aAI-Stack \u5167\u5efa\u6279\u6b21\u6392\u7a0b\u6a5f\u5236\uff0c\u9810\u8a2d\u63a1\u7528\u6574\u7d44\u6392\u7a0b\uff0c\u6240\u6709 Pod \u7686\u5c31\u7dd2\u5f8c\u624d\u540c\u6642\u555f\u52d5\uff0c\u907f\u514d\u6b7b\u9396\uff1b\u4ea6\u53ef\u4f9d\u9700\u6c42\u555f\u7528\u96c6\u4e2d\u653e\u7f6e\u7b56\u7565\uff0c\u4ee5\u6e1b\u5c11\u8cc7\u6e90\u788e\u7247\u3002\u56e0\u6b64\uff0c\u8a13\u7df4\u3001\u63a8\u8ad6\u8207\u958b\u767c\u5de5\u4f5c\u5f97\u4ee5\u5728\u540c\u4e00\u8cc7\u6e90\u6c60\u4e2d\u5404\u81ea\u6b63\u78ba\u6392\u7a0b\uff0c\u5be6\u73fe Slurm on K8s \u8cc7\u6e90\u5171\u7528\u7684\u76ee\u6a19\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\uff1a\u6eab\u5099\u7bc0\u9ede\u81ea\u52d5\u66ff\u63db\uff0c\u5206\u9418\u7d1a\u6062\u5fa9\u8a13\u7df4<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u6eab\u5099\uff08Warm Standby\uff09\u662f\u6307\u9810\u5148\u5b8c\u6210\u6e96\u5099\u3001\u8655\u65bc\u5f85\u547d\u72c0\u614b\u7684\u5099\u7528\u7bc0\u9ede\uff0c\u53ef\u5728\u6545\u969c\u767c\u751f\u6642\u76f4\u63a5\u66ff\u63db\uff0c\u7701\u53bb\u7b49\u5f85\u8cc7\u6e90\u8207\u91cd\u5efa\u74b0\u5883\u7684\u6642\u9593\u3002AI-Stack \u7684\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\u529f\u80fd\u5373\u63a1\u7528\u6b64\u6a5f\u5236\uff0c\u4e26\u4ee5\u4e09\u500b\u6b65\u9a5f\u6db5\u84cb\u524d\u8ff0\u8a13\u7df4\u6062\u5fa9\u7684\u4e09\u9805\u689d\u4ef6\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>\u6545\u969c\u5075\u6e2c<\/strong>\uff1a\u5e73\u53f0\u900f\u904e AlertManager \u5373\u6642\u5075\u6e2c GPU \u6216\u8a2d\u5099\u5931\u806f\uff0c\u4e26\u89f8\u767c\u81ea\u52d5\u6062\u5fa9\u6d41\u7a0b\u3002<\/li>\n\n\n\n<li><strong>\u9694\u96e2\u8207\u66ff\u63db<\/strong>\uff1a\u5e73\u53f0\u81ea\u52d5\u9694\u96e2\u6545\u969c\u7bc0\u9ede\uff0c\u4e26\u5f9e\u8cc7\u6e90\u6c60\u559a\u9192\u4e00\u53f0\u9592\u7f6e\u7684\u5065\u5eb7\u7bc0\u9ede\u52a0\u5165\u53e2\u96c6\u3002<\/li>\n\n\n\n<li><strong>\u5f9e Checkpoint \u6062\u5fa9\u8a13\u7df4<\/strong>\uff1a\u65b0\u7bc0\u9ede\u5b8c\u6210\u521d\u59cb\u5316\u5f8c\uff0c\u4f5c\u696d\u5f9e\u6700\u8fd1\u4e00\u6b21\u4fdd\u5b58\u7684 Checkpoint \u6062\u5fa9\u57f7\u884c\u3002<\/li>\n<\/ol>\n\n\n\n<p class=\"wp-block-paragraph\">\u9808\u7279\u5225\u8aaa\u660e\u7684\u662f\uff0c\u7bc0\u9ede\u6545\u969c\u6642\uff0c\u57f7\u884c\u4e2d\u7684\u4f5c\u696d\u4ecd\u6703\u4e2d\u65b7\u3002\u81ea\u52d5\u6062\u5fa9\u7bc0\u7701\u7684\u662f\u91cd\u65b0\u6392\u968a\u3001\u91cd\u65b0\u7533\u8acb\u8cc7\u6e90\u8207\u91cd\u5efa\u53e2\u96c6\u7684\u6642\u9593\uff0c\u800c\u975e\u4f7f\u4f5c\u696d\u5b8c\u5168\u4e0d\u53d7\u5f71\u97ff\u3002\u4e09\u7a2e\u505a\u6cd5\u7684\u6062\u5fa9\u8def\u5f91\u8207\u6bd4\u8f03\uff0c\u5982\u5716 3\u6240\u793a\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img data-recalc-dims=\"1\" decoding=\"async\" width=\"1024\" height=\"474\" data-attachment-id=\"15766\" data-permalink=\"https:\/\/ai-stack.ai\/zh-hant\/ai-training-at-scale-slurm-on-k8s\/gpu-failure-recovery-checkpoint-auto-recovery\" data-orig-file=\"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/49cc68cd-edited.png?fit=1280%2C592&amp;quality=100&amp;ct=202603031250&amp;ssl=1\" data-orig-size=\"1280,592\" data-comments-opened=\"0\" data-image-title=\"gpu-failure-recovery-checkpoint-auto-recovery\" data-image-description=\"\" data-image-caption=\"&lt;p&gt;GPU \u6545\u969c\u5f8c\u4e09\u7a2e\u6062\u5fa9\u8def\u5f91\u6bd4\u8f03\uff1a\u7121\u4fdd\u8b77\u3001\u50c5 Checkpoint\u3001Checkpoint \u642d\u914d\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9&lt;\/p&gt;\n\" data-large-file=\"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/49cc68cd-edited.png?fit=1024%2C474&amp;quality=100&amp;ct=202603031250&amp;ssl=1\" src=\"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/49cc68cd-edited-1024x474.png?resize=1024%2C474&#038;quality=100&#038;ct=202603031250&#038;ssl=1\" alt=\"\" class=\"wp-image-15766\"\/><figcaption class=\"wp-element-caption\">\u5716 2\u3000GPU \u6545\u969c\u5f8c\u4e09\u7a2e\u505a\u6cd5\u7684\u6062\u5fa9\u8def\u5f91\u6bd4\u8f03<\/figcaption><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u6eab\u5099\u6a5f\u5236\u7684\u4ee3\u50f9\u4e3b\u8981\u6709\u5169\u9805\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u9700\u9810\u7559\u9592\u7f6e\u7bc0\u9ede<\/strong>\uff1a\u9019\u4e9b\u7bc0\u9ede\u5e73\u6642\u4e0d\u53c3\u8207\u8a13\u7df4\uff0c\u7b49\u540c\u4ee5\u90e8\u5206\u904b\u7b97\u8cc7\u6e90\u63db\u53d6\u6062\u5fa9\u901f\u5ea6\u3002\u9810\u7559\u6578\u91cf\u61c9\u4f9d\u53e2\u96c6\u898f\u6a21\u8207\u6545\u969c\u640d\u5931\u8a55\u4f30\u3002<\/li>\n\n\n\n<li><strong>\u5b58\u6a94\u5340\u9593\u7684\u9032\u5ea6\u4ecd\u9808\u91cd\u7b97<\/strong>\uff1a\u81ea\u52d5\u6062\u5fa9\u7e2e\u77ed\u7684\u662f\u6062\u5fa9\u6642\u9593\uff0c\u7121\u6cd5\u6e1b\u5c11\u5169\u6b21\u5b58\u6a94\u4e4b\u9593\u640d\u5931\u7684\u9032\u5ea6\uff0c\u56e0\u6b64\u5b58\u6a94\u7b56\u7565\u4f9d\u7136\u91cd\u8981\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u56e0\u6b64\uff0cCheckpoint \u8207\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\u5fc5\u9808\u642d\u914d\u904b\u7528\uff1a\u524d\u8005\u6c7a\u5b9a\u640d\u5931\u591a\u5c11\u9032\u5ea6\uff0c\u5f8c\u8005\u6c7a\u5b9a\u591a\u5feb\u6062\u5fa9\u8a13\u7df4\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">AI-Stack Slurm on K8s \u9069\u7528\u60c5\u5883\uff1a\u54ea\u4e9b\u5718\u968a\u6700\u9069\u5408\u5c0e\u5165\uff1f<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e26\u975e\u6240\u6709 AI \u5718\u968a\u90fd\u9700\u8981 Slurm on K8s\u3002\u7d9c\u5408\u524d\u8ff0\u7684\u6392\u7a0b\u67b6\u69cb\u8207\u6545\u969c\u6062\u5fa9\u6a5f\u5236\uff0c\u4ee5\u4e0b\u56db\u985e\u60c5\u5883\u7684\u5c0e\u5165\u6548\u76ca\u6700\u70ba\u986f\u8457\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><strong>\u60c5\u5883\u4e00\uff1a\u5df2\u6709 Slurm \u4f7f\u7528\u8005\u7684\u5b78\u8853\u7814\u7a76\u8207\u8d85\u7b97\u55ae\u4f4d<\/strong><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5927\u5b78\u3001\u7814\u7a76\u6a5f\u69cb\u8207\u570b\u5bb6\u7d1a\u8a08\u7b97\u4e2d\u5fc3\u7684\u7814\u7a76\u4eba\u54e1\u591a\u5df2\u719f\u6089 Slurm \u6307\u4ee4\uff0c\u4e26\u7d2f\u7a4d\u5927\u91cf\u4f5c\u696d\u8173\u672c\u3002\u9019\u985e\u55ae\u4f4d\u5e0c\u671b\u5c07\u88f8\u6a5f HPC \u53e2\u96c6\u5347\u7d1a\u70ba\u96f2\u539f\u751f\u67b6\u69cb\uff0c\u540c\u6642\u907f\u514d\u4f7f\u7528\u8005\u91cd\u65b0\u5b78\u7fd2\u65b0\u5de5\u5177\u3002Slurm on K8s \u8b93\u7814\u7a76\u4eba\u54e1\u5ef6\u7e8c\u65e2\u6709\u7684\u4f5c\u696d\u63d0\u4ea4\u65b9\u5f0f\uff0c\u7dad\u904b\u5247\u4ea4\u7531\u5bb9\u5668\u5316\u5e73\u53f0\u8ca0\u8cac\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u60c5\u5883\u4e8c\uff1a\u5927\u578b\u8a9e\u8a00\u6a21\u578b\u7684\u9810\u8a13\u7df4\u8207\u6301\u7e8c\u8a13\u7df4<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">LLM \u9810\u8a13\u7df4\u3001\u6301\u7e8c\u9810\u8a13\u7df4\u6216\u5927\u898f\u6a21\u5fae\u8abf\uff0c\u901a\u5e38\u9700\u8981\u6578\u5341\u5f35\u4ee5\u4e0a\u7684 GPU\uff0c\u4e26\u9023\u7e8c\u57f7\u884c\u6578\u5929\u81f3\u6578\u9031\u3002\u8a13\u7df4\u6846\u67b6\u5df2\u9810\u5148\u6574\u5408\uff0c\u642d\u914d Checkpoint \u8207\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\uff0c\u53ef\u6709\u6548\u964d\u4f4e\u9577\u6642\u9593\u8a13\u7df4\u4e2d\u65b7\u6240\u9020\u6210\u7684\u640d\u5931\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u60c5\u5883\u4e09\uff1a\u591a\u5718\u968a\u5171\u7528 GPU \u7684\u4f01\u696d AI \u7814\u767c\u4e2d\u5fc3<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u534a\u5c0e\u9ad4\u3001\u88fd\u9020\u3001\u91d1\u878d\u3001\u91ab\u7642\u7b49\u7522\u696d\u7684\u4f01\u696d AI \u7814\u767c\u4e2d\u5fc3\uff0c\u5e38\u898b\u7684\u60c5\u6cc1\u662f\u5c11\u6578\u5718\u968a\u9700\u8981\u8de8\u7bc0\u9ede\u8a13\u7df4\uff0c\u591a\u6578\u5718\u968a\u5247\u5f9e\u4e8b\u63a8\u8ad6\u670d\u52d9\u3001\u958b\u767c\u6e2c\u8a66\u6216\u5c0f\u898f\u6a21\u8a13\u7df4\u3002\u8207\u5176\u70ba\u5927\u578b\u8a13\u7df4\u53e6\u5efa\u5c08\u5c6c\u53e2\u96c6\uff0c\u4e0d\u5982\u8b93 Slurm \u53e2\u96c6\u8207\u5176\u4ed6 AI \u5de5\u4f5c\u8ca0\u8f09\u5171\u7528\u540c\u4e00\u8cc7\u6e90\u6c60\uff0c\u8a13\u7df4\u5b8c\u6210\u5f8c\u5373\u53ef\u5c07\u8cc7\u6e90\u91cb\u653e\u7d66\u5176\u4ed6\u7528\u9014\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u60c5\u5883\u56db\uff1a\u5c0d\u8a13\u7df4\u4e2d\u65b7\u6210\u672c\u9ad8\u5ea6\u654f\u611f\u7684\u5de5\u4f5c\u8ca0\u8f09<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">GPU \u6bcf\u5c0f\u6642\u6210\u672c\u8d8a\u9ad8\u3001\u4f5c\u696d\u57f7\u884c\u6642\u9593\u8d8a\u9577\uff0c\u55ae\u6b21\u4e2d\u65b7\u9020\u6210\u7684\u640d\u5931\u5c31\u8d8a\u5927\u3002\u82e5\u5718\u968a\u66fe\u56e0\u55ae\u4e00 GPU \u6545\u969c\u800c\u640d\u5931\u6578\u5c0f\u6642\u751a\u81f3\u6578\u5929\u7684\u9032\u5ea6\uff0cCheckpoint \u642d\u914d\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\u6240\u5e36\u4f86\u7684\u6548\u76ca\u6700\u70ba\u76f4\u63a5\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u9069\u7528\u786c\u9ad4\u8207\u7db2\u8def\uff1a\u4ec0\u9ebc\u6a23\u7684\u74b0\u5883\u5c0e\u5165\u6548\u76ca\u6700\u5927\uff1f<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">GPU \u4f3a\u670d\u5668<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u591a\u5361 GPU \u4f3a\u670d\u5668\u3001\u591a\u53f0\u7bc0\u9ede<\/strong>\uff1a\u5178\u578b\u914d\u7f6e\u662f\u6bcf\u53f0 8 \u5f35 GPU \u7684\u4f3a\u670d\u5668\uff08\u4f8b\u5982 8 \u5361 H200\uff09\uff0c\u4ee5\u591a\u53f0\u7d44\u6210\u8a13\u7df4\u53e2\u96c6\u3002\u7bc0\u9ede\u6578\u8d8a\u591a\uff0c\u6392\u7a0b\u8207\u6545\u969c\u6062\u5fa9\u7684\u50f9\u503c\u8d8a\u9ad8\u3002<\/li>\n\n\n\n<li><strong>\u9810\u7559\u7a7a\u9592\u7bc0\u9ede<\/strong>\uff1a\u82e5\u8981\u555f\u7528\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\u529f\u80fd\uff0c\u9700\u898f\u5283\u4e00\u5b9a\u6578\u91cf\u7684\u5099\u7528\u7a7a\u9592\u7bc0\u9ede\uff0c\u4f5c\u70ba\u6545\u969c\u6642\u7684\u66ff\u88dc\u4f86\u6e90\u3002<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\">\u9ad8\u901f\u7db2\u8def<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u8de8\u7bc0\u9ede\u8a13\u7df4\u6bcf\u4e00\u6b65\u90fd\u8981\u4ea4\u63db\u5927\u91cf\u8cc7\u6599\uff0c\u7db2\u8def\u5f80\u5f80\u662f\u6c7a\u5b9a\u8a13\u7df4\u6548\u7387\u7684\u95dc\u9375\u3002<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>InfiniBand \u6216 RoCE<\/strong>\uff1a\u5efa\u8b70\u7bc0\u9ede\u4e4b\u9593\u63a1\u7528 InfiniBand \u6216\u652f\u63f4 RDMA \u7684 RoCE \u9ad8\u901f\u7db2\u8def\u3002AI-Stack \u5728\u5efa\u7acb\u53e2\u96c6\u6642\uff0c\u5373\u53ef\u555f\u7528\u9ad8\u6027\u80fd\u7db2\u8def\u3002<\/li>\n\n\n\n<li><strong>\u4ee5\u5be6\u6e2c\u70ba\u6e96<\/strong>\uff1a\u7db2\u8def\u898f\u683c\u4e0d\u7b49\u65bc\u5be6\u969b\u6548\u80fd\u3002\u53e2\u96c6\u5efa\u7acb\u5f8c\uff0c\u53ef\u5148\u4ee5 NCCL Benchmark \u5be6\u6e2c\u8de8\u7bc0\u9ede\u901a\u8a0a\u983b\u5bec\uff0c\u78ba\u8a8d\u9054\u6a19\u518d\u958b\u59cb\u6b63\u5f0f\u8a13\u7df4\u3002<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\">\u5132\u5b58<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u5171\u4eab\u5132\u5b58<\/strong>\uff1a\u5efa\u8b70\u4f7f\u7528 GPFS\u3001HDFS \u7b49\u5171\u4eab\u5132\u5b58\uff0c\u4f9b Jail\uff08Slurm \u4f7f\u7528\u8005\u74b0\u5883\uff09\u3001Checkpoint\u3001\u8a13\u7df4\u8cc7\u6599\u8207\u5de5\u4f5c\u5340\u639b\u8f09\u3002<\/li>\n\n\n\n<li><strong>Checkpoint \u5beb\u5165\u6548\u80fd<\/strong>\uff1a\u5927\u578b\u6a21\u578b\u7684 Checkpoint \u53ef\u9054\u6578\u767e GB \u4ee5\u4e0a\uff0c\u5132\u5b58\u7684\u5beb\u5165\u901f\u5ea6\u6703\u76f4\u63a5\u5f71\u97ff\u5b58\u6a94\u983b\u7387\u8207\u8a13\u7df4\u6548\u7387\u3002<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th><strong>\u8cc7\u6e90<\/strong><\/th><th><strong>\u5efa\u8b70\u914d\u7f6e<\/strong><\/th><th><strong>\u70ba\u4ec0\u9ebc\u91cd\u8981<\/strong><\/th><\/tr><\/thead><tbody><tr><td>GPU \u7bc0\u9ede<\/td><td>\u591a\u53f0\u591a\u5361\u4f3a\u670d\u5668\uff08\u4f8b\u5982\u6bcf\u53f0 8 \u5361\uff09\uff0cNVIDIA \u6216 AMD<\/td><td>\u8de8\u7bc0\u9ede\u8a13\u7df4\u7684\u7b97\u529b\u57fa\u790e<\/td><\/tr><tr><td>\u5099\u7528\u7bc0\u9ede<\/td><td>\u4f9d\u53e2\u96c6\u898f\u6a21\u9810\u7559\u7a7a\u9592\u7bc0\u9ede<\/td><td>\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\u7684\u66ff\u88dc\u4f86\u6e90<\/td><\/tr><tr><td>\u7db2\u8def<\/td><td>InfiniBand \u6216 RoCE<\/td><td>\u6c7a\u5b9a\u7bc0\u9ede\u9593\u7684\u901a\u8a0a\u6548\u7387<\/td><\/tr><tr><td>\u5132\u5b58<\/td><td>GPFS\u3001HDFS \u7b49\u5171\u4eab\u5132\u5b58<\/td><td>Jail \u74b0\u5883\u3001Checkpoint \u8207\u8a13\u7df4\u8cc7\u6599\u5171\u7528<\/td><\/tr><\/tbody><\/table><figcaption class=\"wp-element-caption\">\u8868 2\u3000\u5c0e\u5165 Slurm on K8s \u7684\u5efa\u8b70\u786c\u9ad4\u914d\u7f6e<\/figcaption><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u7d50\u8a9e\uff1a\u5c0e\u5165 Slurm on K8s \u524d\u7684\u4e94\u9805\u8a55\u4f30\u91cd\u9ede<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u672c\u6587\u4ecb\u7d39\u7684\u67b6\u69cb\u4e2d\uff0cSlurm \u8ca0\u8cac\u8cc7\u6e90\u6392\u7a0b\uff0cKubernetes \u8ca0\u8cac\u7dad\u904b\u7ba1\u7406\uff0cCheckpoint \u8207\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\u5247\u5171\u540c\u63d0\u5347\u6545\u969c\u97cc\u6027\u3002\u56e0\u6b64\uff0c\u4f01\u696d\u8a55\u4f30\u8de8\u7bc0\u9ede\u8a13\u7df4\u5e73\u53f0\u6642\uff0c\u8207\u5176\u554f\u300c\u8a72\u9078\u64c7\u54ea\u4e00\u5957\u6392\u7a0b\u7cfb\u7d71\u300d\uff0c\u4e0d\u5982\u601d\u8003\u66f4\u6839\u672c\u7684\u554f\u984c\uff1a<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u73fe\u6709\u7684\u4f7f\u7528\u8005\u7fd2\u6163\u3001GPU \u898f\u6a21\u8207\u53ef\u63a5\u53d7\u7684\u9032\u5ea6\u640d\u5931\u4e0b\uff0c\u4f01\u696d\u9700\u8981\u4ec0\u9ebc\u6a23\u7684\u6392\u7a0b\u8207\u7dad\u904b\u67b6\u69cb\uff1f\u53c8\u9858\u610f\u70ba\u6545\u969c\u6062\u5fa9\u9810\u7559\u591a\u5c11\u8cc7\u6e90\uff1f<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u5c0e\u5165\u6216\u8abf\u6574\u67b6\u69cb\u4e4b\u524d\uff0c\u5efa\u8b70\u5148\u5f9e\u4ee5\u4e0b\u4e94\u500b\u9762\u5411\u76e4\u9ede\u73fe\u6cc1\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>\u4f7f\u7528\u8005\u7fd2\u6163<\/strong>\uff1a\u7d44\u7e54\u5167\u662f\u5426\u5df2\u6709 Slurm \u4f7f\u7528\u8005\u8207\u65e2\u6709\u4f5c\u696d\u8173\u672c\uff1f\u9019\u5c07\u6c7a\u5b9a\u4fdd\u7559 Slurm \u7684\u50f9\u503c\u3002<\/li>\n\n\n\n<li><strong>\u8cc7\u6e90\u5171\u7528<\/strong>\uff1a\u8a13\u7df4\u3001\u63a8\u8ad6\u8207\u958b\u767c\u662f\u5426\u9700\u8981\u5171\u7528\u540c\u4e00\u500b GPU \u8cc7\u6e90\u6c60\uff1f\u6216\u8005\u7dad\u6301\u5c08\u5c6c\u53e2\u96c6\u66f4\u70ba\u5408\u9069\uff1f<\/li>\n\n\n\n<li><strong>\u7db2\u8def\u9a57\u8b49<\/strong>\uff1a\u8de8\u7bc0\u9ede\u7db2\u8def\u662f\u5426\u5df2\u900f\u904e NCCL Benchmark \u7b49\u5de5\u5177\u5be6\u6e2c\uff0c\u800c\u975e\u50c5\u53c3\u8003\u898f\u683c\u8868\uff1f<\/li>\n\n\n\n<li><strong>\u5b58\u6a94\u7b56\u7565<\/strong>\uff1aCheckpoint \u7684\u5b58\u6a94\u9593\u9694\u8207\u5132\u5b58\u6548\u80fd\uff0c\u662f\u5426\u7b26\u5408\u53ef\u63a5\u53d7\u7684\u9032\u5ea6\u640d\u5931\uff1f\u662f\u5426\u66fe\u5be6\u969b\u6f14\u7df4\u5f9e Checkpoint \u6062\u5fa9\uff1f<\/li>\n\n\n\n<li><strong>\u5099\u63f4\u6210\u672c<\/strong>\uff1a\u55ae\u6b21\u6545\u969c\u6703\u9020\u6210\u591a\u5c11\u640d\u5931\uff1f\u503c\u5f97\u9810\u7559\u591a\u5c11\u9592\u7f6e\u7bc0\u9ede\uff1f<\/li>\n<\/ol>\n\n\n\n<p class=\"wp-block-paragraph\">\u6280\u8853\u6703\u6301\u7e8c\u6f14\u9032\uff0c\u53e2\u96c6\u4e5f\u6703\u4e0d\u65b7\u64f4\u5145\uff1b\u4f46\u53ea\u8981 GPU \u4ecd\u53ef\u80fd\u6545\u969c\u3001\u8cc7\u6e90\u4ecd\u9700\u5206\u914d\uff0c\u9019\u4e94\u500b\u9762\u5411\u5c31\u503c\u5f97\u4f01\u696d\u5b9a\u671f\u6aa2\u8996\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u5e38\u898b\u554f\u984c\uff08FAQ\uff09<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Q\uff1aSlurm on K8s \u548c\u76f4\u63a5\u4f7f\u7528 Kubernetes \u6709\u4ec0\u9ebc\u4e0d\u540c\uff1f<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Kubernetes \u7684\u9810\u8a2d\u6392\u7a0b\u5668\u9010\u4e00\u70ba Pod \u5206\u914d\u7bc0\u9ede\uff0c\u7528\u65bc\u5206\u6563\u5f0f\u8a13\u7df4\u6642\u53ef\u80fd\u767c\u751f\u8cc7\u6e90\u6b7b\u9396\uff0c\u4e5f\u7121\u6cd5\u6cbf\u7528 Slurm \u6307\u4ee4\u8207\u65e2\u6709\u4f5c\u696d\u8173\u672c\u3002Slurm on K8s \u7531 Slurm \u8ca0\u8cac\u4f5c\u696d\u6392\u7a0b\uff0c\u4fdd\u7559 HPC \u4f7f\u7528\u8005\u719f\u6089\u7684\u64cd\u4f5c\u65b9\u5f0f\uff0c\u4e26\u7531 Kubernetes \u8ca0\u8cac\u5bb9\u5668\u5316\u90e8\u7f72\u8207\u7dad\u904b\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Q\uff1a\u4ec0\u9ebc\u662f\u6574\u7d44\u6392\u7a0b\uff08Gang Scheduling\uff09\uff1f<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u6574\u7d44\u6392\u7a0b\u5c07\u540c\u4e00\u4f5c\u696d\u7684\u6240\u6709 Pod \u8996\u70ba\u6574\u9ad4\uff0c\u53ea\u6709\u5728\u6240\u9700\u8cc7\u6e90\u5168\u6578\u5c31\u7dd2\u6642\u624d\u540c\u6642\u555f\u52d5\uff0c\u53ef\u907f\u514d\u591a\u500b\u5206\u6563\u5f0f\u4f5c\u696d\u5404\u81ea\u5360\u7528\u90e8\u5206 GPU \u800c\u4e92\u76f8\u7b49\u5f85\u7684\u6b7b\u9396\u3002Kubernetes \u539f\u751f\u7684 Gang \u6392\u7a0b\u5df2\u65bc v1.37 \u9032\u5165 Beta\uff0c\u4f46\u9810\u8a2d\u4e26\u672a\u555f\u7528\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Q\uff1a\u5206\u6563\u5f0f\u8a13\u7df4\u9047\u5230 GPU \u6545\u969c\uff0c\u9700\u8981\u591a\u4e45\u624d\u80fd\u6062\u5fa9\uff1f<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u82e5\u50c5\u4f9d\u9760 Checkpoint\uff0c\u6545\u969c\u5f8c\u6574\u500b\u4f5c\u696d\u9808\u91cd\u65b0\u6392\u968a\u8207\u521d\u59cb\u5316\uff0c\u5e38\u898b\u9700\u8981\u6578\u5c0f\u6642\uff1b\u642d\u914d AI-Stack \u7684\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\uff0c\u4ee5\u6eab\u5099\u7bc0\u9ede\u81ea\u52d5\u66ff\u63db\u6545\u969c\u7bc0\u9ede\uff0c\u6062\u5fa9\u6642\u9593\u53ef\u7e2e\u77ed\u81f3\u5206\u9418\u7d1a\u3002\u5169\u7a2e\u65b9\u5f0f\u90fd\u9700\u91cd\u65b0\u8a08\u7b97\u6700\u8fd1\u4e00\u6b21\u5b58\u6a94\u5f8c\u7684\u9032\u5ea6\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Q\uff1aCheckpoint \u7684\u5b58\u6a94\u983b\u7387\u61c9\u8a72\u5982\u4f55\u8a2d\u5b9a\uff1f<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u6700\u4f73\u5b58\u6a94\u9593\u9694\u53d6\u6c7a\u65bc\u5beb\u5165 Checkpoint \u7684\u6210\u672c\u8207\u6545\u969c\u7387\u3002\u5b58\u6a94\u904e\u65bc\u983b\u7e41\u6703\u58d3\u7e2e\u8a13\u7df4\u6642\u9593\uff0c\u9593\u9694\u904e\u9577\u5247\u6bcf\u6b21\u6545\u969c\u640d\u5931\u7684\u9032\u5ea6\u8f03\u591a\u3002\u4e00\u822c\u800c\u8a00\uff0c\u7bc0\u9ede\u8d8a\u591a\u3001\u6545\u969c\u8d8a\u983b\u7e41\uff0c\u5b58\u6a94\u9593\u9694\u5c31\u61c9\u8d8a\u77ed\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Q\uff1a\u5c0e\u5165 AI-Satck Slurm on K8s \u9700\u8981\u54ea\u4e9b\u786c\u9ad4\uff1f<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u5efa\u8b70\u914d\u7f6e\u5305\u62ec\u591a\u53f0\u591a\u5361 GPU \u4f3a\u670d\u5668\uff08NVIDIA \u6216 AMD\uff09\u3001InfiniBand \u6216 RoCE \u9ad8\u901f\u7db2\u8def\uff0c\u4ee5\u53ca GPFS\u3001HDFS \u7b49\u5171\u4eab\u5132\u5b58\u3002\u82e5\u8981\u555f\u7528\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\uff0c\u53e6\u9700\u9810\u7559\u9592\u7f6e\u7bc0\u9ede\u4f5c\u70ba\u66ff\u88dc\u4f86\u6e90\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n","protected":false},"excerpt":{"rendered":"<p>AI \u8a13\u7df4\u5168\u9762\u8d70\u5411\u8de8\u7bc0\u9ede\u3001\u5927\u898f\u6a21\u5316\uff0c\u4f01\u696d\u9762\u81e8\u8cc7\u6e90\u6392\u7a0b\u3001\u7dad\u904b\u7ba1\u7406\u8207 GPU \u6545\u969c\u4e09\u5927\u6311\u6230\u3002\u672c\u6587\u89e3\u6790\u7522\u696d\u8da8\u52e2\u8207\u73fe\u6709\u505a\u6cd5\u7684\u4fb7\u9650\uff0c\u8aaa\u660e Slurm on K8s \u67b6\u69cb\uff0c\u4ee5\u53ca INFINITIX AI-Stack \u5982\u4f55\u4ee5\u8a13\u7df4\u4e2d\u65b7\u81ea\u52d5\u6062\u5fa9\uff0c\u5c07\u6545\u969c\u6062\u5fa9\u6642\u9593\u7e2e\u77ed\u81f3\u5206\u9418\u7d1a\u3002<\/p>\n","protected":false},"author":253372381,"featured_media":15758,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_feature_clip_id":0,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_post_was_ever_published":false},"categories":[96987561,96987582],"tags":[96989084,96987839,96987615],"class_list":["post-15749","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-technical-support","category-product-features","tag-slurm","tag-hpc","tag-ai"],"blocksy_meta":[],"acf":[],"jetpack_shortlink":"https:\/\/wp.me\/ph344V-461","jetpack_sharing_enabled":true,"jetpack_featured_media_url":"https:\/\/i0.wp.com\/ai-stack.ai\/wp-content\/uploads\/2026\/11\/addd4aed.png?fit=1920%2C1080&quality=100&ct=202603031250&ssl=1","_links":{"self":[{"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/posts\/15749","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/users\/253372381"}],"replies":[{"embeddable":true,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/comments?post=15749"}],"version-history":[{"count":1,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/posts\/15749\/revisions"}],"predecessor-version":[{"id":15770,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/posts\/15749\/revisions\/15770"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/media\/15758"}],"wp:attachment":[{"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/media?parent=15749"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/categories?post=15749"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ai-stack.ai\/zh-hant\/wp-json\/wp\/v2\/tags?post=15749"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}