{"id":598,"date":"2025-03-03T01:49:56","date_gmt":"2025-03-03T01:49:56","guid":{"rendered":"https:\/\/pinglex.ai\/?p=598"},"modified":"2025-03-03T01:55:29","modified_gmt":"2025-03-03T01:55:29","slug":"%e8%aa%9e%e8%a8%80%e5%bb%ba%e6%a8%a1%e8%88%87%e6%8a%80%e8%a1%93%e7%99%bc%e5%b1%95%e5%85%a8%e6%99%af","status":"publish","type":"post","link":"https:\/\/pinglex.ai\/?p=598","title":{"rendered":"\u8a9e\u8a00\u5efa\u6a21\u8207\u6280\u8853\u767c\u5c55\u5168\u666f"},"content":{"rendered":"\n

<\/p>\n\n\n\n

<\/a>\u8a9e\u8a00\u5efa\u6a21\u662f\u81ea\u7136\u8a9e\u8a00\u8655\u7406\uff08NLP\uff09\u7684\u6838\u5fc3\u6280\u8853\uff0c\u65e8\u5728\u901a\u904e\u7d71\u8a08\u6216\u795e\u7d93\u7db2\u7d61\u6a21\u578b\u9810\u6e2c\u6587\u672c\u5e8f\u5217\u7684\u6a5f\u7387\u5206\u4f48\u3002\u5176\u61c9\u7528\u6db5\u84cb\u8a9e\u97f3\u8b58\u5225\u3001\u6a5f\u5668\u7ffb\u8b6f\u3001\u6587\u672c\u751f\u6210\u7b49\u9818\u57df\u3002\u73fe\u4ee3\u8a9e\u8a00\u6a21\u578b\u7d93\u6b77\u4e86\u5f9e\u57fa\u65bc\u898f\u5247\u7684\u6587\u6cd5\u6a21\u578b\u5230\u7d71\u8a08\u6a21\u578b\uff0c\u518d\u5230\u795e\u7d93\u7db2\u7d61\u6a21\u578b\u7684\u6f14\u8b8a\uff0c\u5176\u4e2d\u5927\u898f\u6a21\u9810\u8a13\u7df4\u8a9e\u8a00\u6a21\u578b\uff08\u5982GPT\u3001BERT\uff09\u6210\u70ba\u4e3b\u6d41\u3002<\/p>\n\n\n\n

\u9810\u8a13\u7df4\u7684\u6311\u6230<\/strong><\/p>\n\n\n\n

\u8cc7\u6e90\u53ef\u64f4\u5c55\u6027\uff1a<\/p>\n\n\n\n

\u96a8\u8457\u6a21\u578b\u53c3\u6578\u91cf\u7a81\u7834\u5343\u5104\u7d1a\uff0c\u9810\u8a13\u7df4\u9700\u6d88\u8017\u6578\u5341\u842c\u8a08\u7b97\u5c0f\u6642\u8207\u6578\u767e\u842c\u5143\u6210\u672c\uff0c\u4e14\u80fd\u6e90\u6d88\u8017\u9a5a\u4eba\u3002\u4f8b\u5982\uff0c1750\u5104\u53c3\u6578\u7684GPT-3\u8a13\u7df4\u9700\u6578\u5343\u500bGPU\u6578\u6708\u6642\u9593\uff0c\u51f8\u986f\u5206\u4f48\u5f0f\u8a08\u7b97\u8207\u9ad8\u6548\u7b97\u6cd5\u7684\u91cd\u8981\u6027\u3002<\/p>\n\n\n\n

\u6578\u64da\u504f\u5dee\u8207\u516c\u5e73\u6027\uff1a<\/p>\n\n\n\n

\u9810\u8a13\u7df4\u6578\u64da\u7684\u6f5b\u5728\u504f\u5dee\uff08\u5982\u6027\u5225\u3001\u6587\u5316\u504f\u898b\uff09\u6703\u76f4\u63a5\u5f71\u97ff\u6a21\u578b\u8f38\u51fa\u3002\u7814\u7a76\u8868\u660e\uff0c\u672a\u7d93\u6821\u6b63\u7684\u6578\u64da\u53ef\u80fd\u5c0e\u81f4\u6a21\u578b\u751f\u6210\u6b67\u8996\u6027\u5167\u5bb9\u3002\u89e3\u6c7a\u65b9\u6848\u5305\u62ec\u6578\u64da\u6e05\u6d17\u3001\u53bb\u504f\u7f6e\u7b97\u6cd5\u8a2d\u8a08\uff0c\u4ee5\u53ca\u5f15\u5165\u516c\u5e73\u6027\u8a55\u4f30\u6307\u6a19\u3002<\/p>\n\n\n\n

\u6a21\u578b\u53ef\u89e3\u91cb\u6027\uff1a<\/p>\n\n\n\n

\u5927\u6a21\u578b\u7684\u9ed1\u7bb1\u7279\u6027\u4f7f\u5f97\u6c7a\u7b56\u904e\u7a0b\u96e3\u4ee5\u8ffd\u6eaf\u3002\u7814\u7a76\u805a\u7126\u65bc\u6ce8\u610f\u529b\u6a5f\u5236\u53ef\u8996\u5316\u3001\u7279\u5fb5\u6b78\u56e0\u5206\u6790\u7b49\u65b9\u6cd5\uff0c\u4ee5\u63d0\u5347\u900f\u660e\u5ea6\u548c\u53ef\u4fe1\u5ea6\u3002<\/p>\n\n\n\n

\u91cf\u5316\u6280\u8853\u8207\u6a21\u578b\u58d3\u7e2e<\/strong><\/p>\n\n\n\n

\u91cf\u5316\u6280\u8853\u901a\u904e\u964d\u4f4e\u53c3\u6578\u7cbe\u5ea6\uff08\u598232\u4f4d\u6d6e\u9ede\u8f49\u70ba8\u4f4d\u6574\u6578\uff09\u4f86\u7e2e\u6e1b\u6a21\u578b\u9ad4\u7a4d\u4e26\u52a0\u901f\u63a8\u7406\uff0c\u5206\u70ba\u5169\u5927\u985e\uff1a<\/p>\n\n\n\n

\u8a13\u7df4\u5f8c\u91cf\u5316\uff08PTQ\uff09\uff1a\u76f4\u63a5\u5c0d\u9810\u8a13\u7df4\u6a21\u578b\u9032\u884c\u4f4e\u7cbe\u5ea6\u8f49\u63db\uff0c\u7121\u9700\u984d\u5916\u8a13\u7df4\uff0c\u9069\u7528\u65bc\u5feb\u901f\u90e8\u7f72\u3002\u5178\u578b\u65b9\u6cd5\u5305\u62ecGPTQ\uff08\u57fa\u65bc\u68af\u5ea6\u66f4\u65b0\u7684\u91cf\u5316\uff09\u548cGGML\uff08\u6df7\u5408\u7cbe\u5ea6\u91cf\u5316\uff09\u3002<\/p>\n\n\n\n

\u91cf\u5316\u611f\u77e5\u8a13\u7df4\uff08QAT\uff09\uff1a\u5728\u5fae\u8abf\u968e\u6bb5\u5f15\u5165\u91cf\u5316\u8aa4\u5dee\u6a21\u64ec\uff0c\u63d0\u5347\u6a21\u578b\u5c0d\u4f4e\u7cbe\u5ea6\u904b\u7b97\u7684\u9069\u61c9\u6027\uff0c\u4f8b\u5982QLoRA\u7d50\u54084\u4f4d\u91cf\u5316\u8207LoRA\u5fae\u8abf\uff0c\u53ef\u5728\u6d88\u8cbb\u7d1aGPU\u4e0a\u904b\u884c650\u5104\u53c3\u6578\u6a21\u578b\u3002<\/p>\n\n\n\n

\u91cf\u5316\u6280\u8853\u7684\u6548\u80fd\u53d6\u6c7a\u65bc\u6b0a\u91cd\u5206\u4f48\u7684\u654f\u611f\u5ea6\uff0c\u6975\u7aef\u4f4e\u7cbe\u5ea6\uff08\u59822-bit\uff09\u53ef\u80fd\u5c0e\u81f4\u6027\u80fd\u986f\u8457\u4e0b\u964d\uff0c\u9700\u642d\u914d\u7a00\u758f\u5316\u6216\u6df7\u5408\u91cf\u5316\u7b56\u7565\u3002<\/p>\n\n\n\n

\u5206\u4f48\u5f0f\u8a13\u7df4\u65b9\u6cd5<\/strong><\/p>\n\n\n\n

\u5206\u4f48\u5f0f\u8a13\u7df4\u900f\u904e\u4e26\u884c\u5316\u89e3\u6c7a\u55ae\u4e00\u8a2d\u5099\u7684\u8a08\u7b97\u8207\u5167\u5b58\u9650\u5236\uff0c\u4e3b\u8981\u5206\u70ba\u4e09\u7a2e\u7b56\u7565\uff1a<\/p>\n\n\n\n

\u6578\u64da\u4e26\u884c\uff1a\u5c07\u6578\u64da\u62c6\u5206\u81f3\u591a\u8a2d\u5099\uff0c\u5404\u8a2d\u5099\u6301\u6709\u5b8c\u6574\u6a21\u578b\u526f\u672c\u4e26\u540c\u6b65\u68af\u5ea6\uff0c\u9069\u7528\u65bc\u53c3\u6578\u91cf\u9069\u4e2d\u7684\u6a21\u578b\u3002<\/p>\n\n\n\n

\u6a21\u578b\u4e26\u884c\uff1a\u5c07\u6a21\u578b\u5c64\u62c6\u5206\u81f3\u4e0d\u540c\u8a2d\u5099\uff0c\u5e38\u7528\u65bc\u8d85\u5927\u578b\u6a21\u578b\uff08\u5982Megatron-Turing\u7684\u842c\u5104\u53c3\u6578\u67b6\u69cb\uff09\uff0c\u9700\u89e3\u6c7a\u8de8\u8a2d\u5099\u901a\u4fe1\u958b\u92b7\u554f\u984c\u3002<\/p>\n\n\n\n

\u6d41\u6c34\u7dda\u4e26\u884c\uff1a\u5c07\u6a21\u578b\u6309\u5c64\u5206\u6bb5\u4e26\u4e32\u806f\u57f7\u884c\uff0c\u7d50\u5408\u5fae\u6279\u6b21\uff08Micro-batching\uff09\u96b1\u85cf\u901a\u4fe1\u5ef6\u9072\uff0c\u5178\u578b\u6846\u67b6\u5305\u62ecPyTorch\u7684PipeDream\u3002<\/p>\n\n\n\n

\u5206\u4f48\u5f0f\u8a13\u7df4\u7684\u5be6\u4f5c\u9700\u642d\u914d\u96c6\u5408\u901a\u4fe1\u5eab\uff08\u5982NCCL\uff09\u3001\u53c3\u6578\u4f3a\u670d\u5668\u67b6\u69cb\uff0c\u4ee5\u53ca\u5f48\u6027\u5bb9\u932f\u6a5f\u5236\u4ee5\u61c9\u5c0d\u7bc0\u9ede\u6545\u969c\u3002<\/p>\n\n\n\n

\u5927\u8a9e\u8a00\u6a21\u578b\u5fae\u8abf\u6280\u8853<\/strong><\/p>\n\n\n\n

\u53c3\u6578\u9ad8\u6548\u5fae\u8abf\uff08PEFT\uff09<\/p>\n\n\n\n

PEFT\u6280\u8853\u50c5\u5fae\u8abf\u5c11\u91cf\u53c3\u6578\u5373\u53ef\u9069\u61c9\u4e0b\u6e38\u4efb\u52d9\uff0c\u986f\u8457\u964d\u4f4e\u8a08\u7b97\u6210\u672c\uff1a<\/p>\n\n\n\n

LoRA\uff08Low-Rank Adaptation\uff09\uff1a\u5728Transformer\u5c64\u6ce8\u5165\u4f4e\u79e9\u77e9\u9663\uff0c\u901a\u904e\u77e9\u9663\u5206\u89e3\u6a21\u64ec\u53c3\u6578\u66f4\u65b0\uff0c\u4fdd\u7559\u539f\u59cb\u6b0a\u91cd\u3002<\/p>\n\n\n\n

Prefix Tuning\uff1a\u5728\u8f38\u5165\u7aef\u6dfb\u52a0\u53ef\u5b78\u7fd2\u7684\u63d0\u793a\u5411\u91cf\uff08Prefix\uff09\uff0c\u5f15\u5c0e\u6a21\u578b\u751f\u6210\u4efb\u52d9\u76f8\u95dc\u8f38\u51fa\uff0c\u9069\u7528\u65bc\u751f\u6210\u4efb\u52d9\u3002<\/p>\n\n\n\n

Adapter\uff1a\u5728FFN\u5c64\u9593\u63d2\u5165\u5c0f\u578b\u795e\u7d93\u7db2\u7d61\u6a21\u584a\uff0c\u50c5\u8a13\u7df4\u9069\u914d\u5668\u53c3\u6578\uff0c\u4fdd\u6301\u4e3b\u5e79\u7db2\u7d61\u51cd\u7d50\u3002<\/p>\n\n\n\n

\u5168\u53c3\u6578\u5fae\u8abf\u8207\u6307\u4ee4\u5fae\u8abf<\/strong><\/p>\n\n\n\n

\u5168\u53c3\u6578\u5fae\u8abf\u9700\u66f4\u65b0\u6240\u6709\u53c3\u6578\uff0c\u96d6\u6027\u80fd\u6700\u4f73\u4f46\u6210\u672c\u9ad8\u6602\uff0c\u5e38\u7528\u65bc\u9818\u57df\u9069\u914d\uff08\u5982\u6cd5\u5f8b\u3001\u91ab\u7642\u6587\u672c\uff09\u3002\u6307\u4ee4\u5fae\u8abf\uff08Instruction Tuning\uff09\u5247\u4f7f\u7528\u4efb\u52d9\u6307\u4ee4\u8207\u793a\u4f8b\u6578\u64da\uff0c\u589e\u5f37\u6a21\u578b\u9075\u5faa\u8907\u96dc\u6307\u4ee4\u7684\u80fd\u529b\u3002<\/p>\n\n\n\n

\u6a21\u578b\u58d3\u7e2e\u8207\u90e8\u7f72\u512a\u5316<\/strong><\/p>\n\n\n\n

\u526a\u679d\uff08Pruning\uff09\uff1a\u79fb\u9664\u5197\u4f59\u6b0a\u91cd\u6216\u795e\u7d93\u5143\uff0c\u5206\u70ba\u7d50\u69cb\u5316\u526a\u679d\uff08\u79fb\u9664\u6574\u5c64\u6216\u901a\u9053\uff09\u8207\u975e\u7d50\u69cb\u5316\u526a\u679d\uff08\u79fb\u9664\u55ae\u500b\u6b0a\u91cd\uff09\uff0c\u9700\u642d\u914d\u91cd\u8a13\u7df4\u6062\u5fa9\u6027\u80fd\u3002<\/p>\n\n\n\n

\u77e5\u8b58\u84b8\u993e\uff1a\u901a\u904e\u6559\u5e2b-\u5b78\u751f\u67b6\u69cb\uff0c\u5c07\u5927\u6a21\u578b\u77e5\u8b58\u9077\u79fb\u81f3\u8f15\u91cf\u6a21\u578b\uff0c\u4f8b\u5982TinyBERT\u5728\u50c510%\u53c3\u6578\u91cf\u4e0b\u4fdd\u630170%\u4ee5\u4e0a\u6027\u80fd\u3002<\/p>\n\n\n\n

\u9019\u4e9b\u6280\u8853\u7684\u7d44\u5408\u61c9\u7528\uff08\u5982\u91cf\u5316+\u526a\u679d\uff09\u53ef\u5be6\u73fe\u6a21\u578b\u9ad4\u7a4d\u7e2e\u6e1b10\u500d\u4ee5\u4e0a\uff0c\u540c\u6642\u7dad\u630190%\u4ee5\u4e0a\u7684\u539f\u59cb\u6e96\u78ba\u7387\u3002<\/p>\n\n\n\n

Reference<\/strong><\/p>\n\n\n\n

[1] \u52a0\u7fa4\u94fe\u63a5: https:\/\/docs.qq.com\/doc\/DS3VGS0NFVHNRR0Ru<\/a>#<\/p>\n\n\n\n

[2] Transformer: https:\/\/arxiv.org\/abs\/1706.03762<\/a><\/p>\n\n\n\n

[3] BLOOM: https:\/\/huggingface.co\/docs\/transformers\/model_doc\/bloom<\/a><\/p>\n\n\n\n

[4] Parameter-Efficient Transfer Learning for NLP: https:\/\/arxiv.org\/abs\/1902.00751<\/a><\/p>\n\n\n\n

[5] Intrinsic Dimensionality Explains the Effectiveness of Language Model Fine-Tuning: https:\/\/arxiv.org\/abs\/2012.13255<\/a><\/p>\n\n\n\n

[6] Infused Adapter by Inhibiting and Amplifying Inner Activations: https:\/\/arxiv.org\/abs\/2205.05638<\/a><\/p>\n\n\n\n

[7] P-Tuning\u8bba\u6587: https:\/\/arxiv.org\/abs\/2103.10385<\/a><\/p>\n\n\n\n

[8] P-Tuning V-2: https:\/\/arxiv.org\/abs\/2110.07602<\/a><\/p>\n\n\n\n

[9] \u601d\u7ef4\u94fe\uff08CoT\uff09\u63d0\u793a: https:\/\/arxiv.org\/abs\/2201.11903<\/a><\/p>\n\n\n\n

[10] PAL: Program-aided Language Models: https:\/\/arxiv.org\/abs\/2211.10435<\/a><\/p>\n\n\n\n

[11] Distilling the Knowledge in a Neural Network: https:\/\/arxiv.org\/abs\/1503.02531<\/a><\/p>\n\n\n\n

\"\"<\/figure>\n","protected":false},"excerpt":{"rendered":"

\u8a9e\u8a00\u5efa\u6a21\u662f\u81ea\u7136\u8a9e\u8a00\u8655\u7406\uff08NLP\uff09\u7684\u6838\u5fc3\u6280\u8853\uff0c\u65e8\u5728\u901a\u904e\u7d71\u8a08\u6216\u795e\u7d93\u7db2\u7d61\u6a21\u578b\u9810\u6e2c\u6587\u672c\u5e8f\u5217\u7684\u6a5f\u7387\u5206\u4f48\u3002\u5176\u61c9\u7528\u6db5\u84cb\u8a9e\u97f3\u8b58 […]<\/p>\n","protected":false},"author":3,"featured_media":583,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"nf_dc_page":"","iawp_total_views":0,"footnotes":""},"categories":[47,27],"tags":[34,28,140,139,31,138,137,74],"class_list":["post-598","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ai","category-llm","tag-ai","tag-chatgpt","tag-gpu","tag-nlp","tag-31","tag-138","tag-137","tag-74"],"_links":{"self":[{"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/posts\/598","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/pinglex.ai\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=598"}],"version-history":[{"count":2,"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/posts\/598\/revisions"}],"predecessor-version":[{"id":601,"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/posts\/598\/revisions\/601"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/pinglex.ai\/index.php?rest_route=\/wp\/v2\/media\/583"}],"wp:attachment":[{"href":"https:\/\/pinglex.ai\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=598"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/pinglex.ai\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=598"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/pinglex.ai\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=598"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}