{"id":664,"date":"2025-09-16T14:44:32","date_gmt":"2025-09-16T06:44:32","guid":{"rendered":"https:\/\/www.vcoco.top\/?p=664"},"modified":"2025-09-16T14:44:33","modified_gmt":"2025-09-16T06:44:33","slug":"dpo%e5%be%ae%e8%b0%83","status":"publish","type":"post","link":"https:\/\/www.vcoco.top\/index.php\/2025\/09\/16\/dpo%e5%be%ae%e8%b0%83\/","title":{"rendered":"DPO\u5fae\u8c03"},"content":{"rendered":"\n<ul class=\"wp-block-list\">\n<li>https:\/\/arxiv.org\/pdf\/2305.18290<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\">DPO\u7684\u6838\u5fc3<\/h3>\n\n\n\n<p>DPO\u6838\u5fc3\u521b\u65b0\u5728\u4e8e\u5c06&#8221;\u8bed\u8a00\u6a21\u578b\u6697\u85cf\u7740\u5956\u52b1\u6a21\u578b&#8221;\u8fd9\u4e00\u7406\u8bba\u6d1e\u89c1\u8f6c\u5316\u4e3a\u5b9e\u7528\u7684\u8bad\u7ec3\u65b9\u6cd5\uff0c\u4f7f\u5f97\u6a21\u578b\u5bf9\u9f50\u8fc7\u7a0b\u53d8\u5f97\u66f4\u52a0\u7b80\u5355\u3001\u9ad8\u6548\u548c\u7a33\u5b9a\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"DPO%E5%A6%82%E4%BD%95%E8%A7%A3%E5%86%B3RLHF%E7%9A%84%E6%A0%B8%E5%BF%83%E9%97%AE%E9%A2%98\">DPO\u5982\u4f55\u89e3\u51b3RLHF\u7684\u6838\u5fc3\u95ee\u9898<\/h3>\n\n\n\n<p>DPO\uff08\u76f4\u63a5\u504f\u597d\u4f18\u5316\uff09\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u7b80\u5316\u548c\u89e3\u51b3RLHF\u7684\u95ee\u9898\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u6570\u5b66\u8f6c\u6362<\/strong>\uff1aDPO\u53d1\u73b0\u4e86\u4e00\u79cd\u5de7\u5999\u7684\u6570\u5b66\u53d8\u6362\uff0c\u5c06RLHF\u4e24\u9636\u6bb5\u8fc7\u7a0b\u8f6c\u5316\u4e3a\u5355\u4e00\u635f\u5931\u51fd\u6570<\/li>\n\n\n\n<li><strong>\u65e0\u9700\u663e\u5f0f\u5956\u52b1\u6a21\u578b<\/strong>\uff1a\u5229\u7528\u6700\u4f18\u7b56\u7565\u4e0e\u5956\u52b1\u7684\u6570\u5b66\u5173\u7cfb\uff0cDPO\u8bc1\u660e\u4e86\u53ef\u4ee5\u8df3\u8fc7\u5956\u52b1\u6a21\u578b\u8bad\u7ec3<\/li>\n\n\n\n<li><strong>\u76f4\u63a5\u4f7f\u7528\u504f\u597d\u6570\u636e<\/strong>\uff1aDPO\u76f4\u63a5\u4ece\u6210\u5bf9\u7684\u504f\u597d\u6570\u636e\u5b66\u4e60\uff0c\u907f\u514d\u4e86\u4e2d\u95f4\u5956\u52b1\u6a21\u578b<\/li>\n\n\n\n<li><strong>\u9690\u5f0fKL\u7ea6\u675f<\/strong>\uff1aDPO\u5728\u635f\u5931\u51fd\u6570\u4e2d\u81ea\u7136\u5305\u542b\u4e86KL\u6563\u5ea6\u7ea6\u675f\uff0c\u65e0\u9700\u989d\u5916\u5904\u7406<\/li>\n\n\n\n<li><strong>\u7b80\u5316\u5b9e\u73b0<\/strong>\uff1a\u8f6c\u5316\u4e3a\u6807\u51c6\u76d1\u7763\u5b66\u4e60\u5f62\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5e38\u89c4\u7684\u4f18\u5316\u5668\u548c\u8bad\u7ec3\u6d41\u7a0b<\/li>\n\n\n\n<li><strong>\u8bad\u7ec3\u7a33\u5b9a\u6027\u63d0\u5347<\/strong>\uff1a\u907f\u514d\u4e86RL\u8bad\u7ec3\u4e2d\u7684\u4e0d\u7a33\u5b9a\u6027\u95ee\u9898<\/li>\n\n\n\n<li><strong>\u8ba1\u7b97\u6548\u7387\u66f4\u9ad8<\/strong>\uff1a\u65e0\u9700\u591a\u6b21\u91c7\u6837\u548c\u8bc4\u4f30\uff0c\u5927\u5927\u964d\u4f4e\u4e86\u8ba1\u7b97\u6210\u672c<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\">1.\u4ec0\u4e48\u662fDPO<\/h3>\n\n\n\n<p>\u6838\u5fc3\u8981\u70b9<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u504f\u597d\u6392\u5e8f\uff08PreFerenct Ordering\uff09\uff1aDPO \u7684\u76ee\u6807\u662f\u901a\u8fc7\u4f18\u5316\u6a21\u578b\u751f\u6210\u7684\u6587\u672c\u4e0e\u4eba\u7c7b\u671f\u671b\u7684\u6587\u672c\u4e4b\u95f4\u7684\u504f\u597d\u6392\u5e8f\u6765\u8fdb\u884c\u8bad\u7ec3\u3002\u5047\u8bbe\u6709\u4e00\u7ec4\u6587\u672c(\u4f8b\u5982\u4e0a\u8fb9\u6240\u8bf4\u5df2\u77e5\u4f18\u8d28\u6587\u7ae0\u548c\u5dee\u6587\u7ae0)\uff0cDPO\u4f1a\u6839\u636e\u67d0\u4e9b\u6807\u51c6\uff08\u6bd4\u5982\u6587\u672c\u7684\u6d41\u7545\u6027\u3001\u76f8\u5173\u6027\u7b49\uff09\u5bf9\u8fd9\u4e9b\u6587\u672c\u8fdb\u884c\u6392\u5e8f\u3002\u7136\u540e\uff0c\u6a21\u578b\u901a\u8fc7\u5b66\u4e60\u5982\u4f55\u751f\u6210\u66f4\u7b26\u5408\u8fd9\u79cd\u6392\u5e8f\u7684\u6587\u672c\u6765\u8fdb\u884c\u4f18\u5316\u3002<\/li>\n\n\n\n<li>\u65e0\u9700\u5956\u52b1\u6a21\u578b\uff1a\u4e0eRLHF\u4e0d\u540c\uff0cDPO\u4e0d\u9700\u8981\u4e00\u4e2a\u590d\u6742\u7684\u5956\u52b1\u6a21\u578b\u3002RLHF\u4e2d\u9700\u8981\u901a\u8fc7\u5927\u91cf\u7684\u4eba\u5de5\u53cd\u9988\u6765\u8bad\u7ec3\u4e00\u4e2a\u5956\u52b1\u6a21\u578b\uff0c\u800cDPO\u901a\u8fc7\u76f4\u63a5\u5229\u7528\u6392\u5e8f\u6570\u636e\u6765\u4f18\u5316\u6a21\u578b\uff0c\u65e0\u9700\u590d\u6742\u7684\u5956\u52b1\u51fd\u6570\u3002<\/li>\n\n\n\n<li>\u4f18\u5316\u76ee\u6807\uff1aDPO\u7684\u4f18\u5316\u76ee\u6807\u662f\u4f7f\u6a21\u578b\u751f\u6210\u7684\u6587\u672c\u5c3d\u53ef\u80fd\u7b26\u5408\u7ed9\u5b9a\u7684\u504f\u597d\u6392\u5e8f\u3002\u4f8b\u5982\uff0c\u5982\u679c\u7ed9\u5b9a\u4e00\u4e2a\u6587\u672c\u96c6\u5408\uff0cDPO\u4f1a\u6839\u636e\u8fd9\u4e9b\u6587\u672c\u7684\u4f18\u52a3\u987a\u5e8f\u6765\u8bad\u7ec3\u6a21\u578b\uff0c\u4f7f\u5f97\u6a21\u578b\u751f\u6210\u7684\u6587\u672c\u4e0e\u504f\u597d\u7684\u987a\u5e8f\u66f4\u52a0\u4e00\u81f4\u3002<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\">2.\u6700\u4f18\u7b56\u7565\u7684\u6570\u5b66\u5f62\u5f0f<\/h3>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><div class='fancybox-wrapper lazyload-container-unload' data-fancybox='post-images' href='https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image.png'><img class=\"lazyload lazyload-style-1\" src=\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\"  loading=\"lazy\" decoding=\"async\" width=\"381\" height=\"106\" data-original=\"https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image.png\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAANSURBVBhXYzh8+PB\/AAffA0nNPuCLAAAAAElFTkSuQmCC\" alt=\"\" class=\"wp-image-668\" style=\"aspect-ratio:3.5943396226415096;width:381px;height:auto\"\/><\/div><\/figure><\/div>\n\n\n<p class=\"has-text-align-center\"><img decoding=\"async\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAWYAAABeCAIAAACfGik1AAAQAElEQVR4AexdD2wb13l\/iIex8LATXJSGM4iwITNKbc6ZTDtlSKQ2BcOiYlhUvbCsM4lOwFDJGKmJXC6uwjpW5Hq0HE81u0lhY9NEYkqoyzF1RLmaKUPVOQt05TzTmm3Ki3JKK1BAA7FoYAI1ckCE7bs7UiIlin8kHkWqj3g63r33ve997\/e9+937d9Qj\/4c\/GAGMAEYgawQeQfiDEcAIYASyRgBTRtZQYUGMAEYAIUwZuBVgBARHYC0VgCljLXkT1wUjIDgCmDIEhxgXgBFYSwhgylhL3sR1wQgIjgCmDMEhxgUsDwGcqzgRwJRRnH7BVmEEihQBTBlF6hhsFkagOBHAlFGcfsFWYQSKFAFMGUXqmOWZhXNhBIRGAFOG0Ahj\/RiBNYUApow15U5cGYyA0AhgyhAaYawfI7CmEMCUgdaUP3FlMAICI4ApQ2CAsXqMwNpCAFPG2vInrg1GQGAEMGUIDDBWjxFYWwgskzLWFgi4NhgBjEC2CGDKyBYpLIcRwAgAApgyAAQcMAIYgWwRwJSRLVJYDiNQaASKsjxMGUXpFmwURqBYEcCUUayewXZhBIoSAUwZRekWbBRGoFgRwJRRrJ7Bdi0PgQy5opSj3X2XySBV9MnMXXe7g4quhp2YMlYDdVzm6iAQJY\/tb6VV9dtFq1N+\/koV7aivps17LGThWQNTRv7ciDUVNwL0+QbjJ0f6bBpiXUpDmeg05W7ZW7G\/h06ZXlyRhPptbwtt1DkKbSymjOJqCNgagRBgyDZdJ7KcNUlT9jCm3UbVnoaOHudgWCAD8q92ndT0bufGs3WtQwXtamDKyL8rS0\/jmrf4Idne6tl49JypMiVhIFRucI0GBi6YdpYWFJv0p9skvtdO+AtIGpgySquNYGuXg0DwbKuH0VtfSt3DWI7GoskjebHTIPK1nqUKZhGmjIJBjQtaJQQ+9\/zk\/aj89Wb1Ej2MVTIrT8Wuk7cclTN97T2TeVKYSQ2mjEwI4fQSR4DqaR9F6iM6SYnXY0nzxd89okF010VySYm8JmDKWC6cOF9JIPDQd6mPER0+oiFKwtxlGblee6RBhC5f8j1cVvYcM2HKyBGw9OLTzrqK+GebQqFKG6riknPf29qp2fQF4NTcEGCGP\/QjpFEr1+SgZA4LpVqDEPnhYCFmQTFlzMGej5Nyg0U39zhTnrwWCIwuHcY+g8\/9WzcGuq3a7VyTZtwXBkt+Y2I+cMyXDsbvg+665pm9HLz50lqEetTPaIEzft4fEd42TBn5xVikPu217uB0Mr7WLBa\/RBsksgMm+9X7Yx6rikCk0106GwO4ahbzYZb89TBCO1Xytc4YaJ1cpUTo9mhQ+CdOYSlD+ObFTDgb63tCmQqi329sPE8LAu86qeknVhlnAOMzG9\/PlveJ3abej1zaqQueu1xmIQ\/RobY6S25r+ZErZvOVbOsybztDth9s838xH1HQs9ujMCoRPykTZyyV3cqlUFQZfSA52VVTpVCobMtYt1wGsFBgxpBFcxXLqqCW\/tFgRmUrFSgZymAGzLEhf2VVbI5gGx9hnp\/1mXTqDpLqfzXxd2wabKSNb6l\/VVMn0Gbbraa+n6r50oMdDTmsfhHqkz9W+vspQbiMNwih6HDrnmMiy49zmxD8MjI5+fmDuI6sv0Vqy9Evf6hqzY2fslafXjB8h33oyp\/I2BwQv5UrwA0VYbT42RgMJ63w2E6vf0Hq8oBdoCTlZTbNVfpNtprUmOD7x0uGMijSjwiVtS9wf2KMnSC4doab0ZJZrtq163mcw84f2ERtp02bs+iGQl\/g3Q7i7Cs53M98IdkdiTq76xDBydJd\/2ALZj2pCRmHjnM14zLn\/\/CQtFn8tXarOgZa\/ktYoJHYZ3fo\/OYTfkF5cEGh\/OVkCLqbYsmjWbQHPsNKjoICm0VzFW1hO1P0J4Jvz3hkJSgVMG9wZJAwOC6alGLW\/7O002D0RQntz\/qat7MRYAkz3NV1N5fl9036Ixq6q9MnTFMm1G+7DDxpRJwNOY4CoDoChXCvzUOYDN+OgSZQKQvUKg0mie9Et\/ADruRywzQwBpJIHk2OFuZKcGAzNtcNZRuhaiGaRvAlYCgRyrg7cm17i+lbfEOPksd0trtI9rr3TA1\/UwJAjN\/rY7Tfy6W3LdLqtGj4w36BVqbWyTs+mJvUaD1R2HeHAJFUgfb00tLn6mWpX+VMlSMvcZV6Q2XE\/Svhx9lJ1kbCU3BdvhHG+PAtbCgAsKIMzbVcuhXqODWT+4QTZMshlAhlVDZddRokXENnX2G+EiUOOfqSXhmgYKZLvjNpajxKdhkP6hqfVSiMTs95c029rk5RY\/vvhF7FEyoVIkdv5oBXbqJbTefa2BEmQozvH3VOtgXnpiAuzUQjKT7RhKrEJdN+T5PktEhZJZ0XilK2+pq66iqF0RNbqZlyN6pqum7Pi2Q+Y2iPRVdn1O3dVtPm9YDCxmf3zitk80vkSlGUDMaKYGNif+Cjxv01jU06xbY6G\/fjD9QphSI2S1XRRoad9fyMVUXVEfeHpxRVlexl1bONuv2N5qaaikqF+f2lprGjDwAfUdncUyVW5LK\/oG8LbUlREW9OjTqFom2Ye+AsBhbmjDI2v4WWRKlTdTUH97LOmOLTwu4jippzcarN1Fy\/Bpm+ehDNehQM4ssIJUIZIkK8ge1iwAyTrjOEdli8C371YIoGXMvLEx4oU86Gf2ZaPN5e949kpK2tV9JhEIcitPPnCRPh4o3lCI1PCDhjJH2pz6FlLUeItr3cQy\/DnVGqvXobTOErFn1eGcj0RJmN+K11NaqqigqdexqhT+6FoKP+N3PtJOqzmGde9nYcEEVIp48bBYc83aOfo7Kvz8lkPGHINxuuPenwurzvGB94jrXRho6dkTAo9MfaPatBLJGiiXvjydUHb+4xOr9x1Nt7wXv9bbHTqIOpJeXxQODeQDM45i+01d+SaOpl4O6Be5+NXTJ853hgrFsL6hiJ3nG113FhaOhomb+jpu78Yi5CaIoeB1HoZHBPGjhdYWAGu7ofP9P3ojRC2myffOei6a+DkYhnENrdYmCh9CyaX7JB0YFW8x9avD\/Wss4Y5NrkXU\/3xxH0V\/FWnaG5bpVWgsZx+ndwFDCUCGXwCEw6G5rYGQzHe80L30n8LU0j6WNbeDn2GPL1b321RQ6TfJOfwpBWfKBWtmWX\/rCp8+XEiXAWZZqGm4nNkvQXIXusbW1pgzOxw5KUOfGC0Jx0aPkn3URXw9lgYlrm81m6R2+eaR66\/+kNa6XY9MFnAZsKaR2fcZ9eXbwxLaEocuWo+fJW668clsPVUgKFp4AVtkvgbuTlo+Sv\/9jUpJkZAeop16hZ9MIUtFGRUjYnw0umOUb9H04esXwX5piYyQmgMHVtTfljNXr961Z9ghLJZug1j4eTkKYvdPiiIj0\/mCSeqlUh2sPfKutklp5myVe+1tfMb3Qi60+aZeBH3gbu\/pc8vh3Kgwjp8y1qION\/cVJwsaKQOXPwfyb1dfLxmzRCcsvrGsn2Z0yHmx2vss1pIbAIZdf8EguNksPRphc1M6QvgiSap6WQFr5JRpBIuUMC51xYurlyyYU5lA5lRMnWZ20hJLN4zmg2sOAET1U1XoY2yp6j2S+5r\/mD7PsD9jr2To2EboOQ8kk5sdvUabPqF\/9iwiz0X+czxs7E6mZbZ2faYNrNdx9iOZb8ItR2dzPv9sj5htbhVMUtmVlicH9k10lF037PhHLXdhQKjkofh9tvyQyJCV\/+cQahr6ENymZbsxLA+Cq5aEJr\/6BZNt7vmUbiA9XsBMfD26MwSblPJefuTJTNh1fCyodufYxQpUpJSLTHOzvN6sU\/fsV8laAxErwNDDJLXXqTo+ZzPugqhCeB1DiZHZZ3zBJm2D\/z\/XOmNNVdv1MFj9avqFBCj4bLn9WBHeRk8cfrUh4fsippro7VSuDqrVqrzaLhl+cWAItQDs2P144AR2\/zjlD\/5TASa6q3Qyxzm4KHnVolh\/OEkLK5JqQLfVoilAF8sSe+RMLf87NUf59U\/bQ4I0Chm6MIqVVPZhQUUmCH5SI\/qbGjpSmnJdR1MCQjgJnoX7pptUopCtN35u2EnpCxmm3y22DwX2n2RbnBsLm1UaUwe8No3GNjb0PqkrWtBx5X8\/mSzkLXPBEkrtdwDXP8Fjyu5U8mTQklSae5mApSDBKr5ZI0MolJD6PAZ2hzvSXGy703oOtk18yJbJSwmuh33Gnfu2GY5MHOXPZsTqDAbMK8qkmKq6NSylLkfHSas9ya313OGYdqOcrmKLj4tq6WAmXAtBO3pKpqn18iiQ5ecG+uVj8ad9aWx6Ro5gE3FcVH0VdsbVZn8GFwZBAefbvkfwXRjP9YXQ88ReE0FqIPvkDiTal4B27HtKMSGLNkNzCJlcR+EVoYUs33sdmo7P5mKff5sHxvNYEm6Yn5eorVzR3PQQ9WZf1gwGVv2jXywwZXmemk\/eIbMv8xm3+r3qqFu055xNbZrGbrKNkCD6\/ogz8lFRr5HDphcui\/QCx9k70j5E+wwnCZVYhSMIKzDYQjFAlddqVcxuaa6NG1+hK8gZiHcFUuTlS8GaZEYcyfMMMfpakJEGMVoCmn8cRfdnYbCMZtfAt4jItcfHg4fgv6JeUaZcIgaLFUvmIiNykaiWr3cvSaoHQxsJmaHxOm\/OR4vLJzqmbCrDP4vWc8PSVtXV26uc5pEP6k+Cljfkn14vNSeN7C6kN4sK3uNVKyTz1P9oR4I4qGfz\/X8Q56Opyey6P0f127BnFSKay2MJTtxCf1Grhr5mCdnQYXSR6F+2ouKn4Ct2Ps6bfk8CTbgQlCkQEz+8OT8SFVvIxsv6PAj4y0mu1SESIRilILlh7KJdtl6gOyW1f9qFINfWbREyop8o8sXgn6epkITUY+TyqXIDhQIS5Kdr\/L3hHVT8BFNoGViQ71dF32+P93emQIenOSx7aI0GzY3XGpvE5NsOmxv\/D0NBJJJCxxx2IQzAh0a4mv3F3vw20CkVGyo+HCHTgBD9M9r9rQP3XoD3S4nieYPmN7MmnQN4P8ukCop4uELv3xJnZUxWWdP3CThQi8Px+10rPgb6COShnHikm6FgKbqflNdDc2mI0HG5wwNEtUVAYO4q+j5DsXWGck0tPs0s2VzcR3uJJ5mY3P81+xU0bE+4rxCkvG9HsNe1QKhQoW4LbtbYEVQYm2NsF1YvnOckTdgbEfD5Cs9rCUqIy6O+imdpNs+ERDQ92eTtGZ90zzLAOC4\/coJFHKE599EJvnAOsC+18br3f1NfNDqpzVhz1OEsWmJ2W7nkboLknxd1mSqpkZmAx4eI8c9PtDEku3wwD9jyQBhLbvUqPwrTssnnMp8qMu0w7qxLM6XS3XMYA7IuteNyghntSqxRLiN23uig6r5svul3WNtS+O1vSd2ZfIGMx4kEZP75Ilayb22T\/yWMW9BxXVjeamV37xuMN+6IHbWFWxrabrLgr9FeMMYQAACfVJREFU0h9GlKcPrGXcDRVVKiO76ANFIiRmrhnrjUa9QndFYuoeSNiewyXzh\/UEa8H0p5M5jVxmo8Hz5roGc6teUXXQlvyfQhjxE3pTe0stq5cvI35cCGym5re13mLUq8pD99hFnbgS+N5tcRllVIdOp9\/fyjZ7cAbExkOG5gpVBUmiLImXISbPodgpQ6zr5Ueb9wMBdp\/4aOA+f\/3ZDQv\/wmgMEGn9IUl0GPqN\/LVI3jY0ds07MOIyPW8duB\/w9g0E+q3qDXxq7Aidw0i5tjax3xFLyd8Xu8pDqn820JH00M1JP8yABgJXW7j7TaT96Vjg1kW9eLEGifxpAkXKZAc0mgOa8i9GJ\/+0SGa95hkNIj+m5hKYKdJ5Oax3jwWuevvegMl\/kfZwijtiTj7FyWa9K3Bj4IMbA8cNJkfg\/oi39\/qQI9YfjIvPUr8eRmptCghgTtpxPRAYgRXTXsdLcmKdxOAai3n4mkmClJ0TsauxUZchPvooe9rqvepyeQL3R3utByTxblK8uNh3+WPsvCmDEudcY0lLfUXJY\/vbZy3ePofdc9260dlwwp8gKpIbO63PyxczBloIbKbmt06qPW7RSpLW+BATJs97wvq+scCA970fcc74XqIzMjTXh1EgV7SVq3SC0Xk\/LXbKyL7C0ueaVZOXPNnvQZoNeXpp1Yt67lbMvpxcJGHW9lkben1+CiaXzPOyIkIsJuL3xfrYFhVIhvmW9p\/DBMK17tYeeB7K37juOBQ017c5Xa1dv63923APN\/0Jqe2+CRCHINK+aCB8v\/CwjQsuw+5XjbbOnv4phL7w\/\/BNv1hnP5nUOwCZPITo4CUfYWiqSXGv5UH7kirKYLCKYPZnQed\/SXnEDJ9ojVriWwQJdhNYtvuvFwC7dBlzKZMeZ1ijZkktFhXue8XYaevpD6PZqP\/NE36x3t6esJk5Y3ONzLAVJYiy5K5cTHv+vtYOZaBN+rfaNjpPuVP02VPhFfn30z0bLG8dTjWRkUo+5zjgiz1GqmbBLtUMaiLv67ZZySwH4DDf4hqBh\/CY186toa4Ta2zQmeo0Ge2u48oKdbPjKvTJILVDWxkvd7fFfijYfpbiriX1Jq2EEIUuGutecG+0DX10WrN4ZZSTXMFhNtj1ZlD7tiWnlaKU5VGnFIoWHyTRZ+sURjeMw+B86cC\/D05\/CvOjSwslpEQ879w2vKqPIxAc9SPRU\/LFw7uELAmnScAmxKc+DXWZrtU7+Z5jTEJyoEm7mRDdv2D8+wb3hjNDI52ahE5x5uZK34OnR1Zv+scKXObXGqIMhKQv9bm2dDVk80r7pNN47huwfsFPqC4TvDTZZtkX58h9juunc7kJwaqOiOk5tSiN5pUmEeq3vZY7Zv7\/5Yjr7DfGhlx210A\/9PClovw\/oKCrbw6+vNJ+Fl9p5fFAfFx6P+AyZCR7\/n3w8d9l4hZe+3S\/Z1pTXcnvzY+Gzre7RVr7D2B8wCdnPCYBm0laZhkZWDi3tUlrH+Gc0T\/Qe1wjXZ+gAxpGpuYanmTnRWTSbCkuQXtup2uKMhAi1F0Br25jxrV6RlzfN2JPZPHcYMsgDfeJzoYsC3e1p8vFwDLQ3v22kNBzK2DDOqnpl4GTO3NbTJA0er3P594cZ2FUf91rFoqZoTZpgohdOUL0nfFsem3M7VG6Rhpq2sbtza+q6wwbHLFNg2mKSEpaFrBJGpa4yKa5jo9DJ0O6a6fgo781RhkAOQz8iYxPS1YokcUhX94C8MV+47A6cxdmlolGIuG7pPucsW4bvwyE5C8IObcyV8f4DrG5iMwn6wliGYgto6DMpmQtsVmpESNE3ZpbSEuTkyJJzbfrDZdgoMeGvgbG3dqdTcYkncLUV0SIM4HPDqOQWK3ckmSOEBclQRlCVFwoneyLtt4IivrMuyoyfB5jH2h7643t\/0aGGN4eeW1Nxu42L4mP2SAgq9WJ0RcklXk6IzgyqFY9JZpTWi6RoIifzJxxLseqnkyMwASYOLZtVFhLMGXkFd8o5f+dTH9Yv8zwuiXxVa68WvZnqkxWr5cgun8IOu1pEZgKUpt3yRN2NEzSkGWrtDxtrqJJDJP+8Nyuf4GtwpSRV4AJZXOmPaOdaQTMSsFHonmtbgkoqzxi2ono9\/rTDzEiFEl\/Uzq\/ze8h+eEAQvueWfkqT0Egoj29NNrZZMhl2+6yDcOUsWzocMaSQECsf0GDIj1uMo21DDU8iv7zVuwnWtltEa0+Qu94O\/ajBWlyFkUS5XZOI\/ULesnKFryyrAumjCyBwmKlioCozmqtRJ5L8f1rKeoRuvWx2vRy2PxtXavVqKs2\/sfj9gXbIlJkKpaoqM\/lZsqbLQcK1EPFlFEsnsd2CIaAxHTKQJBd3UvtDL47cm17ddNLjsCo125zeT\/y2l9SJ22LEMyyPCi+e6FrmNC2p3oxLw\/aU6jAlJECFEGiZmnnkRpFVYWiMwj6g501e1t8WW5UBXkcVoTAbqtdxziPpf4hxTDlL1MrYTV2RUWsSmZoVMd7HhwSZJv\/UhXClLEUMvmOJ7u7NplaZCgyNc2g0MgV+sGmxJ8qzXdxWF8SAiL1aa91fdcr52EdJCkBIeb2zbByd+671Dg1q3ugLx61sTsGU7zvJ5xhmDKEwzZZs\/JkoJnxU0hzQCOapsgIUv5dwsv7ybL4Kv8IrJOa3K7t79bxO+UT9Iu0F+53KBMiSuSUIdt075R1nDUVeGstpoxCNZD1hOjOyChSVT8lQuP3Qkiu2jm\/cahQRvx5l0Oo7R90\/OGYrqdUNmilcdekU2emWzwXDcv8EZY0qjMkYcrIAFAek2d+P41Q+UYxCgYpVK5Ulsg2oTwisPqqtup7R716cWyz7erbs2wLHtW7An2mgvMF2IspA0AoUJBoDHLRiPOY0Xw+gp7aVZKj53xAtco61hPzPz6yyqasoPjVqwWmjBW4Laess7TPHZJ3eXsNsq8hsUmvzik3FsYIFAkCjxSJHWvfDGb61iAVDvbomkmN67p199qvMa7hmkQAU0ah3Lpe3TF6w3G80\/vRgLWgi2KFqiAu588DAUwZpeFnbCVGoEgQwJRRJI7AZmAESgMBTBml4SdsJUagSBDAlFEkjsBmYARKA4G1TBml4QFsJUagpBDAlFFS7sLGYgRWGwFMGavtAVw+RqCkEMCUUVLuwsZiBFYbgYWUsdr24PIxAhiBokYAU0ZRuwcbhxEoNgQwZRSbR7A9GIGiRgBTRlG7Bxu3NhEo5Vphyihl72HbMQIFRwBTRsEhxwViBEoZAUwZpew9bDtGoOAIYMooOOS4wOUhgHMVBwKYMorDD9gKjECJIIApo0Qchc3ECBQHApgyisMP2AqMQIkggCmjRBy1PDNxLoxAvhHAlJFvRLE+jMCaRgBTxpp2L64cRiDfCPw\/AAAA\/\/+4qRw3AAAABklEQVQDAE+4MV1t36ZWAAAAAElFTkSuQmCC\" alt=\"\"><\/p>\n\n\n\n<p>\u63a8\u5bfc\u5230\u4e0b\u9762<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full\"><div class='fancybox-wrapper lazyload-container-unload' data-fancybox='post-images' href='https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image-1.png'><img class=\"lazyload lazyload-style-1\" src=\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\"  loading=\"lazy\" decoding=\"async\" width=\"484\" height=\"67\" data-original=\"https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image-1.png\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAANSURBVBhXYzh8+PB\/AAffA0nNPuCLAAAAAElFTkSuQmCC\" alt=\"\" class=\"wp-image-669\"\/><\/div><\/figure><\/div>\n\n\n<p>\u8fd9\u4e2a\u516c\u5f0f\u63ed\u793a\u4e86\u4e00\u4e2a\u60ca\u4eba\u7684\u53d1\u73b0\uff1a<strong>\u5956\u52b1\u51fd\u6570\u53ef\u4ee5\u7528\u7b56\u7565\u7684\u6bd4\u503c\u8868\u793a<\/strong>\u3002<\/p>\n\n\n\n<p>\u5177\u4f53\u610f\u4e49\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>\u4e0d\u9700\u8981\u663e\u5f0f\u5956\u52b1\u6a21\u578b<\/strong>\uff1a\u6211\u4eec\u53ef\u4ee5\u76f4\u63a5\u4ece\u7b56\u7565\u6bd4\u503c\u8ba1\u7b97\u5956\u52b1<\/li>\n\n\n\n<li><strong>\u5173\u952e\u7b80\u5316<\/strong>\uff1a\u5728\u6bd4\u8f83\u4e24\u4e2a\u7b54\u6848\u4f18\u52a3\u65f6\uff0c\u5e38\u6570\u9879\u03b2 log Z(x)\u4f1a\u6d88\u9664<\/li>\n\n\n\n<li><strong>\u76f4\u63a5\u4f18\u5316<\/strong>\uff1a\u53ef\u4ee5\u76f4\u63a5\u4f18\u5316\u7b56\u7565\uff0c\u800c\u4e0d\u662f\u5148\u5b66\u4e60\u5956\u52b1\u6a21\u578b\u518d\u4f18\u5316\u7b56\u7565<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\">3. \u635f\u5931\u51fd\u6570<\/h3>\n\n\n\n<p>\u5355\u6837\u672c\u4f3c\u7136\u51fd\u6570\uff1a<\/p>\n\n\n\n<p><\/p>\n\n\n\n<p>\u5728\u6982\u7387\u8bba\u4e2d\uff0c\u5982\u679c\u4e8b\u4ef6\u662f\u72ec\u7acb\u7684\uff0c\u90a3\u4e48\u5b83\u4eec\u540c\u65f6\u53d1\u751f\u7684\u6982\u7387\u7b49\u4e8e\u5404\u81ea\u53d1\u751f\u6982\u7387\u7684\u4e58\u79ef\u3002<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u5047\u8bbe\u6570\u636e\u96c6\u4e2d\u6bcf\u4e2a\u504f\u597d\u6837\u672c $(x, y_w, y_l)$ \u90fd\u662f\u72ec\u7acb\u91c7\u6837\u7684<\/li>\n\n\n\n<li>\u6574\u4e2a\u6570\u636e\u96c6\u88ab\u540c\u65f6\u89c2\u6d4b\u5230\u7684\u6982\u7387\uff0c\u7b49\u4e8e\u4e8c\u79d8\u9601\u6837\u672c\u88ab\u89c2\u6d4b\u5230\u7684\u6982\u7387\u4e4b\u79ef<\/li>\n<\/ul>\n\n\n\n<p>\u6574\u4e2a\u6570\u636e\u96c6\u7684\u4f3c\u7136\u51fd\u6570\uff1a<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full\"><div class='fancybox-wrapper lazyload-container-unload' data-fancybox='post-images' href='https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image-2.png'><img class=\"lazyload lazyload-style-1\" src=\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\"  loading=\"lazy\" decoding=\"async\" width=\"288\" height=\"97\" data-original=\"https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image-2.png\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAANSURBVBhXYzh8+PB\/AAffA0nNPuCLAAAAAElFTkSuQmCC\" alt=\"\" class=\"wp-image-670\"\/><\/div><\/figure><\/div>\n\n\n<p>\u4e4b\u524d\u63a8\u5bfc\u7684\u504f\u597d\u6982\u7387\u516c\u5f0f\uff1a<\/p>\n\n\n\n<p>$$p\\left(y_{w} \\succ y_{l} \\mid x\\right)=\\sigma\\left(\\beta \\log \\frac{\\pi_{\\theta}\\left(y_{w} \\mid x\\right)}{\\pi_{\\mathrm{ref}}\\left(y_{w} \\mid x\\right)}-\\beta \\log \\frac{\\pi_{\\theta}\\left(y_{l} \\mid x\\right)}{\\pi_{\\mathrm{ref}}\\left(y_{l} \\mid x\\right)}\\right)$$<\/p>\n\n\n\n<p>\u5c06\u8fd9\u4e2a\u8868\u8fbe\u5f0f\u4ee3\u5165\u635f\u5931\u51fd\u6570\uff0c\u6700\u7ec8\u7684DPO\u635f\u5931\u51fd\u6570\uff1a<\/p>\n\n\n\n<p>$\\mathcal{L}<em>{\\mathrm{DPO}}(\\pi<\/em>\\theta)=-\\mathbb{E}<em>{(x,y_m,y_l)\\sim\\mathcal{D}}\\left[\\log\\sigma\\left(\\beta\\log\\frac{\\pi<\/em>\\theta(y_w|x)}{\\pi_{\\mathrm{ref}}(y_w|x)}-\\beta\\log\\frac{\\pi_\\theta(y_l|x)}{\\pi_{\\mathrm{ref}}(y_l|x)}\\right)\\right]$<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full\"><div class='fancybox-wrapper lazyload-container-unload' data-fancybox='post-images' href='https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image-3.png'><img class=\"lazyload lazyload-style-1\" src=\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\"  loading=\"lazy\" decoding=\"async\" width=\"535\" height=\"70\" data-original=\"https:\/\/www.vcoco.top\/wp-content\/uploads\/2025\/09\/image-3.png\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAANSURBVBhXYzh8+PB\/AAffA0nNPuCLAAAAAElFTkSuQmCC\" alt=\"\" class=\"wp-image-675\"\/><\/div><\/figure><\/div>","protected":false},"excerpt":{"rendered":"<p>DPO\u7684\u6838\u5fc3 DPO\u6838\u5fc3\u521b\u65b0\u5728\u4e8e\u5c06&#8221;\u8bed\u8a00\u6a21\u578b\u6697\u85cf\u7740\u5956\u52b1\u6a21\u578b&#8221;\u8fd9\u4e00\u7406\u8bba\u6d1e\u89c1\u8f6c\u5316\u4e3a\u5b9e\u7528\u7684\u8bad [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[105],"tags":[],"class_list":["post-664","post","type-post","status-publish","format-standard","hentry","category-reinforcement-learning"],"_links":{"self":[{"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/posts\/664","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/comments?post=664"}],"version-history":[{"count":8,"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/posts\/664\/revisions"}],"predecessor-version":[{"id":676,"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/posts\/664\/revisions\/676"}],"wp:attachment":[{"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/media?parent=664"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/categories?post=664"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.vcoco.top\/index.php\/wp-json\/wp\/v2\/tags?post=664"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}