{"id":196,"date":"2026-10-11T20:55:03","date_gmt":"2026-10-11T12:55:03","guid":{"rendered":"https:\/\/99website.top\/blog\/?p=196"},"modified":"2026-10-11T20:55:03","modified_gmt":"2026-10-11T12:55:03","slug":"attention-is-all-you-need-4%ef%bc%9a%e5%87%ba%e9%94%99%e7%9a%84%e4%bb%a3%e4%bb%b7-loss","status":"publish","type":"post","link":"https:\/\/99website.top\/blog\/?p=196","title":{"rendered":"Attention Is All You Need – #4\uff1a\u51fa\u9519\u7684\u4ee3\u4ef7 – Loss"},"content":{"rendered":"\n

Attention Is All You Need – #4\uff1aLoss\uff0c\u795e\u7ecf\u7f51\u7edc\u72af\u9519\u7684\u4ee3\u4ef7<\/h1>\n

\u5199\u5728\u524d\u9762<\/h3>\n

\u5929\u554a\uff0c\u5929\u90fd\u9ed1\u4e86\uff0c\u4ece\u4e0b\u5348\u4e00\u70b9\u5230\u665a\u4e0a\u516d\u70b9\u534a\uff0c99MLP<\/code>\u7684XOR\u8bad\u7ec3\u6b63\u786e\u7387\u8fd8\u662f\u597d\u4f4e\uff0c\u65e2\u7136\u521d\u59cb\u5316\u5bf9\u8bad\u7ec3\u6210\u529f\u7387\u6ca1\u6709\u663e\u8457\u5e2e\u52a9\uff0c\u90a3\u95ee\u9898\u4f1a\u4e0d\u4f1a\u5728Loss\u4e0a\u5462\uff1f\u8fd9\u7bc7\u535a\u5ba2\u6211\u5c31\u8981\u597d\u597d\u7814\u7a76\u4e00\u4e0bLoss<\/p>\n

Loss\u7a76\u7adf\u662f\u4ec0\u4e48<\/h2>\n

\u73b0\u572899MLP\u4f7f\u7528\u7684\u635f\u5931\u51fd\u6570\u662f\uff1a<\/p>\n

\n\\boxed{L=(\\hat y-y)^2}\n<\/div>\n

\u5176\u4e2d\uff1a<\/p>\n

    \n
  • L<\/span>\uff1aLoss\uff0c\u4e5f\u5c31\u662f\u635f\u5931\u503c\u3002<\/li>\n
  • \\hat y<\/span>\uff1a\u795e\u7ecf\u7f51\u7edc\u9884\u6d4b\u7684\u8f93\u51fa\u3002<\/li>\n
  • y<\/span>\uff1a\u6b63\u786e\u7b54\u6848\u3002<\/li>\n<\/ul>\n

    \u8fd9\u53eb\u505a \u5e73\u65b9\u8bef\u5dee\u635f\u5931\uff08Squared Error Loss\uff09<\/strong>\u3002<\/p>\n

    \u4f8b\u5982\uff0c\u6211\u4eec\u8ba9\u795e\u7ecf\u7f51\u7edc\u5b66\u4e60 XOR\uff1a<\/p>\n

    \n\n\n\n\n\n\n\n
    \u8f93\u5165<\/th>\n\u6b63\u786e\u7b54\u6848 y<\/span><\/th>\n\u9884\u6d4b \\hat y<\/span><\/th>\nLoss<\/th>\n<\/tr>\n<\/thead>\n
    (0,0)<\/td>\n0<\/td>\n0.1<\/td>\n0.01<\/td>\n<\/tr>\n
    (0,1)<\/td>\n1<\/td>\n0.8<\/td>\n0.04<\/td>\n<\/tr>\n
    (1,0)<\/td>\n1<\/td>\n0.9<\/td>\n0.01<\/td>\n<\/tr>\n
    (1,1)<\/td>\n0<\/td>\n0.2<\/td>\n0.04<\/td>\n<\/tr>\n<\/tbody>\n<\/table><\/div>\n

    \u6bcf\u4e2a\u6837\u672c\u90fd\u4f1a\u4ea7\u751f\u81ea\u5df1\u7684 Loss\u3002\u56db\u4e2a\u6837\u672c\u7684\u5e73\u5747 Loss \u4e3a\uff1a<\/p>\n

    \nL_{\\mathrm{avg}}=\\frac{0.01+0.04+0.01+0.04}{4}=0.025\n<\/div>\n

    \u800c\u4f60\u7684 99MLP \u5f53\u524d\u662f\u9010\u4e2a\u6837\u672c\u66f4\u65b0\u6743\u91cd\u7684\uff0c\u6240\u4ee5\u6bcf\u6b21\u53cd\u5411\u4f20\u64ad\u5b9e\u9645\u4e0a\u4f7f\u7528\u7684\u662f\u5f53\u524d\u6837\u672c\u7684 Loss \u5bfc\u6570<\/strong>\u3002<\/p>\n

    \u4e00\u4e2a\u975e\u5e38\u53cd\u76f4\u89c9\u7684\u95ee\u9898<\/h2>\n

    \u73b0\u5728\u768499MLP\uff0c\u5728\u8f93\u51fa\u5c42\u8f93\u51faLoss\u7684\u65f6\u5019\uff0c\u7531\u4e8e\u8f93\u51fa\u5c42\u4f7f\u7528\u7684\u662fSigmoid\uff1a<\/p>\n

    \n\\hat y=\\sigma(z)=\\frac1{1+e^{-z}}\n<\/div>\n

    \u5b83\u7684\u5bfc\u6570\u662f\uff1a<\/p>\n

    \n\\frac{\\partial\\hat y}{\\partial z}=\\hat y(1-\\hat y)\n<\/div>\n

    \u6839\u636e\u94fe\u5f0f\u6cd5\u5219\uff1a<\/p>\n

    \n\\boxed{\n\\frac{\\partial L}{\\partial z}\n=\n2(\\hat y-y)\\hat y(1-\\hat y)\n}\n<\/div>\n

    \u95ee\u9898\u5c31\u5728\u8fd9\u91cc\uff0c\u5047\u5982\u7f51\u7edc\u51fa\u4e86\u4e00\u4e2a\u8d85\u7ea7\u5927\u7684\u9519\u8bef\uff0c\u6bd4\u5982\u6b63\u786e\u7b54\u6848\u662f1\uff0c\u4f46\u662f\u7f51\u7edc\u9884\u6d4b0.01<\/p>\n

    \u90a3\u5e26\u5165\u4ee5\u540e\uff0c\u4e2d\u95f4\u7684\\hat y(1-\\hat y)<\/span>\u5c31\u4f1a\u8ba9Loss\u53d8\u5f97\u5f88\u5c0f\u5f88\u5c0f<\/p>\n

    \u8fd9\u5c31\u662f\u5e73\u65b9\u8bef\u5dee\u4e0e Sigmoid \u7ec4\u5408\u65f6\u53ef\u80fd\u51fa\u73b0\u7684\u95ee\u9898\uff1a\u7f51\u7edc\u660e\u660e\u72af\u4e86\u5927\u9519\uff0c\u4f46\u5728\u8f93\u51fa\u63a5\u8fd1 0 \u6216 1 \u65f6\uff0cSigmoid \u7684\u5bfc\u6570\u592a\u5c0f\uff0c\u5bfc\u81f4\u53cd\u5411\u4f20\u64ad\u7684\u68af\u5ea6\u88ab\u538b\u7f29\u3002<\/strong><\/p>\n

    GPT\u63d0\u5230\uff0c\u53e6\u4e00\u79cd\u635f\u5931\u51fd\u6570\u2014\u2014Binary Cross-Entropy\uff08\u4e8c\u5143\u4ea4\u53c9\u71b5\uff0cBCE\uff09<\/strong>\u2014\u2014\u5c31\u80fd\u5728\u6570\u5b66\u4e0a\u6d88\u9664\u8f93\u51fa\u5c42\u8fd9\u4e2a\u989d\u5916\u7684 Sigmoid \u5bfc\u6570\u56e0\u5b50\u3002<\/p>\n

    \u4e8c\u5143\u4ea4\u53c9\u71b5 – Binary Cross-Entropy<\/h2>\n

    \u4e3a\u4ec0\u4e48\u8981\u6709\u4e8c\u5143\u4ea4\u53c9\u71b5<\/h3>\n

    Binary Cross-Entropy\uff0c\u7b80\u79f0 BCE\uff0c\u4e2d\u6587\u53eb\u4e8c\u5143\u4ea4\u53c9\u71b5\u635f\u5931\u3002<\/p>\n

    \u5b83\u7684\u516c\u5f0f\u662f\u8fd9\u4e2a\uff1a<\/p>\n

    \n\\boxed{L=-[y\\ln(\\hat y)+(1-y)\\ln(1-\\hat y)]}\n<\/div>\n

    \u5929\u554a\u8fd9\u90fd\u662f\u5565\u554a\u8fdelog\u90fd\u6765\u4e86<\/p>\n

    \u8fd9\u91cc\u7684 \\ln<\/span> \u662f\u81ea\u7136\u5bf9\u6570\uff0c\u662f\u4ee5 e<\/span> \u4e3a\u5e95\u7684\u5bf9\u6570\u3002\u5b83\u6ee1\u8db3\uff1a<\/p>\n

    \n\\ln(1)=0,\\quad \\ln(e)=1\n<\/div>\n

    \u6700\u5173\u952e\u7684\u6c42\u5bfc\u6027\u8d28\uff1a<\/p>\n

    \n\\boxed{\\frac{d}{dx}\\ln x=\\frac1x}\n<\/div>\n

    \u5373\\ln x<\/span>\u7684\u5bfc\u6570\u662f\\frac1x<\/span><\/p>\n

    \u65e2\u7136\u662fLoss\u51fd\u6570\uff0c\u90a3\\hat{y}<\/span>\u5bf9BCE\u7684\u5bfc\u6570\u662f\u4ec0\u4e48\u5462\uff1f<\/p>\n

    \u5f00\u59cb\u63a8\u5bfc\u5427\uff01<\/p>\n

    \u56e0\u4e3a\u53cd\u5411\u4f20\u64ad\u6700\u7ec8\u9700\u8981\u77e5\u9053\uff1a\u6fc0\u6d3b\u524d\u8f93\u51fa z<\/span> \u53d8\u5316\u4e00\u70b9\uff0cLoss \u4f1a\u600e\u4e48\u53d8\u3002<\/p>\n

    \u6839\u636e\u94fe\u5f0f\u6cd5\u5219\uff1a<\/p>\n

    \n\\boxed{\\frac{\\partial L}{\\partial z}\n=\\frac{\\partial L}{\\partial\\hat y}\n\\frac{\\partial\\hat y}{\\partial z}}\n<\/div>\n

    \u4e00\u6b65\u4e00\u6b65\u6162\u6162\u6765\uff0c\u5148\u628ay<\/span>\u770b\u505a\u5e38\u6570\uff1a<\/p>\n

    \u5bfc\u6570\u6709\u4e2a\u6027\u8d28\u5c31\u662f\u4e24\u4e2a\u53ef\u5bfc\u51fd\u6570\u4e4b\u548c\uff08\u6216\u5dee\uff09\u7684\u5bfc\u6570\uff0c\u7b49\u4e8e\u5b83\u4eec\u5404\u81ea\u5bfc\u6570\u7684\u548c\uff08\u6216\u5dee\uff09\u3002<\/strong><\/p>\n

    \u6240\u4ee5\u4e00\u4e2a\u4e00\u4e2a\u6c42\u5427\uff01<\/p>\n

    \n\\frac{\\partial}{\\partial \\hat{y}}[y\\ln(\\hat y)] = \\frac{y}{\\hat{y}}\n<\/div>\n

    \u7b2c\u4e00\u9879\u5b8c\u6210\uff01<\/p>\n

    \u7b2c\u4e8c\u9879\uff1a<\/p>\n

    \n\\frac{\\partial}{\\partial \\hat{y}}[(1-y)\\ln(1-\\hat y)] = \\frac{1-y}{1-\\hat y}\\times(-1) = -\\frac{1-y}{1-\\hat y}\n<\/div>\n

    \u6240\u4ee5\uff1a<\/p>\n

    \n\\frac{\\partial L}{\\partial \\hat{y}} = -(\\frac{y}{\\hat{y}}-\\frac{1-y}{1-\\hat y}) = \\frac{1-y}{1-\\hat y} – \\frac{y}{\\hat{y}}\n<\/div>\n

    ok\uff01\\hat{y}<\/span>\u5bf9L<\/span>\u7684\u5bfc\u6570\u6c42\u597d\u5566\uff01<\/p>\n

    \u800c\u4e14\u4e5f\u5df2\u77e5Sigmoid\u7684\u5bfc\u6570\uff1a<\/p>\n

    \n\\frac{\\partial\\hat y}{\\partial z}=\\hat y(1-\\hat y)\n<\/div>\n

    \u6240\u4ee5\uff1a<\/p>\n

    \n\\begin{aligned}\n\\frac{\\partial L}{\\partial z} &=(\\frac{1-y}{1-\\hat y} – \\frac{y}{\\hat{y}})\\hat y(1-\\hat y)\\\\\n&=(\\hat{y}-y)\n\\end{aligned}\n<\/div>\n

    \u5f97\u5230\u4e86\uff01\uff01\uff01\uff01\uff01\uff01\uff01\uff01\uff01\uff01\uff01<\/p>\n

    \u6765\u5199\u4ee3\u7801\u5427~<\/p>\n

    for epoch in range(20000):\n    for simple in np.random.permutation(len(train_data)):\n        y = train_data[simple][2]\n        haty = network.forward([train_data[simple][0],train_data[simple][1]])\n        # \u56e0\u4e3aforward\u6700\u540e\u8fd4\u56de\u7684\u662f\u6bcf\u4e2a\u795e\u7ecf\u5143\u7684\u8f93\u51fa\uff0c\u7136\u540e\u6700\u540e\u4e00\u5c42\u5c31\u4e00\u4e2a\u795e\u7ecf\u5143\uff0c\u6240\u4ee5\u7528[0]\n        # dL_dhaty = 2*(haty[0] - y)\n        dL_dhaty = ((1-y) \/ (1 - haty)) - (y\/haty)\n        network.backward(dL_dhaty,0.01)\n<\/code><\/pre>\n
    \u56e0\u4e3abackward\u91cc\u5df2\u7ecf\u7b97\u4e86\\frac{\\partial \\hat{y}}{\\partial z}<\/span>\uff0c\u6240\u4ee5\u54b1\u4eec\u76f4\u63a5\u4f20\u5165\\frac{\\partial L}{\\partial \\hat{y}}<\/span>\u5c31\u597d\u5566~<\/p>\n
    \u5f53 Sigmoid \u7684\u8f93\u51fa\u5728\u6d6e\u70b9\u8ba1\u7b97\u4e2d\u6070\u597d\u53d8\u6210 0 \u6216 1 \u65f6\uff0c\u8fd9\u4e2a\u516c\u5f0f\u53ef\u80fd\u51fa\u73b0\u9664\u4ee5\u96f6\u7684\u95ee\u9898\uff0c\u4ea7\u751f NaN<\/code> \u6216\u65e0\u7a77\u5927\uff0c\u8fdb\u800c\u6c61\u67d3\u7f51\u7edc\u6743\u91cd\uff0c\u5bfc\u81f4\u8bad\u7ec3\u5931\u8d25\u3002<\/p>\n
    \u4e0d\u8fc7\uff0c\u6211\u4eec\u521a\u521a\u5df2\u7ecf\u63a8\u5bfc\u51fa\u4e86 BCE + Sigmoid \u7684\u5408\u5e76\u68af\u5ea6\uff1a<\/p>\n
    \n\\frac{\\partial L}{\\partial z}=\\hat y-y\n<\/div>\n
    \u65e2\u7136\u5982\u6b64\uff0c\u4e3a\u4ec0\u4e48\u4e0d\u76f4\u63a5\u628a\u8fd9\u4e2a\u68af\u5ea6\u4f20\u5165\u53cd\u5411\u4f20\u64ad\u5462\uff1f<\/p>\n
    \u8fd9\u6837\u65e2\u80fd\u7b80\u5316\u8ba1\u7b97\uff0c\u4e5f\u53ef\u4ee5\u907f\u514d\u524d\u9762\u5206\u5f0f\u5e26\u6765\u7684\u6570\u503c\u95ee\u9898\uff01<\/p>\n
    \u6240\u4ee5\u6211\u6253\u7b97\u6539\u4e00\u6539Layer.backward()<\/code>\u8fd9\u6837\u5c31\u53ef\u4ee5\u907f\u514d\u8fd9\u4e2a\u95ee\u9898\u5566<\/p>\n
    \u7ed9Layer.backward()<\/code>\u52a0\u4e00\u4e2agradient_is_dz<\/code>\u53c2\u6570\uff0c\u544a\u8bc9\u4ed6\u8f93\u5165\u7684\u68af\u5ea6\u662f\u4e0d\u662f\u5df2\u7ecf\u4e58\u5b8cSigmoid\u7684\u5bfc\u6570\u4e86<\/p>\n
    def backward(self, output_gradient, learning_rate,gradient_is_dz = False):\n    # dhaty_dz = self.leaky_ReLU_derivative(self.z)\n    if not gradient_is_dz:\n        if self.activation == "sigmoid":\n            dhaty_dz = self.sigmoid_derivative(self.z)\n        else:\n            dhaty_dz = self.leaky_ReLU_derivative(self.z)\n        dL_dhaty = output_gradient\n        delta = dL_dhaty * dhaty_dz\n    else:\n        delta = output_gradient\n\n    dL_dW = delta @ self.x.T #.T\u8f6c\u7f6e\u77e9\u9635\n    dL_db = delta\n\n    #\u5728\u66f4\u65b0\u68af\u5ea6\u524d\u7b97\u51faLoss\u5bf9\u8d21\u732e\u7684\u5f71\u54cd\n    dL_dx = self.W.T @ delta\n\n    #\u68af\u5ea6\u4e0b\u964d\n    self.W -= learning_rate * dL_dW\n    self.b -= dL_db * learning_rate\n\n    return dL_dx\n<\/code><\/pre>\n
    \u540c\u65f6Network.backward()<\/code>\u4e5f\u8981\u505a\u4fee\u6539\uff0c\u5224\u65ad\u5982\u679c\u662f\u6700\u540e\u4e00\u5c42\u800c\u4e14\u63a5\u53d7\u8fdb\u6765\u7684\u8f93\u5165\u662f\u5df2\u7ecf\u4e58\u5b8cSigmoid\u5bfc\u6570\u7684\u68af\u5ea6\uff0c\u5c31\u8ba9Layer.backward<\/code>\u76f4\u63a5\u8ba1\u7b97\uff0c\u4e0d\u4e58Sigmoid\u5bfc\u6570\u4e86<\/p>\n
    def backward(self, output_gradient, learning_rate,output_is_dz = False):\n    gradient = output_gradient\n\n    # hhh\u6211\u4e0a\u6b21\u5199\u7adf\u7136\u5fd8\u4e86reversed\u8fd9\u4e2a\u51fd\u6570\n    for layer in reversed(self.layers):\n        gradient = layer.backward(gradient, learning_rate, gradient_is_dz=(output_is_dz and self.layers[-1] is layer))\n\n    return gradient\n<\/code><\/pre>\n
    \u6700\u540e\u8bad\u7ec3\u5b8c\u6210\u4ee5\u540e\uff0c\u544a\u8bc9Network.backward()<\/code>\u8fd9\u662f\u6700\u540e\u4e00\u5c42\u800c\u4e14\u8f93\u5165\u7ed9\u8fd9\u4e2a\u51fd\u6570\u7684\u662fdL_dz<\/code><\/p>\n
    for epoch in range(20000):\n    for simple in np.random.permutation(len(train_data)):\n        y = train_data[simple][2]\n        haty = network.forward([train_data[simple][0],train_data[simple][1]])\n        # \u56e0\u4e3aforward\u6700\u540e\u8fd4\u56de\u7684\u662f\u6bcf\u4e2a\u795e\u7ecf\u5143\u7684\u8f93\u51fa\uff0c\u7136\u540e\u6700\u540e\u4e00\u5c42\u5c31\u4e00\u4e2a\u795e\u7ecf\u5143\uff0c\u6240\u4ee5\u7528[0]\n        # dL_dhaty = 2*(haty[0] - y)\n        dL_dz = haty-y #\u6700\u540e\u76f4\u63a5\u4f20\u5165dL_dz\n        network.backward(dL_dz,0.01,output_is_dz=True)\n<\/code><\/pre>\n
    \u6765\u770b100\u6b21\u7684\u5b9e\u9a8c\u7ed3\u679c\uff01<\/p>\n
    xychart-beta\n    title "\u4ee4\u4eba\u5f00\u5fc3\u7684\u5b9e\u9a8c\u7ed3\u679c"\n    x-axis ["\u6807\u51c6\u6b63\u6001","\u65e7He","\u65b0He","\u968f\u673a\u6253\u4e71","BCE Loss"]\n    y-axis "\u8bad\u7ec3\u6210\u529f\u7387 (%)" 0 --> 100\n    bar [46,45,44,55,67]\n<\/code><\/div>\n
    \u8fd9\u6b21\u6709\u4e86\u66f4\u5927\u7684\u8fdb\u6b65\uff01<\/p>\n
    \u800c\u4e14\u6709\u4e00\u4e2a\u597d\u6d88\u606f\uff1a\u6210\u529f\u7684 67 \u6b21\uff0c\u5168\u90e8\u8fbe\u5230\u6bd4 0.5 \u66f4\u4e25\u683c\u7684\u6807\u51c6\u2014\u2014\u6b63\u786e\u8f93\u51fa\u5c0f\u4e8e 0.1\uff0c\u6b63\u786e\u8f93\u51fa 1 \u5927\u4e8e 0.9\u3002<\/strong><\/p>\n
    \u5bf9\u4e8e\u5931\u8d25\u768433\u4e2a\u6837\u4f8b\uff0c\u6709\u4e24\u4e2a\u73b0\u8c61\uff1a<\/p>\n
    \u73b0\u8c61\u4e00\uff1a\u6709\u4e9b\u7f51\u7edc\u8f93\u51fa\u5168\u90e8\u63a5\u8fd1 0.5<\/h4>\n
    \u4f8b\u5982\u7b2c 13 \u6b21\uff1a<\/p>\n
    \n\n\n\n\n\n\n\n
    \u8f93\u5165<\/th>\n\u771f\u5b9e\u503c<\/th>\n\u9884\u6d4b\u503c<\/th>\n<\/tr>\n<\/thead>\n
    (1,1)<\/td>\n0<\/td>\n0.500049<\/td>\n<\/tr>\n
    (0,1)<\/td>\n1<\/td>\n0.500050<\/td>\n<\/tr>\n
    (1,0)<\/td>\n1<\/td>\n0.500049<\/td>\n<\/tr>\n
    (0,0)<\/td>\n0<\/td>\n0.500049<\/td>\n<\/tr>\n<\/tbody>\n<\/table><\/div>\n
    \u8fd9\u51e0\u4e4e\u7b49\u4e8e\u4e0d\u7ba1\u8f93\u5165\u4ec0\u4e48\uff0c\u7f51\u7edc\u90fd\u5728\u731c 0.5\uff01<\/p>\n
    XOR \u6570\u636e\u96c6\u6070\u597d\u6709\u4e24\u4e2a 0\u3001\u4e24\u4e2a 1\uff0c\u6240\u4ee5\u6052\u5b9a\u8f93\u51fa 0.5 \u662f\u4e00\u4e2a\u7279\u6b8a\u7684\u505c\u6ede\u72b6\u6001\u3002<\/p>\n
    \u8fd9\u53ef\u80fd\u610f\u5473\u7740\u7f51\u7edc\u9677\u5165\u4e86\u5bf9\u79f0\u3001\u68af\u5ea6\u5f88\u5c0f\u6216\u5176\u4ed6\u96be\u4ee5\u9003\u79bb\u7684\u4f18\u5316\u533a\u57df\u3002\u4ec5\u51ed\u6700\u7ec8\u8f93\u51fa\uff0c\u8fd8\u4e0d\u80fd\u786e\u5b9a\u5177\u4f53\u539f\u56e0\u3002<\/p>\n
    \u73b0\u8c61\u4e8c\uff1a\u51fa\u73b0\u5927\u91cf\u63a5\u8fd1 1\/3 \u548c 2\/3 \u7684\u8f93\u51fa<\/h4>\n
    \u7f51\u7edc\u4f3c\u4e4e\u5b66\u4f1a\u4e86\u533a\u5206\u5176\u4e2d\u4e00\u79cd\u8f93\u5165\uff0c\u4f46\u5bf9\u5269\u4e0b\u4e09\u79cd\u8f93\u5165\u7ed9\u51fa\u4e86\u51e0\u4e4e\u76f8\u540c\u7684\u9884\u6d4b\u3002<\/p>\n
    \u800c\u8fd9\u4e09\u7ec4\u7684\u771f\u5b9e\u6807\u7b7e\u5206\u522b\u662f 0\u30011\u30010<\/code>\uff0c\u5e73\u5747\u503c\u6070\u597d\u4e3a\uff1a<\/p>\n
    \n\\frac{0+1+0}{3}=\\boxed{\\frac13}\n<\/div>\n
    \u8fd9\u6b63\u597d\u89e3\u91ca\u4e86\u4e3a\u4ec0\u4e48 BCE \u8bad\u7ec3\u4e2d\u53ef\u80fd\u51fa\u73b0\u63a5\u8fd1 0.3333 \u7684\u9884\u6d4b\u503c\u3002<\/p>\n
    \u5982\u679c\u7f51\u7edc\u6682\u65f6\u65e0\u6cd5\u533a\u5206\u8fd9\u4e09\u7ec4\u8f93\u5165\uff0c\u90a3\u4e48\u4ece BCE \u7684\u89d2\u5ea6\u770b\uff0c\u9884\u6d4b\u5b83\u4eec\u6807\u7b7e\u7684\u5e73\u5747\u503c\u5c31\u662f\u5408\u7406\u7684\u6700\u4f18\u5e38\u6570\u8f93\u51fa\u3002<\/p>\n
    \u540c\u7406\uff0c\u4e5f\u53ef\u4ee5\u89e3\u91ca\u4e00\u4e9b\u63a5\u8fd1 0.6667<\/span> \u7684\u8f93\u51fa\u3002<\/p>\n
    \u5f53\u7136\uff0c\u662f\u5426\u771f\u7684\u56e0\u4e3a\u7f51\u7edc\u628a\u67d0\u4e9b\u8f93\u5165\u6620\u5c04\u5230\u4e86\u76f8\u8fd1\u7684\u9690\u85cf\u5c42\u8868\u793a\uff0c\u8fd8\u9700\u8981\u8fdb\u4e00\u6b65\u89c2\u5bdf\u9690\u85cf\u5c42\u6570\u636e\u3002<\/p>\n
    \u5728\u8fd9\u6b21\u5b9e\u9a8c\u4e2d\uff0c\u6362\u7528 BCE \u540e\uff0c99MLP \u7684\u8bad\u7ec3\u6210\u529f\u7387\u7531\u4e4b\u524d\u7684 55% \u4e0a\u5347\u5230\u4e86 67%\uff0c\u63d0\u9ad8\u4e86 12 \u4e2a\u767e\u5206\u70b9\u3002\u4e0d\u8fc7\u4e24\u6b21\u5b9e\u9a8c\u6ca1\u6709\u4f7f\u7528\u5b8c\u5168\u76f8\u540c\u7684\u968f\u673a\u79cd\u5b50\uff0c\u56e0\u6b64\u4e0d\u80fd\u6392\u9664\u968f\u673a\u6ce2\u52a8\u3002\u81f3\u5c11\u5728\u5f53\u524d\u5b9e\u9a8c\u4e2d\uff0c\u6211\u4eec\u89c2\u5bdf\u5230\u4e86 BCE \u5e26\u6765\u7684\u79ef\u6781\u53d8\u5316<\/strong>\u3002<\/p>\n
    \u4e0d\u9519\u4e0d\u9519\uff0c\u603b\u7b97\u662f\u6709\u4e86\u5927\u5e45\u5ea6\u63d0\u5347\uff01\uff01<\/p>\n
    \u63a5\u4e0b\u6765\u561b\u3002\u3002\u3002\u3002\u662f\u65f6\u5019\u8ba4\u8bc6\u4e0bTransformer<\/code>\u4e86\uff01<\/p>\n","protected":false},"excerpt":{"rendered":"
    Attention Is All You Need – #4\uff1aLoss\uff0c\u795e\u7ecf\u7f51\u7edc\u72af\u9519\u7684\u4ee3\u4ef7 \u5199\u5728\u524d\u9762 \u5929\u554a\uff0c\u5929\u90fd\u9ed1\u4e86\uff0c\u4ece\u4e0b\u5348\u4e00\u70b9\u5230\u665a\u4e0a\u516d\u70b9\u534a\uff0c99MLP\u7684XOR\u8bad\u7ec3\u6b63\u786e\u7387\u8fd8\u662f\u597d\u4f4e\uff0c\u65e2\u7136\u521d\u59cb\u5316\u5bf9\u8bad\u7ec3\u6210\u529f\u7387\u6ca1\u6709\u663e\u8457\u5e2e\u52a9\uff0c\u90a3\u95ee\u9898\u4f1a\u4e0d\u4f1a\u5728Loss\u4e0a\u5462\uff1f\u8fd9\u7bc7\u535a\u5ba2\u6211\u5c31\u8981\u597d\u597d\u7814\u7a76\u4e00\u4e0bLoss Loss\u7a76\u7adf\u662f\u4ec0\u4e48 \u73b0\u572899MLP\u4f7f\u7528\u7684\u635f\u5931\u51fd\u6570\u662f\uff1a \\boxed{L=(\\hat y-y)^2} \u5176\u4e2d\uff1a  […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[25,23,26,24],"tags":[],"class_list":["post-196","post","type-post","status-publish","format-standard","hentry","category-ai","category-transformer","category-26","category-24"],"_links":{"self":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts\/196","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=196"}],"version-history":[{"count":0,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts\/196\/revisions"}],"wp:attachment":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=196"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=196"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=196"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}