{"id":169,"date":"2026-10-07T02:21:14","date_gmt":"2026-10-06T18:21:14","guid":{"rendered":"https:\/\/99website.top\/blog\/?p=169"},"modified":"2026-10-07T11:03:54","modified_gmt":"2026-10-07T03:03:54","slug":"attentionisallyouneed-2%ef%bc%9a%e4%bb%8e%e7%a5%9e%e7%bb%8f%e5%85%83%e5%88%b0%e7%9f%a9%e9%98%b5","status":"publish","type":"post","link":"https:\/\/99website.top\/blog\/?p=169","title":{"rendered":"AttentionIsAllYouNeed #2\uff1a\u4ece\u795e\u7ecf\u5143\u5230\u77e9\u9635"},"content":{"rendered":"\n

Attention Is All You Need #2\uff1a\u4ece\u795e\u7ecf\u5143\u5230\u77e9\u9635<\/h1>\n

OK\uff01\u73b0\u5728\u662f2026\u5e7410\u67085\u65e5\uff0c\u624b\u5934\u8fd8\u6709\u4e00\u4e2a\u79d1\u521b\u6bd4\u8d5b\u3002\u3002\u3002\u3002\u597d\u7d2f\u3002\u3002\u3002<\/p>\n

\u8fd9\u7bc7\u535a\u5ba2\u7684\u76ee\u7684\u662f\u628a\u4e0a\u4e00\u7bc7\u535a\u5ba2<\/a>\u7684MLP\u4ece\u5355\u4e2a\u795e\u7ecf\u5143\u904d\u5386\u5347\u7ea7\u6210\u77e9\u9635\u8ba1\u7b97\uff01<\/p>\n

\u53ef\u662f\u3002\u3002\u3002\u4ec0\u4e48\u662f\u77e9\u9635\u554a<\/p>\n

\u524d\u7f6e\u77e5\u8bc6<\/h2>\n

\u77e9\u9635<\/h3>\n

GPT\u8bb2\u5230<\/p>\n

\u6bd4\u5982\u4e00\u5c42\u6709 3 \u4e2a\u8f93\u5165\u30012 \u4e2a\u795e\u7ecf\u5143\u3002<\/p>\n

\u4ee5\u524d\u4f60\u53ef\u80fd\u5199\uff1a<\/p>\n

\u795e\u7ecf\u51431: w11, w12, w13\n\u795e\u7ecf\u51432: w21, w22, w23\n<\/code><\/pre>\n
\u73b0\u5728\u76f4\u63a5\u628a\u6240\u6709\u6743\u91cd\u585e\u8fdb\u4e00\u4e2a\u77e9\u9635\uff1a<\/p>\n
\u6211\u53bb\u4e86\u77e9\u9635\u7684Latex\u548b\u5199\u554a\uff0c\u5b66\u4e60\u4e00\u4e0b<\/p>\n
\u65e0\u62ec\u53f7\u77e9\u9635\uff1a<\/p>\n
\\begin{array}{c}\na & b \\\\\nc & d\n\\end{array}\n<\/code><\/pre>\n
\n\\begin{array}{c}\na & b \\\\\nc & d\n\\end{array}\n<\/div>\n
\u65b9\u62ec\u53f7\u77e9\u9635<\/p>\n
\\begin{bmatrix}\na & b \\\\\nc & d\n\\end{bmatrix}\n<\/code><\/pre>\n
\n\\begin{bmatrix}\na & b \\\\\nc & d\n\\end{bmatrix}\n<\/div>\n
\u5706\u62ec\u53f7\u77e9\u9635<\/p>\n
\\begin{pmatrix}\na & b \\\\\nc & d\n\\end{pmatrix}\n<\/code><\/pre>\n
\n\\begin{pmatrix}\na & b \\\\\nc & d\n\\end{pmatrix}\n<\/div>\n
\u5927\u62ec\u53f7\u77e9\u9635<\/p>\n
\\begin{Bmatrix}\na & b \\\\\nc & d\n\\end{Bmatrix}\n<\/code><\/pre>\n
\n\\begin{Bmatrix}\na & b \\\\\nc & d\n\\end{Bmatrix}\n<\/div>\n
ok~\u5dee\u4e0d\u591a\u5566<\/p>\n
\u6240\u4ee5\u77e9\u9635\u662f\u4ec0\u4e48\u5462\uff1f<\/p>\n
\n
\u77e9\u9635\u662f\u7ebf\u6027\u4ee3\u6570<\/a>\u9886\u57df<\/a>\u7684\u6838\u5fc3\u6570\u5b66\u672f\u8bed\uff0c\u6307\u7531\u82e5\u5e72\u5143\u7d20\u6309\u884c\u3001\u5217\u6392\u5217\u5f62\u6210\u7684\u77e9\u5f62\u9635\u5217\uff0c\u6700\u5e38\u89c1\u4e3a\u5b9e\u6570\u77e9\u9635\u4e0e\u590d\u6570\u77e9\u9635\uff0c\u5143\u7d20\u4e5f\u53ef\u53d6\u81ea\u57df\u3001\u73af<\/a>\u4e43\u81f3\u7b26\u53f7\u3001\u51fd\u6570\u7b49\u5176\u4ed6\u6570\u5b66\u5bf9\u8c61\u3002\u4f5c\u4e3a\u65e9\u671f\u89e3\u7ebf\u6027\u65b9\u7a0b\u7ec4\u7684\u5de5\u5177\uff0c\u5176\u96cf\u5f62\u5bf9\u5e94\u65b9\u7a0b\u7ec4\u7684\u7cfb\u6570\u8868\u4e0e\u589e\u5e7f\u77e9\u9635\uff0c\u65e0\u9700\u4e3a\u6b63\u65b9\u5f62\u9635\u5217\u3002<\/p>\n<\/blockquote>\n
\u6ca1\u660e\u767d\uff0c\u901a\u4fd7\u4e00\u70b9\u7684\u5462<\/p>\n
\u6700\u901a\u4fd7\u5730\u8bf4\uff1a<\/p>\n
\u77e9\u9635\u5c31\u662f\u201c\u6309\u884c\u548c\u5217\u6392\u597d\u7684\u4e00\u5806\u6570\u201d\u3002<\/strong><\/p>\n
\u6bd4\u5982\uff1a<\/p>\n
\n\\begin{bmatrix}\n1 & 2 & 3 \\\\\n4 & 5 & 6\n\\end{bmatrix}\n<\/div>\n
\u8fd9\u5c31\u662f\u4e00\u4e2a\u4e24\u884c\u4e09\u5217\u7684\u5c0f\u8868\u683c<\/p>\n
\u4f46\u662f\u77e9\u9635\u771f\u6b63\u5389\u5bb3\u7684\u5730\u65b9\u662f\uff1a\u5b83\u4e0d\u53ea\u662f\u201c\u8868\u683c\u201d\uff0c\u8fd8\u53ef\u4ee5\u8868\u793a\u5f88\u591a\u7ec4\u6570\u636e\uff0c\u6216\u8005\u4e00\u5957\u53d8\u6362\u89c4\u5219<\/strong>\u3002<\/p>\n
\u653e\u5230\u6211\u73b0\u5728\u7684\u795e\u7ecf\u7f51\u7edc\u91cc\u5c31\u7279\u522b\u597d\u7406\u89e3\uff1a<\/p>\n
\nW = \\begin{bmatrix}\nw_{11} & w_{12} & w_{13} \\\\\nw_{21} & w_{22} & w_{23} \n\\end{bmatrix}\n<\/div>\n
\u53ef\u4ee5\u76f4\u63a5\u628a\u5b83\u8bfb\u6210\uff1a<\/p>\n
\n
\u7b2c\u4e00\u884c\uff0c\u662f\u7b2c\u4e00\u4e2a\u795e\u7ecf\u5143\u7684\u5168\u90e8\u6743\u91cd<\/p>\n
\u7b2c\u4e8c\u884c\uff0c\u662f\u7b2c\u4e8c\u4e2a\u795e\u7ecf\u5143\u7684\u5168\u90e8\u6743\u91cd<\/p>\n<\/blockquote>\n
\u4e4b\u524d\u8bf4\u8fc7\u6743\u91cd\u662f\u5411\u91cf\uff0c\u6240\u4ee5<\/p>\n
\u77e9\u9635 = \u591a\u4e2a\u5411\u91cf\u645e\u5728\u4e00\u8d77<\/strong><\/p>\n
\u7528Python<\/code>\u8bb2\u89e3\u7684\u8bdd\u5c31\u662f<\/p>\n
neuron1.weights = [w11, w12, w13]\nneuron2.weights = [w21, w22, w23]\n<\/code><\/pre>\n
\u73b0\u5728\u53ea\u662f\u5199\u6210\uff1a<\/p>\n
W = [\n    [w11, w12, w13],\n    [w21, w22, w23]\n]\n<\/code><\/pre>\n
\u672c\u8d28\u4e0a\u6ca1\u53d8<\/p>\n
\u4e3a\u4e86\u795e\u7ecf\u7f51\u7edc\u7684\u8bdd\uff0c\u5148\u5403\u900f \u52a0\u6cd5\u3001\u4e58\u4e00\u4e2a\u6570\u3001\u77e9\u9635\u4e58\u6cd5<\/strong>\u5c31\u597d\u5566~<\/p>\n
\u77e9\u9635\u52a0\u6cd5<\/h4>\n
\u5bf9\u5e94\u4f4d\u7f6e\u76f4\u63a5\u52a0\uff1a<\/p>\n
\u73b0\u5728\u6709<\/p>\n
\nA = \\begin{bmatrix}\n1 & 2 \\\\\n3 & 4\n\\end{bmatrix}, B = \\begin{bmatrix}\n5 & 6 \\\\\n7 & 8\n\\end{bmatrix}\n<\/div>\n
\u90a3\u4e48\u52a0\u4e00\u5757\u5c31\u662f\uff1a<\/p>\n
\nA + B = \\begin{bmatrix}\n1+5 & 2+6 \\\\\n3+7 & 4+8\n\\end{bmatrix} = \\begin{bmatrix}\n6 & 8 \\\\\n10 & 12\n\\end{bmatrix}\n<\/div>\n
\u6240\u4ee5\u77e9\u9635\u52a0\u6cd5\u6709\u4e2a\u524d\u63d0\uff1a\u5f62\u72b6\u5fc5\u987b\u4e00\u6837<\/strong>\u3002<\/p>\n
\u5355\u6570\u5b57\u4e58\u77e9\u9635<\/h4>\n
\u7136\u540e\u77e9\u9635\u4e58\u4e00\u4e2a\u6570\u5b57\uff0c\u5c31\u662f\u6328\u4e2a\u4e58<\/p>\n
\n2\\begin{bmatrix}\n1 & 2 \\\\\n3 & 4\n\\end{bmatrix} = \\begin{bmatrix}\n2 & 4 \\\\\n6 & 8\n\\end{bmatrix}\n<\/div>\n
\u77e9\u9635\u4e58\u6cd5<\/h4>\n
\u771f\u6b63\u6709\u70b9\u4e1c\u897f\u7684\u662f\u77e9\u9635\u4e58\u6cd5<\/strong><\/p>\n
\u5148\u770b\u4e2a\u719f\u6089\u7684\u60c5\u51b5\uff1a<\/p>\n
\nA = \\begin{bmatrix}\n1 & 2 & 3 \\\\\n4 & 5 & 6\n\\end{bmatrix}\n<\/div>\n
\u7136\u540e\u6709\u4e00\u4e2a\u5411\u91cf\uff1a<\/p>\n
\nx=\\begin{bmatrix}\n10\\\\\n20\\\\\n30\n\\end{bmatrix}\n<\/div>\n
\u8ba1\u7b97\uff1a<\/p>\n
\nAx\n<\/div>\n
GPT\u8bf4\u7b97\u7684\u65f6\u5019\uff0c\u4e0d\u8981\u53bb\u60f3\u201c\u77e9\u9635\u600e\u4e48\u4e58\u201d\uff0c\u76f4\u63a5\u60f3\u4e00\u53e5<\/p>\n
\n
A\u7684\u6bcf\u4e00\u884c\uff0c\u90fd\u53bb\u548cx\u505a\u4e00\u6b21\u70b9\u4e58<\/p>\n<\/blockquote>\n
\u7b2c\u4e00\u884c\uff1a<\/p>\n
\n1\\times10+2\\times20+3\\times30\n=140\n<\/div>\n
\u7b2c\u4e8c\u884c\uff1a<\/p>\n
\n4\\times10+5\\times20+6\\times30\n=320\n<\/div>\n
\u6240\u4ee5\uff1a<\/p>\n
\nAx=\n\\begin{bmatrix}\n140\\\\\n320\n\\end{bmatrix}\n<\/div>\n
\u901a\u4fd7\u7406\u89e3\u77e9\u9635\u4e58\u6cd5\uff1a<\/p>\n
\n
\u5de6\u8fb9\u6bcf\u4e00\u884c\uff0c\u548c\u53f3\u8fb9\u6bcf\u4e00\u5217\u505a\u70b9\u4e58\u3002<\/p>\n<\/blockquote>\n
\u6bd4\u5982\u4e24\u4e2a\u77e9\u9635\uff1a<\/p>\n
\nA=\n\\begin{bmatrix}\n1&2 \\\\\n3&4\n\\end{bmatrix}\\\\\n\\\\\nB=\n\\begin{bmatrix}\n5&6\\\\\n7&8\n\\end{bmatrix}\n<\/div>\n
\u90a3\u4e48 AB<\/span> \u7684\u5de6\u4e0a\u89d2\uff0c\u5c31\u662f\uff1a<\/p>\n
\n1\\times5+2\\times7=19\n<\/div>\n
\u5de6\u4e0b\u89d2\uff1a<\/p>\n
\n3\\times5+4\\times7=43\n<\/div>\n
\u53f3\u4e0b\u89d2\uff1a<\/p>\n
\n3\\times6+4\\times8=50\n<\/div>\n
\u4e8e\u662f\uff1a<\/p>\n
\nAB=\n\\begin{bmatrix}\n19&22\\\\\n43&50\n\\end{bmatrix}\n<\/div>\n
\u89c4\u5f8b\u5c31\u662f<\/p>\n
\n\\boxed{\u884c\\cdot\u5217}\n<\/div>\n
\u8fd8\u6709\u4e00\u4e2a\u4eca\u665a\u4e00\u5b9a\u8981\u8bb0\u4f4f\u7684\u5c3a\u5bf8\u89c4\u5219\uff1a<\/p>\n
\n(a\\times b)(b\\times c)\n=\na\\times c\n<\/div>\n
\u4e2d\u95f4\u5fc5\u987b\u4e00\u6837\uff0c\u4e2d\u95f4\u6d88\u6389\uff0c\u7559\u4e0b\u4e24\u8fb9\u3002<\/strong><\/p>\n
\u8fd9\u73a9\u610f\u513f\u653e\u795e\u7ecf\u7f51\u7edc\u91cc\u9a6c\u4e0a\u5c31\u6d3b\u4e86\u3002\u5047\u8bbe\u4e00\u5c42\u6709 3 \u4e2a\u8f93\u5165\uff0c2 \u4e2a\u795e\u7ecf\u5143\uff1a<\/p>\n
\nW=\n\\begin{bmatrix}\nw_{11}&w_{12}&w_{13}\\\\\nw_{21}&w_{22}&w_{23}\n\\end{bmatrix}\n<\/div>\n
\u8fd9\u662f 2\\times3<\/span>\u3002\n\u8f93\u5165\uff1a<\/p>\n
\nx=\n\\begin{bmatrix}\nx_1\\\\\nx_2\\\\\nx_3\n\\end{bmatrix}\n<\/div>\n
\u8fd9\u662f 3\\times1<\/span>\u3002\n\u6240\u4ee5\uff1a<\/p>\n
\nWx\n<\/div>\n
\u5c3a\u5bf8\u5c31\u662f\uff1a<\/p>\n
\n(2\\times3)(3\\times1)\n\\rightarrow\n2\\times1\n<\/div>\n
\u7ed3\u679c\u6b63\u597d\u4e24\u4e2a\u6570\uff1a<\/p>\n
\n\\begin{bmatrix}\nz_1\\\\\nz_2\n\\end{bmatrix}\n<\/div>\n
\u4e5f\u5c31\u662f \u4e24\u4e2a\u795e\u7ecf\u5143\u5404\u81ea\u7b97\u51fa\u6765\u7684\u7ed3\u679c\u3002<\/p>\n
\u8f6c\u7f6e<\/h4>\n
\u628a\u77e9\u9635\u6cbf\u4e3b\u5bf9\u89d2\u7ebf\u201c\u7ffb\u4e00\u4e0b\u201d\uff0c\u884c\u53d8\u5217\uff0c\u5217\u53d8\u884c\u3002<\/strong><\/p>\n
\u6bd4\u5982\uff1a<\/p>\n
\nA=\n\\begin{bmatrix}\n1&2&3\\\\\n4&5&6\n\\end{bmatrix}\n<\/div>\n
\u5b83\u6709 2 \u884c 3 \u5217\uff0c\u4e5f\u5c31\u662f\uff1a<\/p>\n
\nA:2\\times3\n<\/div>\n
\u8f6c\u7f6e\u8bb0\u4f5c\uff1a<\/p>\n
\nA^T\n<\/div>\n
\u7ed3\u679c\u662f\uff1a<\/p>\n
\nA^T = \\begin{bmatrix}\n1 & 4 \\\\\n2 & 5 \\\\\n3 & 6\n\\end{bmatrix}\n<\/div>\n
\u4e00\u53e5\u8bdd\uff1a<\/p>\n
\n\\boxed{\\text{\u8f6c\u7f6e = \u884c\u5217\u4e92\u6362}}\n<\/div>\n
\u54c8\u8fbe\u739b\u79ef<\/h4>\n
\n\\boxed{\\odot}\n<\/div>\n
\u8fd9\u4e2a\u4e1c\u897f\u5728\u795e\u7ecf\u7f51\u7edc\u91cc\u901a\u5e38\u5c31\u662f<\/p>\n
\n
\u5bf9\u5e94\u4f4d\u7f6e\u7684\u5143\u7d20\u5206\u522b\u76f8\u4e58<\/p>\n<\/blockquote>\n
\u6bd4\u5982\uff1a<\/p>\n
\n\\begin{bmatrix}\n1\\\\\n2\\\\\n3\n\\end{bmatrix}\n\\odot\n\\begin{bmatrix}\n4\\\\\n5\\\\\n6\n\\end{bmatrix}\n=\n\\begin{bmatrix}\n1\\times4\\\\\n2\\times5\\\\\n3\\times6\n\\end{bmatrix}\n=\n\\begin{bmatrix}\n4\\\\\n10\\\\\n18\n\\end{bmatrix}\n<\/div>\n
\u533a\u522b\uff1a<\/p>\n
    \n
  • AB<\/span>\uff1a\u77e9\u9635\u4e58\u6cd5\uff0c\u884c \u00d7 \u5217\u518d\u6c42\u548c<\/strong><\/li>\n
  • A\\odot B<\/span>\uff1a\u9010\u5143\u7d20\u4e58\u6cd5\uff0c\u540c\u4f4d\u7f6e\u76f4\u63a5\u4e58<\/strong><\/li>\n
  • \\vec a\\cdot\\vec b<\/span>\uff1a\u5411\u91cf\u70b9\u4e58\uff0c\u4e58\u5b8c\u8fd8\u8981\u52a0\u8d77\u6765\u5f97\u5230\u4e00\u4e2a\u6570<\/strong><\/li>\n<\/ul>\n
    \u5916\u79ef<\/h4>\n
    \u5916\u79ef\u53ef\u4ee5\u7406\u89e3\u6210\u4e00\u4e2a\u5217\u5411\u91cf \u00d7 \u4e00\u4e2a\u884c\u5411\u91cf\uff0c\u7ed3\u679c\u53d8\u6210\u4e00\u4e2a\u77e9\u9635\u3002<\/strong><\/p>\n
    \u6bd4\u5982\uff1a<\/p>\n
    \na=\n\\begin{bmatrix}\n1\\\\\n2\n\\end{bmatrix},\n\\qquad\nb^T=\n\\begin{bmatrix}\n3&4&5\n\\end{bmatrix}\n<\/div>\n
    \u90a3\u4e48\uff1a<\/p>\n
    \nab^T=\n\\begin{bmatrix}\n1\\\\\n2\n\\end{bmatrix}\n\\begin{bmatrix}\n3&4&5\n\\end{bmatrix}\n=\n\\begin{bmatrix}\n3&4&5\\\\\n6&8&10\n\\end{bmatrix}\n<\/div>\n
    \u6240\u4ee5\uff1a<\/p>\n
    \n\\boxed{\\text{\u5916\u79ef = \u6bcf\u4e2a\u5de6\u8fb9\u5143\u7d20\uff0c\u5206\u522b\u4e58\u53f3\u8fb9\u6240\u6709\u5143\u7d20}}\n<\/div>\n
    \u70b9\u79ef\uff1a<\/p>\n
    \n\\begin{bmatrix} 1\\\\ 2\\\\ 3 \\end{bmatrix} \\cdot \\begin{bmatrix} 4\\\\ 5\\\\ 6 \\end{bmatrix} =32\n<\/div>\n
    \u6700\u540e\u5f97\u5230 \u4e00\u4e2a\u6570<\/strong>\u3002<\/p>\n
    \u5916\u79ef\u5f97\u5230\u4e00\u4e2a\u77e9\u9635<\/strong><\/p>\n
    \u81f3\u6b64\uff0c\u628aMLP\u4ece\u771f\u6b63\u7684\u795e\u7ecf\u5143\u8f6c\u4e3a\u77e9\u9635\u7684\u524d\u7f6e\u77e5\u8bc6\uff0c\u6b63\u5f0f\u5b66\u4e60\u5b8c\u5566~<\/p>\n
    \u90a3\u4e48\u63a5\u4e0b\u6765\u7684\u4efb\u52a1\u5c31\u662f\uff01\u628a99\u7684MLP\u53d8\u6210\u77e9\u9635\u7248\uff01<\/p>\n
    \u542cGPT\u8bf4Neuron<\/code>\u7c7b\u5c31\u8981\u6d88\u5931\u4e86\uff0c\u8be5\u600e\u4e48\u505a\u5462\uff1f<\/p>\n
    \u5148\u6293\u4f4f\u4eca\u5929\u552f\u4e00\u7684\u6838\u5fc3\u53d8\u5316\uff1a<\/p>\n
    \n\\boxed{\\text{Neuron \u6d88\u5931\uff0cLayer \u76f4\u63a5\u6301\u6709\u6574\u4e2a }W\\text{ \u548c }b}\n<\/div>\n
    \u539f\u6765\u7684\u795e\u7ecf\u7f51\u7edc\u7ed3\u6784\u7c7b\u4f3c\uff1a<\/p>\n
    Network\n \u251c\u2500 Layer\n \u2502   \u251c\u2500 Neuron\n \u2502   \u251c\u2500 Neuron\n \u2502   \u2514\u2500 Neuron\n \u2514\u2500 Layer\n     \u2514\u2500 Neuron\n<\/code><\/pre>\n
    \u73b0\u5728\u6211\u4eec\u8981\u53d8\u6210\uff1a<\/p>\n
    Network\n \u251c\u2500 Layer(W, b)\n \u2514\u2500 Layer(W, b)\n<\/code><\/pre>\n
    forward\u77e9\u9635\u5316<\/h3>\n
    \u522b\u6025\u7740\u78b0\u53cd\u5411\u4f20\u64ad\u3002\u54b1\u4eec\u5148\u771f\u6b63\u611f\u53d7\u5230\u77e9\u9635\u5230\u5e95\u66ff\u4ee3\u4e86\u4ec0\u4e48\u3002<\/p>\n
    \u5047\u8bbe\u8fd9\u4e00\u5c42\uff1a<\/p>\n
      \n
    • \u8f93\u51653\u4e2a\u6570<\/li>\n
    • \u6709\u4e24\u4e2a\u795e\u7ecf\u5143<\/li>\n<\/ul>\n
      \u54b1\u4eec\u73b0\u5728\u7684\u4ee3\u7801\uff1a<\/p>\n
      neuron1.weights = [w11, w12, w13]\nneuron1.bias = b1\n\nneuron2.weights = [w21, w22, w23]\nneuron2.bias = b2\n<\/code><\/pre>\n
      \u73b0\u5728\u7528\u77e9\u9635\u8868\u793a\uff1a<\/p>\n
      \nW=\n\\begin{bmatrix}\nw_{11}&w_{12}&w_{13}\\\\\nw_{21}&w_{22}&w_{23}\n\\end{bmatrix}\n<\/div>\n
      \u7136\u540ebias\u4e5f\u4e0d\u7528\u4e24\u4e2a\u795e\u7ecf\u5143\u5206\u522b\u5b58\u4e86\uff1a<\/p>\n
      \nb=\n\\begin{bmatrix}\nb_1\\\\\nb_2\n\\end{bmatrix}\n<\/div>\n
      \u8f93\u5165\u53d8\u6210\uff1a<\/p>\n
      \nx=\n\\begin{bmatrix}\nx_1\\\\\nx_2\\\\\nx_3\n\\end{bmatrix}\n<\/div>\n
      \u4e8e\u662f\u6574\u4e2aLayer<\/code>\uff1a<\/p>\n
      \n\\boxed{z = Wx+b}\n<\/div>\n
      \u5c31\u5b8c\u4e8b\u4e86\uff01\uff1f<\/p>\n
      \u56e0\u4e3a<\/p>\n
      \nWx+b\n=\n\\begin{bmatrix}\nw_{11}&w_{12}&w_{13}\\\\\nw_{21}&w_{22}&w_{23}\n\\end{bmatrix}\n\\begin{bmatrix}\nx_1\\\\\nx_2\\\\\nx_3\n\\end{bmatrix}\n+\n\\begin{bmatrix}\nb_1\\\\\nb_2\n\\end{bmatrix}\n<\/div>\n
      \u5c55\u5f00\u770b\u770b\u5462<\/p>\n
      \nWx+b\n=\n\\begin{bmatrix}\nw_{11}&w_{12}&w_{13}\\\\\nw_{21}&w_{22}&w_{23}\n\\end{bmatrix}\n\\begin{bmatrix}\nx_1\\\\\nx_2\\\\\nx_3\n\\end{bmatrix}\n+\n\\begin{bmatrix}\nb_1\\\\\nb_2\n\\end{bmatrix}\\\\\n\\\\\n\n=\n\\begin{bmatrix}\nw_{11}x_1+w_{12}x_2+w_{13}x_3+b_1\\\\\nw_{21}x_1+w_{22}x_2+w_{23}x_3+b_2\n\\end{bmatrix}\n<\/div>\n
      \u6240\u4ee5\u77e9\u9635\u538b\u6839\u6ca1\u6709\u6539\u53d8\u795e\u7ecf\u7f51\u7edc\u7684\u6570\u5b66\u539f\u7406\u3002<\/p>\n
      \u5b83\u53ea\u662f\u628a\uff1a<\/p>\n
      for neuron in neurons:\n    for x, w in zip(inputs, neuron.weights):\n        z += x * w\n<\/code><\/pre>\n
      \u53d8\u6210\uff1a<\/p>\n
      z = w @ x + b\n<\/code><\/pre>\n
      \n
      \u77e9\u9635\u5316\u4e0d\u662f\u6362\u4e86\u4e00\u79cd\u795e\u7ecf\u7f51\u7edc\uff0c\u800c\u662f\u628a\u5927\u91cf\u76f8\u540c\u7ed3\u6784\u7684\u795e\u7ecf\u5143\u8ba1\u7b97\u5408\u5e76\u6210\u4e00\u6b21\u7ebf\u6027\u4ee3\u6570\u8fd0\u7b97\u3002<\/p>\n<\/blockquote>\n
      \u795e\u4e86\uff01<\/p>\n
      \u4e0b\u9762\u5f00\u59cb\u5199\u4ee3\u7801~~~<\/p>\n
      \u56e0\u4e3a\u77e9\u9635\u8fd0\u7b97\u9700\u8981numpy<\/code>\uff0c\u6240\u4ee5\u5148\u5bfc\u5165<\/p>\n
      import numpy as np\n<\/code><\/pre>\n
      \n
      \u6211\u4e5f\u662f\u7ec8\u4e8e\u5f00\u59cb\u7528numpy\u4e86\u545c\u545c\u545c\u592a\u597d\u4e86<\/p>\n<\/blockquote>\n
      class Layer:\n    def __init__(self, neuron_num, weights_num):\n        # \u521d\u59cb\u5316\u795e\u7ecf\u5143\u77e9\u9635\uff0crandom.randn\u662f\u751f\u6210\u884cx\u5217\u7684\u77e9\u9635\uff0c\u91cc\u9762\u7684\u6570\u503c\u968f\u673a\n        # \u56e0\u4e3a\u6bcf\u4e2a\u8f93\u5165\u8981\u548c\u6bcf\u4e2a\u6743\u91cd\u70b9\u4e58\uff0c\u53c8\u56e0\u4e3a\u77e9\u9635\u4e58\u6cd5\u662f\u884c\u70b9\u4e58\u5217\uff0c\u6240\u4ee5\u6bcf\u4e2a\u795e\u7ecf\u5143\u7684\u6bcf\u4e2a\u6743\u91cd\u5b58\u5728\u77e9\u9635\u7684\u884c\u4e2d\n        # \u7136\u540e\u6bcf\u884c\u91cc\u7684\u6bcf\u4e2a\u6743\u91cd\u5c31\u53ef\u4ee5\u548c\u8f93\u5165\uff08\u770b\u505a\u591a\u884c1\u5217\u7684\u77e9\u9635\uff09\u70b9\u4e58\u5566~\n        self.W = np.random.randn(neuron_num, weights_num)\n\n        # \u521b\u5efa\u4e00\u4e2aneuron_num\u884c\uff0c1\u5217\u7684\u77e9\u9635\uff0c\u4e5f\u5c31\u662f\u54b1\u4eec\u7684\u504f\u7f6e\u77e9\u9635\uff01\n        # \u504f\u7f6e\u901a\u5e38\u53ef\u4ee5\u76f4\u63a5\u521d\u59cb\u5316\u4e3a 0\n        # \u56e0\u4e3a bias \u4e0d\u9700\u8981\u50cf\u6743\u91cd\u4e00\u6837\u9760\u968f\u673a\u521d\u59cb\u5316\u6765\u6253\u7834\u795e\u7ecf\u5143\u4e4b\u95f4\u7684\u5bf9\u79f0\u6027\u3002\n        self.b = np.zeros((neuron_num,1)) \n\n    def leaky_ReLU(self, x):\n        # np.where\u53ef\u4ee5\u7406\u89e3\u4e3a\u6279\u91cf1f\n        # np.where(\u6761\u4ef6, \u6761\u4ef6\u6210\u7acb\u65f6\u7684\u503c, \u6761\u4ef6\u4e0d\u6210\u7acb\u65f6\u7684\u503c)\n        return np.where(x > 0, x, 0.01*x)\n\n    def forward(self, x):\n        # \u6309\u7167\u54b1\u4eec\u7684\u77e9\u9635\u4e58\u6cd5\u516c\u5f0f\uff0cnumpy\u91cc\u4f7f\u7528@\u8fdb\u884c\u77e9\u9635\u4e58\u6cd5\n        self.z = self.W @ x + self.b\n\n        #\u6fc0\u6d3b\u51fd\u6570\n        self.haty = self.leaky_ReLU(self.z)\n\n        return self.haty\n<\/code><\/pre>\n
      \u7136\u540e\u5199Network<\/code>\uff0c\u7f51\u7edc\u7684\u4f5c\u7528\u5c31\u662f\u8ba9\u6570\u636e\u7a7f\u8fc7\u6bcf\u4e00\u5c42<\/p>\n
      class Network:\n    def __init__(self, layers):\n        self.layers = layers\n\n    def forward(self, x):\n        for layer in self.layers:\n            x = layer.forward(x) #\u4e0a\u4e00\u5c42\u8f93\u51fa\u5c31\u662f\u4e0b\u4e00\u6b21\u8f93\u5165\n\n        return x\n<\/code><\/pre>\n
      Neuron<\/code>\u7c7b\u771f\u7684\u6ca1\u4e86\uff01<\/p>\n
      \u56e0\u4e3a\u5168\u653e\u5230\u77e9\u9635\u91cc\u53bb\u4e86<\/p>\n
      Backward<\/h3>\n
      \u77e9\u9635\u7248Backward\u600e\u4e48\u505a\u5462<\/p>\n
      \u9996\u5148\u770b\u770b\u54b1\u4eec\u7684\u77e9\u9635\u7248Forward<\/p>\n
      def forward(self, x):\n\n    self.x = x #\u4f9d\u65e7\u4fdd\u5b58\u8f93\u5165\n\n    # \u6309\u7167\u54b1\u4eec\u7684\u77e9\u9635\u4e58\u6cd5\u516c\u5f0f\uff0cnumpy\u91cc\u4f7f\u7528@\u8fdb\u884c\u77e9\u9635\u4e58\u6cd5\n    self.z = self.W @ x + self.b\n\n    #\u6fc0\u6d3b\u51fd\u6570\n    self.haty = self.leaky_ReLU(self.z)\n\n    return self.haty\n<\/code><\/pre>\n
      \u6362\u6210\u6570\u5b66\uff1a<\/p>\n
      \nx \\rightarrow z = Wx+b \\rightarrow \\hat{y} = f(z) \\rightarrow L\n<\/div>\n
      backword\u53cd\u8fc7\u6765\uff1a<\/p>\n
      \nL \\rightarrow \\hat{y} \\rightarrow z \\rightarrow W,b,x\n<\/div>\n
      \n
      \u68af\u5ea6\u516c\u5f0f\u7684\u94fe\u5f0f\u6cd5\u5219\uff1a<\/p>\n
      \n\\frac{\\partial L}{\\partial w} = \\frac{\\partial L}{\\partial \\hat{y}} \\cdot \\frac{\\partial \\hat{y}}{\\partial z} \\cdot \\frac{\\partial z}{\\partial w}\n<\/div>\n
      \u628a\u524d\u4e24\u9879\u5408\u8d77\u6765\uff1a<\/p>\n
      \n\\delta\n=\n\\frac{\\partial L}{\\partial z}\n=\n\\frac{\\partial L}{\\partial \\hat y}\n\\cdot\n\\frac{\\partial \\hat y}{\\partial z}\n<\/div>\n
      \u8fd9\u91cc\u6211\u5f97\u8bb0\u4e00\u4e0b\uff0c\u4e3a\u4ec0\u4e48\u5408\u4f53\u5462\uff1f<\/p>\n
      \u56e0\u4e3a\u4e00\u4e2a\u795e\u7ecf\u5143\u4e2d\u7684\u6240\u6709\u6743\u91cd\uff0c\u5171\u7528\u4e00\u4e2a\\delta<\/span><\/p>\n
      \u53c8\u56e0\u4e3a\u795e\u7ecf\u5143\u7ebf\u6027\u8868\u8fbe\u5f0f\uff1a<\/p>\n
      \nz = w_1x_1+w_2x_2+\\cdots+b\n<\/div>\n
      \u4e00\u4e2a\u795e\u7ecf\u5143\u6700\u540e\u53ea\u5f97\u5230\u4e00\u4e2az<\/span>\uff0c\u6240\u4ee5\u4e00\u4e2a\u795e\u7ecf\u5143\u5171\u7528\u4e00\u4e2a\\delta<\/span><\/p>\n
      \u6709\uff1a<\/p>\n
      \n\\frac{\\partial z_i}{\\partial w_{ij}} = x_j\n<\/div>\n
      \n
      i\u8868\u793a\u7b2ci\u4e2a\u795e\u7ecf\u5143\uff0cj\u8868\u793a\u8fd9\u4e2a\u795e\u7ecf\u5143\u7684j\u4e2a\u8f93\u5165\/\u6743\u91cd<\/p>\n<\/blockquote>\n
      \u4e8e\u662f\u5355\u4e2a\u68af\u5ea6\u53ef\u4ee5\u53d8\u6210\uff1a<\/p>\n
      \n\\frac{\\partial L}{\\partial w_{ij}}\n=\n\\delta_i x_j\n<\/div>\n
      \u73b0\u5728\u5047\u8bbe\u4e00\u5c42\u67092\u4e2a\u795e\u7ecf\u5143\uff0c3\u4e2a\u8f93\u5165\uff1a<\/p>\n
      \nW=\n\\begin{bmatrix}\nw_{11}&w_{12}&w_{13}\\\\\nw_{21}&w_{22}&w_{23}\n\\end{bmatrix}\n<\/div>\n
      \u8f93\u5165\uff1a<\/p>\n
      \nx=\n\\begin{bmatrix}\nx_1\\\\\nx_2\\\\\nx_3\n\\end{bmatrix}\n<\/div>\n
      \u56e0\u4e3a\u6709\u4fe9\u795e\u7ecf\u5143\uff0c\u6240\u4ee5\u4f1a\u5f97\u5230\u4e24\u4e2a \\delta<\/span> \uff0c\u4e00\u4eba\u4e00\u4e2a\uff1a<\/p>\n
      \n\\delta=\n\\begin{bmatrix}\n\\delta_1\\\\\n\\delta_2\n\\end{bmatrix}\n<\/div>\n
      \n
      \u8865\u5145\u4e00\u4e0b\uff0c\u5728\u8fd9\u91cc\u7684\\delta<\/span>\u77e9\u9635\u91cc\uff0c\u6bcf\u4e2a\u5143\u7d20\u90fd\u662f\u76f4\u63a5\u7684\\frac{\\partial L}{\\partial \\hat y}\n\\cdot\n\\frac{\\partial \\hat y}{\\partial z}<\/span>\u3002\u6240\u4ee5\u4e24\u4e2a\u77e9\u9635\u4e5f\u8981\u4e00\u4e2a\u5bf9\u4e00\u4e2a\u7684\u76f8\u4e58\u3002\u6240\u4ee5\u8fd9\u91cc\u4e0d\u80fd\u7528\u77e9\u9635\u4e58\u6cd5\uff0c\u7ffb\u770b\u4e4b\u524d\u7684\u7b14\u8bb0\uff0c\u8fd9\u91cc\u8981\u7528\u54c8\u8fbe\u739b\u79ef\uff0c\u5c31\u662f\u76f4\u63a5\u7684\u5bf9\u5e94\u4f4d\u7f6e\u76f8\u4e58<\/p>\n
      numpy<\/code>\u4e2d\u54c8\u8fbe\u739b\u79ef\u5c31\u662f\u76f4\u63a5\u7528*<\/code>\u53f7<\/p>\n<\/blockquote>\n
      \u54b1\u4eec\u5df2\u7ecf\u77e5\u9053\u4e86\u5355\u4e2a\u6743\u91cd\u68af\u5ea6\u7684\u5199\u6cd5\uff0c\u90a3\u73b0\u5728\u5c31\u7ed9\u6240\u6709\u7684\u6743\u91cd\u4e00\u4e2a\u4e00\u4e2a\u5199\u51fa\u6765\u3002<\/p>\n
      \u7b2c\u4e00\u4e2a\u795e\u7ecf\u5143\uff1a<\/p>\n
      \n\\begin{aligned}\n\\frac{\\partial L}{\\partial w_{11}} &= \\delta_1x_1 \\\\[0.8em]\n\\frac{\\partial L}{\\partial w_{12}} &= \\delta_1x_2 \\\\[0.8em]\n\\frac{\\partial L}{\\partial w_{13}} &= \\delta_1x_3\n\\end{aligned}\n<\/div>\n
      \u7b2c\u4e8c\u4e2a\u795e\u7ecf\u5143\uff1a<\/p>\n
      \n\\begin{aligned}\n\\frac{\\partial L}{\\partial w_{21}} &= \\delta_2x_1 \\\\[0.8em]\n\\frac{\\partial L}{\\partial w_{22}} &= \\delta_2x_2 \\\\[0.8em]\n\\frac{\\partial L}{\\partial w_{23}} &= \\delta_2x_3\n\\end{aligned}\n<\/div>\n
      \u7136\u540e\u6bcf\u4e00\u4e2a\u7684\u7b97\u6cd5\u90fd\u4e00\u6837\u5bf9\u5427\uff0c\u54b1\u4eec\u6309\u7167\u795e\u7ecf\u5143\u548c\u6743\u91cd\u539f\u6765\u7684\u77e9\u9635\uff0c\u5bf9\u5e94\u7740\u518d\u6574\u4e00\u4e2a\u201c\u68af\u5ea6\u77e9\u9635\u201d<\/p>\n
      \n\\frac{\\partial L}{\\partial W}\n=\n\\begin{bmatrix}\n\\delta_1x_1 & \\delta_1x_2 & \\delta_1x_3\\\\\n\\delta_2x_1 & \\delta_2x_2 & \\delta_2x_3\n\\end{bmatrix}\n<\/div>\n
      hmmmm\uff0c\u6211\u5f97\u597d\u597d\u770b\u770b\u8fd9\u4e2a\u4e1c\u897f<\/p>\n
      \u6211\u53bb\u8fd9\u4e0d\u5c31\u662f\uff01\uff1f<\/p>\n
      \n\\begin{bmatrix}\n\\delta_1\\\\\n\\delta_2\n\\end{bmatrix}\\begin{bmatrix}\nx_1 &\nx_2 &\nx_3\n\\end{bmatrix} =\\begin{bmatrix}\n\\delta_1x_1 & \\delta_1x_2 & \\delta_1x_3\\\\\n\\delta_2x_1 & \\delta_2x_2 & \\delta_2x_3\n\\end{bmatrix}\n<\/div>\n
      \u5417\uff1f\uff01<\/p>\n
      \u6ca1\u95ee\u9898\uff0c\u884c\u4e58\u5217\uff0c\\delta<\/span>\u77e9\u9635\u7684\u7b2c\u4e00\u884c\u662f\\delta_1<\/span>\uff0c\u7136\u540e\u5206\u522b\u4e58\u4ee5\u7b2c\u4e8c\u4e2a\u77e9\u9635\u7684x_1 ,x_2, x_3<\/span><\/p>\n
      \u4f46\u662f\u54b1\u4eec\u7684\u8f93\u5165\u662f\u4e00\u5217\u591a\u884c\u7684\u77e9\u9635\u554a\uff0c\u8fd9\u548b\u529e<\/p>\n
      \u7ffb\u4e86\u7ffb\u524d\u9762\u7684\u7b14\u8bb0\uff0c\u627e\u5230\u4e86\uff01\u8f6c\u7f6e<\/strong>\uff01<\/p>\n
      \nx^T=\n\\begin{bmatrix}\nx_1 &\nx_2 &\nx_3\n\\end{bmatrix}\n<\/div>\n
      \u884c\u53d8\u6210\u5217\uff0c\u5217\u53d8\u6210\u884c<\/p>\n
      \u6240\u4ee5\u6700\u540e\u7684\u5bf9\u4e8e\u4e00\u4e2a\u6743\u91cd\u77e9\u9635\u7684\u68af\u5ea6\uff1a<\/p>\n
      \n\\boxed{\n\\frac{\\partial L}{\\partial W}\n=\n\\delta x^T\n}\n<\/div>\n
      \n
      \u5728\u8fd9\u91cc\u52a0\u4e00\u4e2a\u6211\u7684\u5c0f\u7591\u95ee\uff0c\u4e3a\u4ec0\u4e48\u4e0d\u8f6c\u7f6e\\delta<\/span>\u5462\uff1f<\/p>\n
      \u56e0\u4e3a\u8f6c\u7f6e\u4e86\u4ee5\u540e<\/p>\n
      \u5f0f\u5b50\u53d8\u6210<\/p>\n
      \n\\delta^T=\n\\begin{bmatrix}\n\\delta_1&\\delta_2\n\\end{bmatrix}\\\\\n\\\\\n\\delta^T x\n<\/div>\n
      \u8fd9\u4e2a\u77e9\u9635\u4e58\u6cd5\u80fd\u7b97\u5417\uff1f<\/p>\n
      \u77e9\u9635\u4e58\u6cd5\u7684\u6cd5\u5219\u662f\uff0c\u4e24\u4e2a\u77e9\u9635\u5fc5\u987b\u9075\u5faa<\/p>\n
      (\u884c1x\u52171)(\u884c2x\u52172)<\/p>\n
      \u5de6\u8fb9\u77e9\u9635\u7684\u5217\u6570<\/strong>\u5fc5\u987b\u7b49\u4e8e\u53f3\u8fb9\u77e9\u9635\u7684\u884c\u6570<\/strong>\u3002<\/p>\n
      \u4e5f\u5c31\u662f\u52171=\u884c2<\/p>\n
      \u73b0\u5728\u662f<\/p>\n
      (1×2)(3×1)<\/p>\n
      \u4e58\u4e0d\u4e86<\/p>\n
      \u90a3\u6389\u4e2a\u4e2a\u4e0d\u5c31\u80fd\u4e58\u4e86\u5417\uff1f<\/p>\n
      \u786e\u5b9e\u53ef\u4ee5\uff0c\u4f46\u662f\u54b1\u4eec\u5f97\u5230\u7684\u662f\u68af\u5ea6\u77e9\u9635\u7684\u8f6c\u7f6e\u7248<\/p>\n
      \u5fc5\u987b\u518d\u8f6c\u7f6e\u56de\u6765\u624d\u80fd\u548c W<\/span> \u7684\u6392\u5217\u5bf9\u5e94\uff0c\u56e0\u6b64\u4e0d\u5982\u4e00\u5f00\u59cb\u76f4\u63a5\u6784\u9020\u4e0e W<\/span> \u540c\u5f62\u72b6\u7684\u68af\u5ea6\u77e9\u9635\u3002<\/p>\n<\/blockquote>\n
      \u597d\uff01\u63a5\u4e0b\u6765\u54b1\u4eec\u5df2\u7ecf\u627e\u5230\u4e86\u68af\u5ea6\u77e9\u9635\uff0c\u90a3\u600e\u4e48\u68af\u5ea6\u4e0b\u964d\u5462\uff1f<\/p>\n
      \u56de\u987e\u4e00\u4e0b\u4e0a\u6b21\u7684\u7b14\u8bb0<\/p>\n
      \u4ee5\u524d\u7684\u5355\u4e2a\u6743\u91cd\uff1a<\/p>\n
      \nw\\leftarrow\nw-\\eta\\frac{\\partial L}{\\partial w}\n<\/div>\n
      w<\/span>\u51cf\u53bb\u5355\u4e2a\u7684\u6743\u91cd\u68af\u5ea6\u4e58\u4ee5\u5b66\u4e60\u7387<\/p>\n
      \u90a3\u77e9\u9635\u5462\uff1f\u4ee5\u6b64\u7c7b\u63a8<\/p>\n
      \nW \\leftarrow W – \\eta\\frac{\\partial L}{\\partial W}\n<\/div>\n
      \u4e5f\u5c31\u662f\u76f4\u63a5<\/p>\n
      \nW \\leftarrow W – \\eta(\\delta x^T)\n<\/div>\n
      \n
      numpy<\/code>\u4e2d\u5e38\u6570\u4e58\u77e9\u9635\u4e5f\u76f4\u63a5\u4f7f\u7528*<\/code><\/p>\n<\/blockquote>\n
      \u5bf9\u4e8e\u504f\u7f6e\u7684\u68af\u5ea6\u4e0b\u964d\u5462\uff1f<\/p>\n
      \u5bf9\u4e8e\u6bcf\u4e2a\u504f\u7f6e<\/p>\n
      \n\\frac{\\partial z}{\\partial b}=1\n<\/div>\n
      \u8fd9\u4e2a\u5c31\u597d\u7406\u89e3\u591a\u5566<\/p>\n
      \n\\boxed{\n\\frac{\\partial L}{\\partial b}=\\delta\n}\n<\/div>\n
      \u6765\u5199\u4ee3\u7801\u8bd5\u8bd5\u770b<\/p>\n
      \u5148\u628aLeaky ReLU<\/code>\u7684\u5bfc\u6570\u5199\u4e86\uff0cnumpy<\/code>\u7248\u7684<\/p>\n
      \nf'(z)=\n\\begin{cases}\n1,&z>0\\\\\n0.01,&z<0\n\\end{cases}\n<\/div>\n
      \u56e0\u4e3a\u662f\u77e9\u9635\uff0c\u6240\u4ee5\u4ee3\u7801\u8981\u7528np.where<\/code><\/p>\n
      def leaky_ReLU_derivative(self, z):\n    return np.where(z > 0, 1, 0.01)\n<\/code><\/pre>\n
      \u90a3\u4e48backward\u5c31\u662f<\/p>\n
      def backward(self, output_gradient, learning_rate):\n    dhaty_dz = self.leaky_ReLU_derivative(self.z)\n    dL_dhaty = output_gradient\n\n    delta = dL_dhaty * dhaty_dz\n\n    dL_dW = delta @ self.x.T #.T\u8f6c\u7f6e\u77e9\u9635\n    dL_db = delta\n\n    #\u68af\u5ea6\u4e0b\u964d\n    self.W -= learning_rate * dL_dW\n    self.b -= dL_db * learning_rate\n<\/code><\/pre>\n
      \u68af\u5ea6\u4e0b\u964d\u5b8c\u6210\uff01\u597d\u8036\uff01<\/p>\n
      \u4f46\u53ea\u4e0b\u964d\u662f\u4e0d\u884c\u7684\uff0c\u54b1\u4eec\u8fd8\u5f97\u628a\u8f93\u5165\u7684\u68af\u5ea6\u53cd\u5411\u4f20\u64ad\u7ed9\u4e0a\u4e00\u5c42\uff0c\u5f53\u505a\u4e0a\u4e00\u5c42\u795e\u7ecf\u5143\u8f93\u51fa\u4ea7\u751f\u7684\u68af\u5ea6<\/p>\n
      \u56de\u987e\u4e00\u4e0b\u4e4b\u524d\u7684\u7b14\u8bb0\uff0c\u600e\u4e48\u6c42\u5355\u4e2a\u8f93\u5165\u4ea7\u751f\u7684\u68af\u5ea6\u5462\uff1f<\/p>\n
      \n\\frac{\\partial L}{\\partial x} = \\frac{\\partial L}{\\partial \\hat{y}} \\cdot \\frac{\\partial \\hat{y}}{\\partial z} \\cdot \\frac{\\partial z}{\\partial x}\n<\/div>\n
      \u7528\u524d\u9762\u7684\\delta<\/span>\u4ee3\u6362\u5c31\u662f<\/p>\n
      \n\\frac{\\partial L}{\\partial x} = \\delta \\frac{\\partial z}{\\partial x}\n<\/div>\n
      \u56e0\u4e3a\u795e\u7ecf\u5143\u6fc0\u6d3b\u51fd\u6570\u524d\u7684\u51fd\u6570\u662f\u7ebf\u6027\u7684\uff0c\u6240\u4ee5<\/p>\n
      \n\\frac{\\partial z}{\\partial x} = w\n<\/div>\n
      \u6240\u4ee5\u5355\u4e2a\u795e\u7ecf\u5143\u901a\u8fc7\u67d0\u4e2a\u6743\u91cd\uff0c\u5bf9\u5bf9\u5e94\u8f93\u5165\u4ea7\u751f\u7684\u68af\u5ea6\u8d21\u732e\u5c31\u662f\uff1a<\/p>\n
      \n\\frac{\\partial L}{\\partial x} = \\delta w\n<\/div>\n
      \u540c\u4e00\u4e2a\u8f93\u5165 x_j<\/span> \u4f1a\u8fde\u63a5\u5230\u8fd9\u4e00\u5c42\u7684\u591a\u4e2a\u795e\u7ecf\u5143\uff0c\u6240\u4ee5\u5b83\u6700\u7ec8\u7684\u68af\u5ea6\uff0c\u8981\u628a\u6240\u6709\u795e\u7ecf\u5143\u5bf9\u5b83\u7684\u68af\u5ea6\u8d21\u732e\u52a0\u8d77\u6765\u3002<\/p>\n
      \u90a3\u6362\u6210\u77e9\u9635\u600e\u4e48\u529e\u5462\uff1f<\/p>\n
      \u4e00\u4e2a\u4e00\u4e2a\u6392\u5217\u8bd5\u8bd5\u770b\u5457<\/p>\n
      \u6bd4\u5982\uff1a<\/p>\n
      \nW= \\begin{bmatrix} w_{11}&w_{12}&w_{13}\\\\ w_{21}&w_{22}&w_{23} \\end{bmatrix} \n<\/div>\n
      \n\\delta= \\begin{bmatrix} \\delta_1\\\\ \\delta_2 \\end{bmatrix}\n<\/div>\n
      \u90a3\u4e48\uff1a<\/p>\n
      \n\\frac{\\partial L}{\\partial x_1} = w_{11}\\delta_1+w_{21}\\delta_2 \n<\/div>\n
      \n\\frac{\\partial L}{\\partial x_2} = w_{12}\\delta_1+w_{22}\\delta_2\n<\/div>\n
      \n\\frac{\\partial L}{\\partial x_3} = w_{13}\\delta_1+w_{23}\\delta_2 \n<\/div>\n
      \u628a\u5b83\u5199\u6210\u77e9\u9635\uff1a<\/p>\n
      \u4e5f\u5c31\u662f\u645e\u5728\u4e00\u8d77\uff1a<\/p>\n
      \nW= \\begin{bmatrix} w_{11}&w_{12} & w_{13}\\\\w_{21}&w_{22}&w_{23} \\end{bmatrix}\n<\/div>\n
      \u60f3\u8981\u77e9\u9635\u4e58\u6cd5\u600e\u4e48\u529e\u5462\uff1f\u54b1\u4eec\u7684\u516c\u5f0f\u662f\u548c\\delta<\/span>\u76f8\u4e58\u3002<\/p>\n
      \u8bd5\u8bd5\u770b\uff1a<\/p>\n
      \nW\\delta \n<\/div>\n
      (2×3)(2×1)<\/p>\n
      \u554a\u5662\uff0c\u4e58\u4e0d\u4e86<\/p>\n
      \u8f6c\u7f6e\u8bd5\u8bd5\u770b\uff01<\/p>\n
      \nW^T = \\begin{bmatrix} w_{11}&w_{21}\\\\ w_{12}&w_{22}\\\\ w_{13}&w_{23} \\end{bmatrix}\n<\/div>\n
      (3×2)(2×1)<\/p>\n
      \u597d\u8036\uff01\u53ef\u4ee5\u5566\uff01\u6700\u540e\u4f1a\u5f97\u5230\u4e09\u884c\u4e00\u5217\u7684\u77e9\u9635<\/p>\n
      \u4e8e\u662f\uff1a<\/p>\n
      \n\\frac{\\partial L}{\\partial x}=W^T \\delta = \\begin{bmatrix} w_{11}\\delta_1+w_{21}\\delta_2\\\\ w_{12} \\delta_1+w_{22}\\delta_2\\\\ w_{13}\\delta_1+w_{23}\\delta_2 \\end{bmatrix}\n<\/div>\n
      \u53d1\u73b0\u4e86\u5417\uff01\u957f\u5f97\u8ddf\u8f93\u5165\u7684\u77e9\u9635\u4e00\u6837\uff01<\/p>\n
      \u81f3\u6b64\uff0c\u68af\u5ea6\u7684\u53cd\u5411\u4f20\u64ad\u5b8c\u6210\uff01<\/p>\n
      \u6765\u5199\u4ee3\u7801~~<\/p>\n
      Layer<\/code>\u7684backward<\/p>\n
      def backward(self, output_gradient, learning_rate):\n    # dhaty_dz = self.leaky_ReLU_derivative(self.z)\n    if self.activation == "sigmoid":\n        dhaty_dz = self.sigmoid_derivative(self.z)\n    else:\n        dhaty_dz = self.leaky_ReLU_derivative(self.z)\n    dL_dhaty = output_gradient\n\n    delta = dL_dhaty * dhaty_dz\n\n    dL_dW = delta @ self.x.T #.T\u8f6c\u7f6e\u77e9\u9635\n    dL_db = delta\n\n    #\u5728\u66f4\u65b0\u68af\u5ea6\u524d\u7b97\u51faLoss\u5bf9\u8d21\u732e\u7684\u5f71\u54cd\n    dL_dx = self.W.T @ delta\n\n    #\u68af\u5ea6\u4e0b\u964d\n    self.W -= learning_rate * dL_dW\n    self.b -= dL_db * learning_rate\n\n    return dL_dx\n<\/code><\/pre>\n
      Network<\/code>\u7684Backward<\/p>\n
      def backward(self, output_gradient, learning_rate):\n    gradient = output_gradient\n\n    # hhh\u6211\u4e0a\u6b21\u5199\u7adf\u7136\u5fd8\u4e86reversed\u8fd9\u4e2a\u51fd\u6570\n    for layer in reversed(self.layers):\n        gradient = layer.backward(gradient, learning_rate)\n\n    return gradient\n<\/code><\/pre>\n
      \u6309\u7167\u4e0a\u6b21\u5b9a\u7684\u795e\u7ecf\u5143\u5206\u5e03\uff0c\u5b9e\u4f8b\u5316\u795e\u7ecf\u7f51\u7edc<\/p>\n
      \n
      \u8fd9\u91cc\u6211\u8fd8\u8e29\u4e86\u4e2a\u5751\uff0c<\/p>\n
      ValueError: non-broadcastable output operand with shape (1,1) doesn't match the broadcast shape (1,2)\n<\/code><\/pre>\n
      \u56e0\u4e3a\u6211\u8fd8\u662f\u7528\u7684python\u9ed8\u8ba4\u7684\u5217\u8868\uff0c\u6ca1\u6709\u7528numpy<\/code>\u7684\u77e9\u9635<\/p>\n
      \u9700\u8981\u8f6c\u6362\u4e00\u4e0b<\/p>\n
      Layer<\/code>\u7684forward<\/code><\/p>\n
          def forward(self, x):\n        x = np.asarray(x, dtype=float).reshape(-1, 1) #\u8f6c\u6362\u4e3anumpy\u77e9\u9635\uff0c\u4e0d\u7ba1\u6709\u51e0\u4e2a\u5143\u7d20\uff0c\u90fd\u7ed9\u6211\u6574\u7406\u6210n\u884c1\u5217\n        self.x = x #\u4f9d\u65e7\u4fdd\u5b58\u8f93\u5165\n\n        # \u6309\u7167\u54b1\u4eec\u7684\u77e9\u9635\u4e58\u6cd5\u516c\u5f0f\uff0cnumpy\u91cc\u4f7f\u7528@\u8fdb\u884c\u77e9\u9635\u4e58\u6cd5\n        self.z = self.W @ x + self.b\n\n        #\u6fc0\u6d3b\u51fd\u6570\n        self.haty = self.leaky_ReLU(self.z)\n\n        return self.haty\n<\/code><\/pre>\n<\/blockquote>\n
      \u5b9e\u4f8b\u5316\uff1a<\/p>\n
      train_data = [\n    (1,1,0),\n    (0,1,1),\n    (1,0,1),\n    (0,0,0)\n]\n\nnetwork = Network([\n    Layer(2, 2),\n    Layer(3, 2),\n    Layer(2, 3),\n    Layer(1, 2)\n])\n\nfor epoch in range(20000):\n    for simple in range(len(train_data)):\n        y = train_data[simple][2]\n        haty = network.forward([train_data[simple][0],train_data[simple][1]])\n        # \u56e0\u4e3aforward\u6700\u540e\u8fd4\u56de\u7684\u662f\u6bcf\u4e2a\u795e\u7ecf\u5143\u7684\u8f93\u51fa\uff0c\u7136\u540e\u6700\u540e\u4e00\u5c42\u5c31\u4e00\u4e2a\u795e\u7ecf\u5143\uff0c\u6240\u4ee5\u7528[0]\n        dL_dhaty = 2*(haty[0] - y)\n        network.backward(dL_dhaty,0.01)\n\nprint(network.forward([train_data[0][0], train_data[0][1]]))\nprint(network.forward([train_data[1][0], train_data[1][1]]))\nprint(network.forward([train_data[2][0], train_data[2][1]]))\nprint(network.forward([train_data[3][0], train_data[3][1]]))\n<\/code><\/pre>\n
      \u8bad\u7ec3\u540e\u7ed3\u679c\uff1a<\/p>\n
      [[-0.00797035]]\n[[-0.00797016]]\n[[-0.00797033]]\n[[-0.00797034]]\n<\/code><\/pre>\n
      omg\u73b0\u5728\u9700\u8981\u7528Sigmoid<\/code>\u9644\u5c5e\u7ed9\u8f93\u51fa\u5c42\u4e86\uff0c\u8ddf\u4e0a\u4e00\u7bc7\u535a\u5ba2\u4e00\u6837<\/p>\n
      class Layer:\n    def __init__(self, neuron_num, weights_num, activation="relu"):\n        # \u521d\u59cb\u5316\u795e\u7ecf\u5143\u77e9\u9635\uff0crandom.randn\u662f\u751f\u6210\u884cx\u5217\u7684\u77e9\u9635\uff0c\u91cc\u9762\u7684\u6570\u503c\u968f\u673a\n        # \u56e0\u4e3a\u6bcf\u4e2a\u8f93\u5165\u8981\u548c\u6bcf\u4e2a\u6743\u91cd\u70b9\u4e58\uff0c\u53c8\u56e0\u4e3a\u77e9\u9635\u4e58\u6cd5\u662f\u884c\u70b9\u4e58\u5217\uff0c\u6240\u4ee5\u6bcf\u4e2a\u795e\u7ecf\u5143\u7684\u6bcf\u4e2a\u6743\u91cd\u5b58\u5728\u77e9\u9635\u7684\u884c\u4e2d\n        # \u7136\u540e\u6bcf\u884c\u91cc\u7684\u6bcf\u4e2a\u6743\u91cd\u5c31\u53ef\u4ee5\u548c\u8f93\u5165\uff08\u770b\u505a\u591a\u884c1\u5217\u7684\u77e9\u9635\uff09\u70b9\u4e58\u5566~\n        self.W = np.random.randn(neuron_num, weights_num)\n\n        # \u521b\u5efa\u4e00\u4e2aneuron_num\u884c\uff0c1\u5217\u7684\u77e9\u9635\uff0c\u4e5f\u5c31\u662f\u54b1\u4eec\u7684\u504f\u7f6e\u77e9\u9635\uff01\n        # \u504f\u7f6e\u77e9\u9635\u90fd\u586b\u51450\u5c31\u597d\uff0c\u504f\u7f6e\u521d\u59cb\u503c\u8bbe\u7f6e\u4e3a0\u66f4\u9ad8\u6548\n        self.b = np.zeros((neuron_num,1)) \n        self.activation = activation #\u6fc0\u6d3b\u51fd\u6570\u53ef\u4ee5\u8bbe\u5b9a\u6a21\u5f0f\u4e86\n\n    def leaky_ReLU(self, x):\n        # np.where\u53ef\u4ee5\u7406\u89e3\u4e3a\u6279\u91cf1f\n        # np.where(\u6761\u4ef6, \u6761\u4ef6\u6210\u7acb\u65f6\u7684\u503c, \u6761\u4ef6\u4e0d\u6210\u7acb\u65f6\u7684\u503c)\n        return np.where(x > 0, x, 0.01*x)\n\n    def leaky_ReLU_derivative(self, z):\n        return np.where(z > 0, 1, 0.01)\n\n    def sigmoid(self, x):\n        return 1 \/ (1 + np.exp(-x))\n\n    def sigmoid_derivative(self, x):\n        s = self.sigmoid(x)\n        return s * (1 - s)\n\n    def forward(self, x):\n        x = np.asarray(x, dtype=float).reshape(-1, 1) #\u8f6c\u6362\u4e3anumpy\u77e9\u9635\uff0c\u4e0d\u7ba1\u6709\u51e0\u4e2a\u5143\u7d20\uff0c\u90fd\u7ed9\u6211\u6574\u7406\u6210n\u884c1\u5217\n        self.x = x #\u4f9d\u65e7\u4fdd\u5b58\u8f93\u5165\n\n        # \u6309\u7167\u54b1\u4eec\u7684\u77e9\u9635\u4e58\u6cd5\u516c\u5f0f\uff0cnumpy\u91cc\u4f7f\u7528@\u8fdb\u884c\u77e9\u9635\u4e58\u6cd5\n        self.z = self.W @ x + self.b\n\n        #\u6fc0\u6d3b\u51fd\u6570\n        if self.activation == "sigmoid":\n            self.haty = self.sigmoid(self.z)\n        else:\n            self.haty = self.leaky_ReLU(self.z)\n\n        return self.haty\n\n    def backward(self, output_gradient, learning_rate):\n        # dhaty_dz = self.leaky_ReLU_derivative(self.z)\n        if self.activation == "sigmoid":\n            dhaty_dz = self.sigmoid_derivative(self.z)\n        else:\n            dhaty_dz = self.leaky_ReLU_derivative(self.z)\n        dL_dhaty = output_gradient\n\n        delta = dL_dhaty * dhaty_dz\n\n        dL_dW = delta @ self.x.T #.T\u8f6c\u7f6e\u77e9\u9635\n        dL_db = delta\n\n        #\u5728\u66f4\u65b0\u68af\u5ea6\u524d\u7b97\u51faLoss\u5bf9\u8d21\u732e\u7684\u5f71\u54cd\n        dL_dx = self.W.T @ delta\n\n        #\u68af\u5ea6\u4e0b\u964d\n        self.W -= learning_rate * dL_dW\n        self.b -= dL_db * learning_rate\n\n        return dL_dx\n<\/code><\/pre>\n
      \u7f51\u7edc\u5b9e\u4f8b\u5316\uff1a<\/p>\n
      network = Network([\n    Layer(2, 2),\n    Layer(3, 2),\n    Layer(2, 3),\n    Layer(1, 2, activation="sigmoid")\n])\n<\/code><\/pre>\n
      \u8bad\u7ec3\u7ed3\u679c\uff1a<\/p>\n
      [[0.00244575]]\n[[0.98330621]]\n[[0.98220073]]\n[[0.00799492]]\n<\/code><\/pre>\n
      \u5b9e\u6d4b\u4e0b\u6765\u6210\u529f\u7387\u8fd8\u662f\u7a0d\u4f4e\uff0c\u4e4b\u540e\u4f1a\u597d\u597d\u5b66\u4e60\u521d\u59cb\u5316\u7684\uff01<\/p>\n
      \u90a3\u4eca\u5929\u5c31\u5148\u5230\u8fd9\u91cc\u5566~\u795d\u8d3a99mlp\u7ec8\u4e8e\u5347\u7ea7\u4e3a\u4e86\u77e9\u9635\u7248\uff01\uff01\uff01\uff01<\/p>\n","protected":false},"excerpt":{"rendered":"
      Attention Is All You Need #2\uff1a\u4ece\u795e\u7ecf\u5143\u5230\u77e9\u9635 OK\uff01\u73b0\u5728\u662f2026\u5e7410\u67085\u65e5\uff0c\u624b\u5934\u8fd8\u6709\u4e00\u4e2a\u79d1\u521b\u6bd4\u8d5b\u3002\u3002\u3002\u3002\u597d\u7d2f\u3002\u3002\u3002 \u8fd9\u7bc7\u535a\u5ba2\u7684\u76ee\u7684\u662f\u628a\u4e0a\u4e00\u7bc7\u535a\u5ba2\u7684MLP\u4ece\u5355\u4e2a\u795e\u7ecf\u5143\u904d\u5386\u5347\u7ea7\u6210\u77e9\u9635\u8ba1\u7b97\uff01 \u53ef\u662f\u3002\u3002\u3002\u4ec0\u4e48\u662f\u77e9\u9635\u554a \u524d\u7f6e\u77e5\u8bc6 \u77e9\u9635 GPT\u8bb2\u5230 \u6bd4\u5982\u4e00\u5c42\u6709 3 \u4e2a\u8f93\u5165\u30012 \u4e2a\u795e\u7ecf\u5143\u3002 \u4ee5\u524d\u4f60\u53ef\u80fd\u5199\uff1a \u795e\u7ecf\u51431: w11, w12, w13 \u795e\u7ecf\u51432: w21, w22, w […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[25,23,26,24],"tags":[],"class_list":["post-169","post","type-post","status-publish","format-standard","hentry","category-ai","category-transformer","category-26","category-24"],"_links":{"self":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts\/169","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=169"}],"version-history":[{"count":4,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts\/169\/revisions"}],"predecessor-version":[{"id":173,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=\/wp\/v2\/posts\/169\/revisions\/173"}],"wp:attachment":[{"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=169"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=169"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/99website.top\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=169"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}