{"id":149,"date":"2020-07-21T08:05:58","date_gmt":"2020-07-21T00:05:58","guid":{"rendered":"http:\/\/www.gaoxigang.com\/?p=149"},"modified":"2020-07-21T08:05:58","modified_gmt":"2020-07-21T00:05:58","slug":"scrapy-%e7%88%ac%e8%99%ab%e6%a1%86%e6%9e%b6-%e5%9f%ba%e4%ba%8eip%e7%9a%84%e4%bc%aa%e8%a3%85","status":"publish","type":"post","link":"https:\/\/www.gaoxigang.com\/index.php\/2020\/07\/21\/scrapy-%e7%88%ac%e8%99%ab%e6%a1%86%e6%9e%b6-%e5%9f%ba%e4%ba%8eip%e7%9a%84%e4%bc%aa%e8%a3%85\/","title":{"rendered":"Scrapy \u722c\u866b\u6846\u67b6-\u57fa\u4e8eIP\u7684\u4f2a\u88c5"},"content":{"rendered":"\n<h2 class=\"wp-block-heading\">\u722c\u866b\u7684\u4f2a\u88c5<\/h2>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\"><p>\u5982\u679c\u4e0d\u8fdb\u884c\u4f2a\u88c5\u5219\u6211\u4eec\u6bcf\u6b21\u91c7\u7528\u76f8\u540cIP\u6293\u53d6\u6570\u636e\u65f6\u53ef\u4ee5\u4f1a\u88ab\u76ee\u524d\u670d\u52a1\u5668\u7684\u9632\u706b\u5899\u4e4b\u522b\uff0c\u4f2a\u88c5\u6709\u4e24\u79cd\uff1a\u914d\u7f6e\u4ee3\u7406IP\u548cuser-agent\u4e2d\u95f4\u4ef6\u7f16\u5199,\u9700\u8981\u5148\u6ce8\u518c<a href=\"https:\/\/www.abuyun.com\/\">\u963f\u5e03\u4e91<\/a><\/p><\/blockquote>\n\n\n\n<h2 class=\"wp-block-heading\">\u52a8\u6001IP\u63a5\u5165\u6307\u5357<\/h2>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\"><p>\u6ce8\u518c\u963f\u5e03\u4e91\u4e4b\u540e\uff0c\u53ef\u4ee5\u9009\u62e91\u5143\u8d2d\u4e701\u5c0f\u65f6\u8fdb\u884c\u52a8\u6001IP\u7684\u6d4b\u8bd5\u3002\u5982\u679c\u8d2d\u4e70\u6210\u529f\u6253\u5f00\u5bf9\u5e94\u7684\u63a5\u5165\u6307\u5357\u4f1a\u6709\u63d0\u793ascrapy\u7684\u76f8\u5173\u914d\u7f6e<\/p><\/blockquote>\n\n\n\n<pre class=\"EnlighterJSRAW\" data-enlighter-language=\"python\" data-enlighter-theme=\"\" data-enlighter-highlight=\"\" data-enlighter-linenumbers=\"\" data-enlighter-lineoffset=\"\" data-enlighter-title=\"\" data-enlighter-group=\"\">import base64\n\n    # \u4ee3\u7406\u670d\u52a1\u5668\n    proxyServer = \"http:\/\/http-dyn.abuyun.com:9020\"\n\n    # \u4ee3\u7406\u96a7\u9053\u9a8c\u8bc1\u4fe1\u606f\n    proxyUser = \"H01234567890123D\"\n    proxyPass = \"0123456789012345\"\n\n    # for Python2\n    proxyAuth = \"Basic \" + base64.b64encode(proxyUser + \":\" + proxyPass)\n\n    # for Python3\n    #proxyAuth = \"Basic \" + base64.urlsafe_b64encode(bytes((proxyUser + \":\" + proxyPass), \"ascii\")).decode(\"utf8\")\n\n    class ProxyMiddleware(object):\n        def process_request(self, request, spider):\n            request.meta[\"proxy\"] = proxyServer\n\n            request.headers[\"Proxy-Authorization\"] = proxyAuth              \n            \n<\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">IP\u4ee3\u7406\u4e2d\u95f4\u4ef6\u7f16\u5199<\/h2>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\"><p>\u6839\u636e\u4e0a\u9762\u7684\u63a5\u5165\u6307\u5357\uff0c\u91c7\u7528\u521b\u5efa\u4e00\u4e2aProxyMiddleware\u914d\u7f6e\u76f8\u5173\u7684\u4fe1\u606f\u5373\u53ef\u5b8c\u6210\u52a8\u6001IP\u7684\u914d\u7f6e<\/p><\/blockquote>\n\n\n\n<pre class=\"EnlighterJSRAW\" data-enlighter-language=\"python\" data-enlighter-theme=\"\" data-enlighter-highlight=\"\" data-enlighter-linenumbers=\"\" data-enlighter-lineoffset=\"\" data-enlighter-title=\"\" data-enlighter-group=\"\">import base64\n\n# \u4ee3\u7406\u670d\u52a1\u5668\nproxyServer = \"http:\/\/http-dyn.abuyun.com:9020\"\n# \u4ee3\u7406\u96a7\u9053\u9a8c\u8bc1\u4fe1\u606f\nproxy_name_pass = b\"HH59908195O5720D:4B4748D2DBD1B53D\"\n# for Python2\nproxyAuth = base64.b64encode(proxy_name_pass)\n\n\nclass ProxyMiddleware(object):\n\n    def process_request(self, request, spider):\n        request.meta[\"proxy\"] = proxyServer\n        request.headers[\"Proxy-Authorization\"] = \"Basic \" + proxyAuth.decode()\n\n<\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">Setting\u4e2d\u914d\u7f6eMiddleware<\/h2>\n\n\n\n<pre class=\"EnlighterJSRAW\" data-enlighter-language=\"python\" data-enlighter-theme=\"\" data-enlighter-highlight=\"\" data-enlighter-linenumbers=\"\" data-enlighter-lineoffset=\"\" data-enlighter-title=\"\" data-enlighter-group=\"\">DOWNLOADER_MIDDLEWARES = {\n   # \u672a\u6765\u5b8c\u6210ajax\u52a0\u8f7d\n   'douban.middlewares.DoubanDownloaderMiddleware': 544,\n   # IP\u4f2a\u88c5\n   'douban.proxymiddlewares.ProxyMiddleware': 542,\n   # User-Agent\u4f2a\u88c5\n   'douban.user_agent_middlewares.UserAgentMiddleware': 543,\n}\n<\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u722c\u866b\u7684\u4f2a\u88c5 \u5982\u679c\u4e0d\u8fdb\u884c\u4f2a\u88c5\u5219\u6211\u4eec\u6bcf\u6b21\u91c7\u7528\u76f8\u540cIP\u6293\u53d6\u6570\u636e\u65f6\u53ef\u4ee5\u4f1a\u88ab\u76ee\u524d\u670d\u52a1\u5668\u7684\u9632\u706b\u5899\u4e4b\u522b\uff0c\u4f2a\u88c5\u6709\u4e24\u79cd\uff1a\u914d\u7f6e\u4ee3\u7406 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[4],"tags":[],"class_list":["post-149","post","type-post","status-publish","format-standard","hentry","category-biji"],"_links":{"self":[{"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/posts\/149","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/comments?post=149"}],"version-history":[{"count":0,"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/posts\/149\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/media?parent=149"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/categories?post=149"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.gaoxigang.com\/index.php\/wp-json\/wp\/v2\/tags?post=149"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}