pip install requests
安装小测
import requests
r=requests.get("http://www.baidu.com")
print(r.status_code)
爬取网页通用框架
#在我电脑上没实验成功
import requests
def getHTMLText(url):
try:
r=requests.get(url,timeout=30)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except:
return "产生异常"
if __name__=="_main_":
url="www.baidu.com"
print(getHTMLText(url))
遵守robots协议
京东商品页面的爬取
import requests
url="https://item.jd.com/100004815031.html"
try:
r=requests.get(url)
r.raise_for_status()
r.encoding=r.apparent_encoding
print(r.text[:1000])
except:
print("爬取失败")
亚马逊商品，我的失败了
分步
import requests
r=requests.get("http://www.amazon.cn/gp/product/B01M8L5Z3Y")
>>> r.status_code
503
>>> r.encding
Traceback (most recent call last):
File "<pyshell#29>", line 1, in <module>
r.encding
AttributeError: 'Response' object has no attribute 'encding'
>>> r.encoding
'ISO-8859-1'
>>> r.encoding=r.apparent_encoding
>>> r.text
'<!DOCTYPE html>\n<!--[if lt IE 7]> <html lang="zh-CN" class="a-no-js a-lt-ie9 a-lt-ie8 a-lt-ie7"> <![endif]-->\n<!--[if IE 7]> <html lang="zh-CN" class="a-no-js a-lt-ie9 a-lt-ie8"> <![endif]-->\n<!--[if IE 8]> <html lang="zh-CN" class="a-no-js a-lt-ie9"> <![endif]-->\n<!--[if gt IE 8]><!-->\n<html class="a-no-js" lang="zh-CN"><!--<![endif]--><head>\n<meta http-equiv="content-type" content="text/html; charset=UTF-8">\n<meta charset="utf-8">\n<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">\n<title dir="ltr">Amazon CAPTCHA</title>\n<meta name="viewport" content="width=device-width">\n<link rel="stylesheet" href="https://images-na.ssl-images-amazon.com/images/G/01/AUIClients/AmazonUI-3c913031596ca78a3768f4e934b1cc02ce238101.secure.min._V1_.css">\n<script>\n\nif (true === true) {\n var ue_t0 = (+ new Date()),\n ue_csm = window,\n ue = { t0: ue_t0, d: function() { return (+new Date() - ue_t0); } },\n ue_furl = "fls-cn.amazon.cn",\n ue_mid = "AAHKV2X7AFYLW",\n ue_sid = (document.cookie.match(/session-id=([0-9-]+)/) || [])[1],\n ue_sn = "opfcaptcha.amazon.cn",\n ue_id = \'CK0JAMS4EZ89132KXB5G\';\n}\n</script>\n</head>\n<body>\n\n<!--\n To discuss automated access to Amazon data please contact api-services-support@amazon.com.\n For information about migrating to our APIs refer to our Marketplace APIs at https://developer.amazonservices.com.cn/index.html/ref=rm_c_sv, or our Product Advertising API at https://associates.amazon.cn/gp/advertising/api/detail/main.html/ref=rm_c_ac for advertising use cases.\n-->\n\n<!--\nCorreios.DoNotSend\n-->\n\n<div class="a-container a-padding-double-large" style="min-width:350px;padding:44px 0 !important">\n\n <div class="a-row a-spacing-double-large" style="width: 350px; margin: 0 auto">\n\n <div class="a-row a-spacing-medium a-text-center"><i class="a-icon a-logo"></i></div>\n\n <div class="a-box a-alert a-alert-info a-spacing-base">\n <div class="a-box-inner">\n <i class="a-icon a-icon-alert"></i>\n <h4>请输入您在下方看到的字符</h4>\n <p class="a-last">抱歉，我们只是想确认一下当前访问者并非自动程序。为了达到最佳效果，请确保您浏览器上的 Cookie 已启用。</p>\n </div>\n </div>\n\n <div class="a-section">\n\n <div class="a-box a-color-offset-background">\n <div class="a-box-inner a-padding-extra-large">\n\n <form method="get" action="/errors/validateCaptcha" name="">\n <input type=hidden name="amzn" value="rK7z73i3swEGN3UkqJHp6A==" /><input type=hidden name="amzn-r" value="/gp/product/B01M8L5Z3Y" />\n <div class="a-row a-spacing-large">\n <div class="a-box">\n <div class="a-box-inner">\n <h4>请输入您在这个图片中看到的字符：</h4>\n <div class="a-row a-text-center">\n <img src="https://images-na.ssl-images-amazon.com/captcha/bysppkyq/Captcha_dzafkozxdq.jpg">\n </div>\n <div class="a-row a-spacing-base">\n <div class="a-row">\n <div class="a-column a-span6">\n <label for="captchacharacters">输入字符</label>\n </div>\n <div class="a-column a-span6 a-span-last a-text-right">\n <a οnclick="window.location.reload()">换一张图</a>\n </div>\n </div>\n <input autocomplete="off" spellcheck="false" id="captchacharacters" name="field-keywords" class="a-span12" autocapitalize="off" autocorrect="off" type="text">\n </div>\n </div>\n </div>\n </div>\n\n <div class="a-section a-spacing-extra-large">\n\n <div class="a-row">\n <span class="a-button a-button-primary a-span12">\n <span class="a-button-inner">\n <button type="submit" class="a-button-text">继续购物</button>\n </span>\n </span>\n </div>\n\n </div>\n </form>\n\n </div>\n </div>\n\n </div>\n\n </div>\n\n <div class="a-divider a-divider-section"><div class="a-divider-inner"></div></div>\n\n <div class="a-text-center a-spacing-small a-size-mini">\n <a href="https://www.amazon.cn/gp/help/customer/display.html/ref=footer_claim?ie=UTF8&nodeId=200347160">使用条件</a>\n <span class="a-letter-space"></span>\n <span class="a-letter-space"></span>\n <span class="a-letter-space"></span>\n <span class="a-letter-space"></span>\n <a href="https://www.amazon.cn/gp/help/customer/display.html/ref=footer_privacy?ie=UTF8&nodeId=200347130">隐私声明</a>\n </div>\n\n <div class="a-text-center a-size-mini a-color-secondary">\n © 1996-2015, Amazon.com, Inc. or its affiliates\n <script>\n if (true === true) {\n document.write(\'<img src="https://fls-cn.amaz\'+\'on.cn/\'+\'1/oc-csi/1/OP/requestId=CK0JAMS4EZ89132KXB5G&js=1" />\');\n };\n </script>\n <noscript>\n <img src="https://fls-cn.amazon.cn/1/oc-csi/1/OP/requestId=CK0JAMS4EZ89132KXB5G&js=0" />\n </noscript>\n </div>\n </div>\n <script>\n if (true === true) {\n var elem = document.createElement("script");\n elem.src = "https://images-cn.ssl-images-amazon.com/images/G/01/csminstrumentation/csm-captcha-instrumentation.min._V" + (+ new Date()) + "_.js";\n document.getElementsByTagName(\'head\')[0].appendChild(elem);\n }\n </script>\n</body></html>\n'
>>> r.request.headers
{'User-Agent': 'python-requests/2.22.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
>>> kv={'user-agent':'Mozilla/5.0'}
>>> url="https://www.amazon.cn/gp/product/B01M8L5Z3Y"
>>> r=requests.get(url,headers={'user-agent':'Mozilla/5.0'})
>>> r.status_code
200
>>> r.request.headers
{'user-agent': 'Mozilla/5.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
>>> r.text[:1000]
'\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n\n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n \n\n\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n\n\n\n\n <!doctype html><html class="a-no-js" data-19ax5a9jf="dingo">\n <head>\n<script type="text/javascript">var ue_t0=ue_t0||+new Date();</script>\n<script type="text/javascript">\nwindow.ue_ihb = (window.ue_ihb || window.ueinit || 0) + 1;\nif (window.ue_ihb === 1) {\nvar ue_hob=+new Date();\nvar ue_id=\'56XCQRCXERFXDRSA5FVV\',\nue_csm = window,\nue_err_chan = \'jserr-rw\',\nue = {};\n(function(d){var e=d.ue=d.ue||{},f=Date.now||function(){return+new Date};e.d=function(b){return f()-(b?0:d.ue_t0)};e.stub=function(b,a){if(!b[a]){var c=[];b[a]=function(){c.push([c.slice.call(arguments),e.d(),d.ue_id])};b[a].replay=function(b){for(var a;a=c.shift();)b(a[0],a[1],a[2])};b[a].isStub=1}};e.exec=function(b,a){return function(){try{return b.apply(this,arguments)}catch(c){ueLogError(c,{attribution:a||"undefined",logLevel:"WARN"})}}}})(ue_csm);\n\nue.stub(ue,"log");ue.stub(ue,"o'
>>>
全部代码，我的失败了
import requests
url="https://www.amazon.cn/gp/product/B01M8L5Z3Y"
try:
kv={'user-agent':'Mozilla/5.0'}
r=requests.get(url,hesders=kv)
r.raise_for_status()
r.encoding=r.apparent_encoding
print(r.text[1000:2000])
except:
print("爬取失败")
#爬取失败
百度 360关键字提交
import requests
kv={'wd':'python'}
r=requests.get("http://www.baidu.com/s",params=kv)
百度搜索全代码
import requests
keyword="Python"
try:
kv={'kw':keyword}
r=requests.get("http://www.baidu.com",params=kv)#！！！这里和PPT上代码不相同,y应该时没有/s
print(r.request.url)
r.raise_for_status()
print(len(r.text))
except:
print("爬取失败")
结果
http://www.baidu.com/?kw=Python
2381
360搜索全代码
/s有无代表意义不一样结果也不一样
1 有/s
import requests
keyword="Python"
try:
kv={'q':keyword}
r=requests.get("http://www.so.com/s",params=kv)
print(r.request.url)
r.raise_for_status()
print(len(r.text))
except:
print("爬取失败")
结果
https://www.so.com/s?q=Python
359659
2 无/s
import requests
keyword="Python"
try:
kv={'q':keyword}
r=requests.get("http://www.so.com",params=kv)
print(r.request.url)
r.raise_for_status()
print(len(r.text))
except:
print("爬取失败")
结果
https://www.so.com/?q=Python
315170
网络图片的爬取和存储
import requests
import os
#这个代码应该是只能爬取html网页上的图片
url="http://www.ngchina.com.cn/photography/photo_of_the_day/5470.html"
#保存位置
root="D:/picss//"
path=root+url.split('/')[-1]
try:
if not os.path.exists(root):
os.mkdir(root)
if not os.path.exists(path):
r=requests.get(url)
with open(path,'wb') as f:
f.write(r.content)
f.close()
print("文件保存成功")
else:
print("文件已存在")
except:
print("爬取失败")
IP地址归属地的自动查询
import requests
import os
url= 'http://m.ip138.com/ip.asp?ip='
try:
r=requests.get(url+'202.204.80.112')
r.raise_for_status()
r.encoding=r.apparent_encoding
print(r.text[-500:])
except:
print("爬取失败")
来源：CSDN
作者：张一%
链接：https://blog.csdn.net/qq_43426335/article/details/104116210