banner
aying

aying

Time Flies,Have Fun!
github
bilibili
douban
首頁封存

requests庫的使用

#Python3WebSpider37
AI 翻譯
這篇文章透過AI由英文翻譯成繁體中文。查看原文
AI 生成的摘要
這段程式碼展示了如何使用 Python 的 `requests` 庫進行各種 HTTP 請求，包括 GET、POST、PUT 和 DELETE 請求。首先，程式碼示範了如何發送 GET 請求並打印回應的狀態碼、內容和 cookies。接著，展示了如何添加查詢參數到 GET 請求中，並解析 JSON 格式的回應。 此外，程式碼還展示了如何抓取網頁內容（例如知乎的探索頁面）和二進制數據（如 GitHub 的圖標），並將其保存到本地。對於 POST 請求，程式碼示範了如何發送表單數據並打印回應內容。 最後，程式碼介紹了如何使用 cookies 維持登錄狀態，並使用會話來管理 cookies，以及如何進行 SSL 證書驗證。
import requests

r = requests.get('https://www.baidu.com/')
print(type(r))
print(r.status_code)
print(type(r.text))
print(r.text)
print(r.cookies)

#發送其他請求
r = requests.post('http://httpbin.org/post')  
r = requests.put('http://httpbin.org/put')  
r = requests.delete('http://httpbin.org/delete')  
r = requests.head('http://httpbin.org/get')  
r = requests.options('http://httpbin.org/get')

#GET請求
#基本實例
r = requests.get('http://httpbin.org/get')
print(r.text)

{
  "args": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate, br, zstd", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.31.0", 
    "X-Amzn-Trace-Id": "Root=1-67c2cfc9-197bdfa835c331751229e13d"
  }, 
  "origin": "154.40.60.12", 
  "url": "http://httpbin.org/get"
}


#添加參數
data = {
    'name': 'germey',  
    'age': 22 
}
r = requests.get("http://httpbin.org/get",params=data)
print(r.text)

{
  "args": {
    "age": "22", 
    "name": "germey"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate, br, zstd", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.31.0", 
    "X-Amzn-Trace-Id": "Root=1-67c2d039-06fc3a62021e926051aec502"
  }, 
  "origin": "154.40.60.12", 
  "url": "http://httpbin.org/get?name=germey&age=22"
}


#網頁的返回類型實際上是 str 類型，但是是 JSON 格式的。想直接解析返回結果，得到一個字典格式的話，可以直接調用 json 方法
import requests  

r = requests.get("http://httpbin.org/get")  
print(type(r.text))  
print(r.json())  
print(type(r.json()))

<class 'str'>
{'args': {}, 'headers': {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Host': 'httpbin.org', 'User-Agent': 'python-requests/2.31.0', 'X-Amzn-Trace-Id': 'Root=1-67c2d074-4c51486a68ce721962b679a3'}, 'origin': '154.40.60.12', 'url': 'http://httpbin.org/get'}
<class 'dict'>

#抓取網頁 以"知乎"-"發現"頁面為例,代碼不適用
import re
#添加headers
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
r = requests.get("https://www.zhihu.com/explore", headers=headers)
#正則表達式來匹配出所有的問題內容
pattern = re.compile('question.*?question_link.*?>(.*?)</a>', re.S)
titles = re.findall(pattern,r.text)
print(titles)

#抓取二進制數據 以github站點圖標為例
import requests

r = requests.get("https://github.com/favicon.ico")
#print(r.text)
#print(r.content)
#保存
with open('favicon.ico', 'wb') as f:
    f.write(r.content)

#POST請求
data = {'name':'germey','age':'22' }
r = requests.post("http://httpbin.org/post",data = data)
print(r.text)

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "age": "22", 
    "name": "germey"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate, br, zstd", 
    "Content-Length": "18", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.31.0", 
    "X-Amzn-Trace-Id": "Root=1-67c2fd51-785a04c7193dac8e4f366d8a"
  }, 
  "json": null, 
  "origin": "154.40.60.12", 
  "url": "http://httpbin.org/post"
}


# requests  還提供了一個內置的狀態碼查詢對象 requests.codes
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
r = requests.get('http://www.jianshu.com',headers=headers)
exit() if not r.status_code == requests.codes.ok else print('Request Successfully')

Request Successfully

#高級用法
#1.文件上傳
files = {'file':open('favicon.ico','rb')}
r = requests.post('http://httpbin.org/post',files = files)
#print(r.text)
#獲取 Cookies
r = requests.get('https://www.baidu.com')
print(r.cookies)
for key,value in r.cookies.items():
    print(key+'='+value)

<RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>
BDORZ=27315

#用cookie來維持登錄狀態
import requests

headers = {
    'cookie':'_xsrf=lFnzNR23XYlIWOlqH7giKRfH0z0aeqlS; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1740900296',
    'Host':'www.zhihu.com',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'
}

r = requests.get('https://www.zhihu.com',headers=headers)
print(r.text)

#會話維持
s = requests.Session()
#請求測試網址，可以設置一個 cookie
s.get('http://httpbin.org/cookies/set/number/123456789')
#再次請求獲取cookie
r  = s.get('http://httpbin.org/cookies')
print(r.text)

{
  "cookies": {
    "number": "123456789"
  }
}


#SSL 證書驗證
#當發送 HTTP 請求的時候，它會檢查 SSL 證書，我們可以使用 verify 參數控制是否檢查此證書
'''response = requests.get('https://www.12306.cn', verify=False)
print(response.status_code)'''
#忽略警告
from requests.packages import urllib3
urllib3.disable_warnings()
response = requests.get('https://www.12306.cn', verify=False)
print(response.status_code)
#捕獲警告到日誌的方式忽略警告
import logging
logging.captureWarnings(True)
response = requests.get('https://www.12306.cn', verify=False)
print(response.status_code)

200
200

#代理設置
# 提取代理API接口，獲取1個代理IP
api_url = "https://dps.kdlapi.com/api/getdps/?secret_id=******&signature=****&num=1&pt=1&sep=1"

# 獲取API接口返回的代理IP
proxy_ip = requests.get(api_url).text

# 用戶名密碼認證(私密代理/獨享代理)
username = "*******"
password = "********"
proxies = {
    "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": proxy_ip},
    "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": proxy_ip}
}
print(proxies)
requests.get('https://www.taobao.com', proxies=proxies)

{'http': 'http://d4785269730:4bkl8hdn@218.95.37.135:41577/', 'https': 'http://d4785269730:4bkl8hdn@218.95.37.135:41577/'}





<Response [200]>

#超時設置
#將超時時間設置為 1 秒
r = requests.get('https://www.taobao.com', timeout=1)
print(r.status_code)       
# timeout 是連接和讀取這二者的 timeout 總和,可以分別傳入
r = requests.get('https://www.taobao.com', timeout=(5, 30))
#永久等待，可以直接將 timeout 設置為 None
r = requests.get('https://www.taobao.com', timeout=None)

200

#身份認證
from requests.auth import HTTPBasicAuth
#r = requests.get('http://127.0.0.1:5244/',auth=HTTPBasicAuth('admin','Hyy823237'))
#默認使用 HTTPBasicAuth 這個類來認證
r = requests.get('http://127.0.0.1:5244/',auth=('admin','Hyy823237'))
print(r.status_code)

200

#Prepared Request
from requests import Request, Session

url = 'http://httpbin.org/post'
data = {'name': 'germey'}
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'
}
s = Session()
req = Request('POST', url, data=data, headers=headers)
prepped = s.prepare_request(req)
r = s.send(prepped)
print(r.text)

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "name": "germey"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate, br, zstd", 
    "Content-Length": "11", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36", 
    "X-Amzn-Trace-Id": "Root=1-67c411e2-5e4075fc632d6ba24a26c1a3"
  }, 
  "json": null, 
  "origin": "154.40.60.12", 
  "url": "http://httpbin.org/post"
}
載入中......
此文章數據所有權由區塊鏈加密技術和智能合約保障僅歸創作者所有。