python的几个http库速度比较
本帖最后由 元始天尊 于 2021-9-5 18:40 编辑## 环境准备
* python3环境，安装模块: httplib2 requests grequests pycurl
* 本地http服务器，如`python3 -m http.server 80`
``` Python
#! /usr/bin/env python3
# # -*- coding: utf-8 -*-
from gevent import monkey as curious_george
curious_george.patch_all(thread=False, select=False)
import time
import aiohttp
import asyncio
import grequests
import http.client
import httplib2
import pycurl
import requests
import urllib
import urllib3
N = 1000
TMOUT = 2
t1 = int(time.time() * 1000)
for i in range(N):
con = http.client.HTTPConnection("127.0.0.1", port=80, timeout=TMOUT)
con.request("GET", "/")
con.getresponse().read()
t2 = int(time.time() * 1000)
print("http.client %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
for i in range(N):
httplib2.Http(cache=None, timeout=TMOUT).request("http://127.0.0.1:80/", "GET")
t2 = int(time.time() * 1000)
print("httplib2 %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
for i in range(N):
requests.get("http://127.0.0.1:80/", timeout=TMOUT)
t2 = int(time.time() * 1000)
print("requests %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
for i in range(N):
urllib.request.urlopen("http://127.0.0.1:80/", timeout=TMOUT).read()
t2 = int(time.time() * 1000)
print("urllib %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
for i in range(N):
urllib3.PoolManager().request("GET", "http://127.0.0.1:80/", timeout=TMOUT)
t2 = int(time.time() * 1000)
print("urllib3 %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
async def test_aiohttp():
for i in range(N):
async with aiohttp.ClientSession() as session:
async with session.get("http://127.0.0.1:80/") as response:
await response.text()
asyncio.get_event_loop().run_until_complete(test_aiohttp())
t2 = int(time.time() * 1000)
print("aiohttp %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
grequests.map()
t2 = int(time.time() * 1000)
print("grequests %dms for %d packet" % ((t2 - t1), N))
t1 = int(time.time() * 1000)
for i in range(N):
curl = pycurl.Curl()
curl.setopt(pycurl.CONNECTTIMEOUT, TMOUT)
curl.setopt(pycurl.TIMEOUT, TMOUT)
curl.setopt(pycurl.URL, "http://127.0.0.1:80/")
curl.setopt(pycurl.WRITEFUNCTION, lambda x: None)
curl.perform()
t2 = int(time.time() * 1000)
print("pycurl %dms for %d packet" % ((t2 - t1), N))
def get_with_pycurl(url, proxy=None):
curl = pycurl.Curl()
buff = io.BytesIO()
hdr = io.BytesIO()
if proxy:
if not proxy.startswith("http"):
proxy = "http://" + proxy
curl.setopt(pycurl.PROXY, proxy)
curl.setopt(pycurl.PROXYTYPE_HTTP, pycurl.PROXYTYPE_HTTP)
curl.setopt(pycurl.USERAGENT, tmp_ua)
curl.setopt(pycurl.CONNECTTIMEOUT, TMOUT)
curl.setopt(pycurl.TIMEOUT, TMOUT)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, buff.write)
curl.setopt(pycurl.WRITEHEADER, hdr)
try:
curl.perform()
curl.close()
except: # conn fail
return None, None
curl.close()
status = int(hdr.getvalue().decode().split("\r\n").split(" "))
body = buff.getvalue().decode()
return status, body
```
## 测试结果
|库|超时|
|-|-|
|pycurl|1314ms|
|http.client|1668ms|
|httplib2|1675ms|
|urllib3|1938ms|
|aiohttp|2189ms|
|urllib|2378ms|
|grequests|3299ms|
|requests|3985ms|
页:
[1]