def scrape(url, headers, method, num=5, payload=None):
if num == 0: return
proxies = random_ip()
try:
if method == 'get':
ret = requests.get(url, headers=headers, timeout=3, proxies=proxies, allow_redirects=False)
else:
ret = requests.post(url, headers=headers, json=payload, timeout=3, proxies=proxies, allow_redirects=False)
if ret.status_code == requests.codes.ok:
ret.encoding = 'utf-8'
print(ret.text)
return ret.text
else:
print(f'状态码为{ret.status_code}')
print(ret.text)
assert False
except Exception as e:
print(f'出错了,还有{num}次', e)
num -= 1
return scrape(url, headers, num)
版权归属:
海绵
许可协议:
本文使用《署名-非商业性使用-相同方式共享 4.0 国际 (CC BY-NC-SA 4.0)》协议授权
评论区