def scrape(url, headers, num=3):
print(url)
if num == 0: return
# proxies = random_ip()
# 设置此处 8888为fiddler端口
proxies = {'http': 'http://localhost:8888', 'https':'http://localhost:8888'}
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
try:
sessions = requests.session()
sessions.keep_alive = False
ret = sessions.get(url, headers=headers, timeout=5,verify=False,proxies=proxies)
if ret.status_code == requests.codes.ok:
ret.encoding = 'utf-8'
if '<title>405' in ret.text:
print(ret.text)
assert False
return ret.text
else:
print(f'状态码为{ret.status_code}')
assert False
# assert False
except Exception as e:
print('出错了',e)
num -= 1
return scrape(url, headers, num)
版权归属:
海绵
许可协议:
本文使用《署名-非商业性使用-相同方式共享 4.0 国际 (CC BY-NC-SA 4.0)》协议授权
评论区