直接上代码了,没什么解释的 每步都有注释的 复制粘贴就可以用了.不能运行直接回复"代码" 给你源码
import requestsfrom lxml import etreeimport jsonclass XiciProxiesSpider(object):def __init__(self):self.num = 1self.start_url = '/free/inha/{}'.format(self.num)self.headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36'}def get_page_from_url(self, url):response = requests.get(url, headers=self.headers)return response.content.decode()def get_data_from_page(self, page):# 把page转换为Element对象html = etree.HTML(page)# 获取包含代理信息的tr列表trs = html.xpath('//tbody//tr')# 遍历trs, 获取数据信息data = {'http': [],# 'https': []}for tr in trs:try:ip = tr.xpath('./td[1]/text()')[0] # IP地址port = tr.xpath('./td[2]/text()')[0] # 端口ip_type = tr.xpath('./td[4]/text()')[0].lower() # 类型 以及大小写转换# 如果ip不是http或https直接返回if ip_type not in data.keys():return# 构建代理数据item = {ip_type: '{}:{}'.format(ip, port)}# 检查代理IP是否可用, 如果可用添加到列表中if self.validate_ip(item, ip_type):data[ip_type].append(item)except Exception as ex:print(ex)print(etree.tostring(tr))print("222",data)return datadef validate_ip(self, item, ip_type):try:test_url = "{}:///weixin_43407092/article/details/89743502".format(ip_type)response = requests.get(test_url, proxies=item, timeout=2)if response.status_code == 200:return Truereturn Falseexcept Exception as ex:return Falsedef save_data(self, data):with open('快代理.txt', 'a') as f:json.dump(data, f, indent=2)self.num += 1def run(self):while True:# 获取页面内宽容page = self.get_page_from_url(self.start_url)# 获取可用代理IPdata = self.get_data_from_page(page)# 保存数据self.save_data(data)if __name__ == '__main__':fps = XiciProxiesSpider()fps.run()
执行结果如下,有用的代理不多.
