# coding:utf-8'''使用requests模块，使用代理，进行淘宝网商品信息的爬取'''#引入需要的模块importrequestsimportrandomimportrefromlxmlimportetree#定义免费代理列表proxy_list = [{"http":"116.8.83.3:8118"}, {"http":"116.8.83.3:8118"},{"http":"113.89.59.161:8118"}, {"http":"113.67.183.196:8118"},{"http":"180.155.135.224:31425"}, {"http":"123.161.153.238:22593"}]#定义访问路由url ='/sem/tbsearch?refpid=mm_26632360_8858797_29866178&keyword=%E5%A5%B3%E8%A3%85&clk1=51ae9dffa6be64388c59f2c5c20274c7&upsid=51ae9dffa6be64388c59f2c5c20274c7'#设置请求头headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}#发送请求response = requests.get(url, headers=headers, proxies=random.choice(proxy_list), verify=False)content = response.text#图片名称的获取reg_name =r'<span class="title" title=".*">(.*?)</span>'pattern_name = pile(reg_name)titles_list = pattern_name.findall(content)titles = []fortitleintitles_list:titles.append(title)print"商品名称爬取结束"#图片的获取reg_name =r'<img data-ks-lazyload="(.*?)" s'pattern_name = pile(reg_name)imgs_list = pattern_name.findall(content)imgs = []#修改连接foriinrange(0, len(imgs_list) - 5):imgs_list[i] ="https:"+ imgs_list[i]#获取图片的url,进行第二次爬取i = 1forimginimgs_list:filename = str(i) +'_'+ img[-20:]f = open('tb_img/%s'% filename,'wb')url = imgresponse = requests.get(url, verify=False)f.write(response.content)f.close()i += 1print"图片爬取结束"
