爬取豆瓣排行榜前250名电影名称
import urllib.requestimport bs4import reheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ""Chrome/70.0.3538.110 Safari/537.36"}def get_page(url):request = urllib.request.Request(url, headers=headers)response = urllib.request.urlopen(request)html = response.read().decode("utf-8")return htmldef parse_html(html):soup = bs4.BeautifulSoup(html, "lxml")movies_lists_soup = soup.find("ol", attrs={"class": "grid_view"})movies_list_soup = movies_lists_soup.find_all("li")for movies_soup in movies_list_soup:movie_name = movies_soup.find('span', attrs={"class": 'title'}).textmovie_star = movies_soup.find('p').textprint(movie_name, end=" ")print(movie_star)# movies_pics = movies_soup.findmovie_pic = movies_soup.find('img')['src']print(movie_pic)urllib.request.urlretrieve(movie_pic, "E:\python\wenjian\spider\images\\" + movie_name + ".jpg")def main():i = 0while i < 10:url = "/top250?start=" + str(i * 25) + "&filter="html = get_page(url)data = parse_html(html)i += 1if __name__ == '__main__':main()
