#!/usr/bin/python
# -*- coding: utf-8 -*-
#coding=utf-8
from bs4 import BeautifulSoup
import requests
import sys
import time
import urllib.request
from urllib import parse
import re
import copy
import datetime
import MySQLdb
import socket
#创建表
use='' #数据库用户名
password='' #数据库密码
host='' #数据库ip
database='' #数据库名
#功能：去除不可见字符
#参数：
# str (in) 要解析的字符串
#返回值：返回处理过后的字符串
def confir(str):
for i in range(0,32):
str = str.replace(chr(i),'')
return str
#功能：打开数据连接
#参数：use 数据库用户名
# password 数据库密码
# host 数据库IP地址
# database 数据库名
#返回值：成功返回数据库句柄
def openMysql(use, password, host, database):
conn = MySQLdb.connect(host, use, password, database, charset='utf8')
return conn
#功能：关闭数据库连接
#参数：数据库句柄
#返回值：无返回值
def closeMysql(conn):
conn.close()
#功能：获取网页数据内容
#参数：
# url (in) 网页url
#返回值：
def getHtml(url):
try:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8'
}
req = urllib.request.Request(url, None, headers, None, False)
response = urllib.request.urlopen(req)
html = response.read().decode('utf-8','ignore')
html = BeautifulSoup(html,'html.parser')
return html
except AttributeError as e:
return None
#功能：解析网页，获取下一页
#参数：
# url 要解析的网页
#返回值：成功返回下一页url
def tableutl(url):
#从网络获取网页内容
html = getHtml(url)
endpage = " "
#解析页，获取最后一条
pages = html.find_all('p', id="page")
for pg in pages:
for pgurl in pg.find_all('a', class_="n"):
if pgurl.getText() == "下一页>":
endpage = pgurl.get('href')
break
return endpage
#功能：解析百度新闻搜索列表页
#参数：
# url (in) 百度搜索列表页
#返回值：
def parsetable(url):
html = getHtml(url)
table = html.find_all('div', id="container", class_ ="container_s")
dict = {
'ListId':'',
'Taskid':'',
'当前页码':'',
'标准编号':'',
'标准名称':'',
'发布部门':'',
'实施日期':'',
'状态':'',
'详情链接':'',
'采集状态':'',
'采集机IP':'',
'采集时间':'',
'当前页面访问次数':''
}
conn = openMysql(use, password, host, database)
cursor = conn.cursor()
#创建表
createtabsql = """CREATE TABLE IF NOT EXISTS baidulistpage(
listid VARCHAR(20),
taskid VARCHAR(20),
current_page_num VARCHAR(100),
standard_encd VARCHAR(100),
standard_name VARCHAR(100),
release_department VARCHAR(100),
Implementation_date VARCHAR(100),
start VARCHAR(20),
detail_link VARCHAR(56),
acqui_status VARCHAR(28),
collector_ip VARCHAR(56),
acqui_time VARCHAR(56),
current_page_visits VARCHAR(56));"""
cursor.execute(createtabsql)
mit()
#创建socket实例
skt = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
#连接8.8.8.8，8.8.8.8是google提供的公共dns服务器
skt.connect(('8.8.8.8',80))
#获得连接信息，是一个tuple (ip，port)
socketIpPort = skt.getsockname()
ip = socketIpPort[0]
skt.close()
for divtag in html.find_all('div', id="wrapper_wrapper"):
dict = {
'ListId':'',
'Taskid':'',
'当前页码':'',
'标准编号':'',
'标准名称':'',
'发布部门':'',
'实施日期':'',
'状态':'',
'详情链接':'',
'采集状态':'',
'采集机IP':'',
'采集时间':'',
'当前页面访问次数':''
}
dt = datetime.datetime.now()
dict['详情链接'] = url
dict['采集机IP'] = ip
dict['采集时间'] = dt.strftime("%Y-%m-%d %H:%M:%S")
for div in divtag.find_all('div', class_="result"):
for h3 in div.find_all('h3'):
for a in h3.find_all('a'):
dict['详情链接'] = a.get('href')
dict['标准名称'] = confir(a.getText().replace(" ", ""))
for p in div.find_all('p'):
tr = confir(p.getText().replace(" ", "")).replace(u'\xa0\xa0', u':')
str1 = re.split(r":", tr)
key = copy.copy(str1)
if len(key) < 2:
dict['实施日期'] = key[0]
else:
dict['发布部门'] = key[0]
dict['实施日期'] = key[1]
#print(confir(p.getText().replace(" ", "")).replace(u'\xa0', u':')) #'\xa0' = &nbps 空格
instelldatasql = "INSERT INTO baidulistpage(listid,taskid,\
current_page_num,standard_encd,standard_name, \
release_department,Implementation_date,start, \
detail_link,acqui_status,collector_ip,acqui_time,\
current_page_visits) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);"
cursor.execute(instelldatasql, [dict['ListId'],dict['Taskid'],
dict['当前页码'],dict['标准编号'],dict['标准名称'],
dict['发布部门'],dict['实施日期'],dict['状态'],
dict['详情链接'],dict['采集状态'],dict['采集机IP'],
dict['采集时间'],'1'])
mit()
cursor.close()
closeMysql(conn)
#功能：将搜索关键词插入数据库
#参数：
# keyword (in) 关键词
#返回值：无返回值
def mainfunction(keyword):
#插入入口数据库
initial = ''
url = initial + "/ns?word=" + parse.quote(keyword) + "&tn=news&from=news&cl=2&rn=20&ct=1"
dict = {
'TaskId':'',
'网站名称':'',
'类别名称':'',
'检索词':'',
'附加参数':'',
'入口链接':'',
'采集状态':'',
'采集时间':'',
'采集机IP':'',
'采集机名称':'',
'采集起始时间':'',
'采集结束时间':'',
'采集间隔':''
}
#创建socket实例
skt = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
#连接8.8.8.8，8.8.8.8是google提供的公共dns服务器
skt.connect(('8.8.8.8',80))
#获得连接信息，是一个tuple (ip，port)
socketIpPort = skt.getsockname()
ip = socketIpPort[0]
skt.close()
dt = datetime.datetime.now()
html = getHtml(initial)
title = html.find_all('title')
#print(title[0].getText())
dict['网站名称'] = title[0].getText()
dict['类别名称'] = '新闻'
dict['检索词'] = keyword
dict['入口链接'] = url
dict['采集机IP'] = ip
dict['采集时间'] = dt.strftime("%Y-%m-%d %H:%M:%S")
dict['采集机名称'] = socket.getfqdn(socket.gethostname())
conn = openMysql(use, password, host, database)
cursor = conn.cursor()
createtabsql = """CREATE TABLE IF NOT EXISTS baiduentrance(
task_id VARCHAR(50),
website_name VARCHAR(100),
class_name VARCHAR(100),
search_term VARCHAR(100),
additi_para VARCHAR(100),
entrance_link VARCHAR(100),
acqui_status VARCHAR(100),
acqui_time VARCHAR(56),
collector_ip VARCHAR(28),
collector_name VARCHAR(56),
acqui_start_time VARCHAR(56),
acqui_end_time VARCHAR(56),
collecor_interval VARCHAR(56)
);"""
cursor.execute(createtabsql)
mit()
dict['采集状态'] = '已采集'
instelldatasql = "INSERT INTO baiduentrance(task_id,website_name,\
class_name,search_term,additi_para, \
entrance_link,acqui_status,acqui_time, \
collector_ip,collector_name,acqui_start_time,\
acqui_end_time,collecor_interval) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);"
cursor.execute(instelldatasql, [dict['TaskId'],dict['网站名称'],
dict['类别名称'],dict['检索词'],dict['附加参数'],
dict['入口链接'],dict['采集状态'],dict['采集时间'],
dict['采集机IP'],dict['采集机名称'],dict['采集起始时间'],
dict['采集结束时间'],dict['采集间隔']])
mit()
cursor.close()
closeMysql(conn)
if __name__ == "__main__":
keyword = sys.argv[1]
url = "/ns?word=" + parse.quote(keyword) + "&tn=news&from=news&cl=2&rn=20&ct=1"
newurl = ''
parsetable(url)
url_1 = "" + tableutl(url)
mainfunction(keyword)
for i in range(1,101):
url_2 = url_1
parsetable(url_2)
url_1 = "" + tableutl(url_2)
运行命令：
python3.4 baidunews.py 吴恩达
baiduNews
