第一段 所有股票 周期：天 复权：不复权
# encoding:utf-8import osimport sysimport pandas as qh_pdimport datetimeimport timeimport randomif __name__ == "__main__":print("本文件引用")qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径sys.path.append(qh_path) # 添加自己指定的搜索路径else:print("外文件引用")qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径sys.path.append(qh_path) # 添加自己指定的搜索路径#引入自己的文件模块import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj # 东方财富import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj # 搜狐交易日import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db # 数据库import QH_Stock_Treasure_Box.qh_Log as qh_log# 日志import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan # 传入参数import QH_DF_Interface as qh_df_interfaceimport QH_IP_KDL as qh_ip_m #ipif __name__ == "__main__":qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_D_Log")qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date() # 同一个文件的函数 日期会自动传入qh_Log.qh_Log("获取交易日期-搜狐，交易日期：{}".format(qh_Trading_day))qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")qh_Log.qh_Log("获取所有股票代码，代码列表：{}".format(qh_all_stock_list_sql))qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")qh_Log.qh_Log("获取所有HTTP代理IP，HTTP代理IP列表：{}".format(qh_ip_list))qh_wangzhan0 = "DF" # 网站代码qh_Log.qh_Log("网站代码：{}".format(qh_wangzhan0))qh_klt_dic = {"天": "D", "周": "W", "月": "M"}qh_Log.qh_Log("周期字典：{}".format(qh_klt_dic))qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}qh_Log.qh_Log("是否复权字典：{}".format(qh_fqt_dic))qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)qh_Log.qh_Log("获取所有数据库参数：{}".format(qh_db_canshu))qh_table_00 = qh_db_canshu['QH_Table'] # 需补充参数的组合表名qh_Log.qh_Log("需补充参数的组合表名：{}".format(qh_table_00))qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)qh_Log.qh_Log("获取需要补充参数的股票流记录SQL：{}".format(qh_sql0))#循环所有股票列表for qh_i, qh_row in enumerate(qh_all_stock_list_sql[1][1]):qh_code = qh_row[15] # 股票代码qh_zjs = qh_row[16] # 证交所qh_name = qh_row[17] # 股票名称# 组合股票代码if str(qh_zjs) == "0":qh_code = "SZ{}".format(qh_code)else:qh_code = "SH{}".format(qh_code)qh_Log.qh_Log("正在获取第{}支股票，股票代码：{}，证交所：{}，股票名称：{}".format(qh_i, qh_code, qh_zjs, qh_name))qh_klt = "天"qh_fqt = "不复权"qh_Log.qh_Log("周期：{}，复权：{}".format(qh_klt, qh_fqt))qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0, qh_klt_dic, qh_fqt_dic,qh_table_00, qh_sql0, qh_Log, qh_ip_list)
第二段 所有股票 所有周期：天/周/月 所有复权：不复权/前复权/后复权
# encoding:utf-8import osimport sysimport pandas as qh_pdimport datetimeimport timeimport randomif __name__ == "__main__":print("本文件引用")qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径sys.path.append(qh_path) # 添加自己指定的搜索路径else:print("外文件引用")qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径sys.path.append(qh_path) # 添加自己指定的搜索路径#引入自己的文件模块import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj # 东方财富import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj # 搜狐交易日import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db # 数据库import QH_Stock_Treasure_Box.qh_Log as qh_log# 日志import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan # 传入参数import QH_DF_Interface as qh_df_interfaceimport QH_IP_KDL as qh_ip_m #ipif __name__ == "__main__":qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_DWM_Log")qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date() # 同一个文件的函数 日期会自动传入qh_Log.qh_Log("获取交易日期-搜狐，交易日期：{}".format(qh_Trading_day))qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")qh_Log.qh_Log("获取所有股票代码，代码列表：{}".format(qh_all_stock_list_sql))qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")qh_Log.qh_Log("获取所有HTTP代理IP，HTTP代理IP列表：{}".format(qh_ip_list))qh_wangzhan0 = "DF" # 网站代码qh_Log.qh_Log("网站代码：{}".format(qh_wangzhan0))qh_klt_dic = {"天": "D", "周": "W", "月": "M"}qh_Log.qh_Log("周期字典：{}".format(qh_klt_dic))qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}qh_Log.qh_Log("是否复权字典：{}".format(qh_fqt_dic))qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)qh_Log.qh_Log("获取所有数据库参数：{}".format(qh_db_canshu))qh_table_00 = qh_db_canshu['QH_Table'] # 需补充参数的组合表名qh_Log.qh_Log("需补充参数的组合表名：{}".format(qh_table_00))qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)qh_Log.qh_Log("获取需要补充参数的股票流记录SQL：{}".format(qh_sql0))# 循环所有股票列表for qh_i,qh_row in enumerate(qh_all_stock_list_sql[1][1]):qh_code = qh_row[15] # 股票代码qh_zjs = qh_row[16] # 证交所qh_name = qh_row[17] # 股票名称# 组合股票代码if str(qh_zjs) == "0":qh_code = "SZ{}".format(qh_code)else:qh_code = "SH{}".format(qh_code)qh_Log.qh_Log("正在获取第{}支股票，股票代码：{}，证交所：{}，股票名称：{}".format(qh_i,qh_code,qh_zjs,qh_name))for qh_key in qh_klt_dic:qh_klt = qh_key#周期for qh_key01 in qh_fqt_dic:qh_fqt = qh_key01 #复权qh_Log.qh_Log("周期：{}，复权：{}".format(qh_klt, qh_fqt))qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0, qh_klt_dic, qh_fqt_dic,qh_table_00, qh_sql0, qh_Log,qh_ip_list)
第三段：调用函数
def Qh_Get_His_Flow(qh_code,qh_name,qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0,qh_klt_dic,qh_fqt_dic,qh_table_00,qh_sql0,qh_Log,qh_ip_list=[]):"""获取所有历史交易数据函数 作者：阙辉:param qh_code: 股票代码 带证交所 SZ000001:param qh_name: 股票名称:param qh_Trading_day: 最近交易日期:param qh_klt:周期:param qh_fqt:复权:param qh_wangzhan0: 网站:param qh_klt_dic:周期字典:param qh_fqt_dic:复权字典:param qh_table_00:需传参数的表名:param qh_sql0: 需传参数的SQL:param qh_ip_list:代理IP:return:"""qh_flow_jilu = []#初始化记录列表qh_wangzhan = qh_wangzhan0 + "_" + str(qh_klt_dic[qh_klt]) + "_" + str(qh_fqt_dic[qh_fqt]) #拼接网站id 关键代码，隔离了周期和复权qh_flow_jilu.append(qh_wangzhan)# qh_flow_jilu 0 网站qh_flow_jilu.append(qh_code) # qh_flow_jilu 1 股票代码qh_flow_jilu.append(qh_name) # qh_flow_jilu 2 股票名称qh_flow_jilu.append(qh_Trading_day) # qh_flow_jilu 3 交易日期# 爬取历史股票qh_type_gg = "QH_JY_HIS" # 获取历史数据类型 历史交易数据qh_flow_jilu.append(qh_type_gg) # #qh_flow_jilu 4 交易类型 历史交易qh_end_date = qh_Trading_day# 最近一个交易日期# 判断最近交易日期的历史数据是否爬取try:qh_sql = qh_sql0.format(qh_wangzhan, qh_code, qh_type_gg)qh_list_qq = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0=qh_sql, qh_ret_sql=False,qh_filed=False)qh_list_qq = qh_list_qq[1][1]if len(qh_list_qq) > 0:qh_list_qq = qh_list_qq[0]# 取第一行qh_pq_yy = qh_list_qq[8] # 爬取状态qh_zong_count = qh_list_qq[10] # 上一次总数量qh_stare_date = qh_list_qq[6] # 上一次爬取交易日期qh_Log.qh_Log("股票流有记录，记录为：爬取状态：{}，上一次总数量：{}，上一次爬取交易日期：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))else:qh_pq_yy = "N" # 爬取状态qh_zong_count = 0 # 总数量qh_stare_date = "1989-01-01" # 默认起始日期qh_Log.qh_Log("股票流无记录，默认记录为：爬取状态：{}，上一次总数量：{}，上一次爬取交易日期：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))except:qh_pq_yy = "N" # 爬取状态qh_zong_count = 0 # 总数量qh_stare_date = "1989-01-01" # 默认起始日期qh_Log.qh_Log("股票流表未创建，默认记录为：爬取状态：{}，上一次总数量：{}，上一次爬取交易日期：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))# 判断是否要请求数据if qh_pq_yy == "Y" and str(qh_end_date) == str(qh_stare_date):qh_Log.qh_Log("股票代码（{}）已爬取，最后交易日期{}，阙辉！".format(qh_code, qh_end_date))else:# 随机从IP列表中获取一个代理IPif len(qh_ip_list) > 0:qh_ip_kk = random.choice(qh_ip_list)qh_htt = qh_ip_kk[1]qh_htt = "https"qh_ip = qh_ip_kk[0]qh_htt_ip = "{}\\:{}".format(qh_htt, qh_ip)qh_proxy = {qh_htt: qh_htt_ip}qh_Log.qh_Log("随机获取代理IP（{}），阙辉！".format(qh_proxy))else:qh_proxy = {}qh_Log.qh_Log("不使用代理IP，阙辉！")# 请求数据qh_Log.qh_Log("开始请求（{}），阙辉！".format(qh_code))qh_his_jiaoyi_list = Qh_DF_His_Flow_Interface01(qh_Trading_day, qh_code,qh_stare_date=qh_stare_date,qh_end_date=qh_end_date, qh_klt=qh_klt,qh_fqt=qh_fqt,qh_spider_id="QH_Spider_00008",qh_out="qh_list", qh_day_t=False,qh_proxy=qh_proxy)qh_his_jiaoyi_list_len = len(qh_his_jiaoyi_list) # 统计本次的数据量qh_new_table = qh_table_00.format(qh_code) # 生成表格名qh_Log.qh_Log("组合存储表格名（{}），阙辉！".format(qh_new_table))# 存储数据 返回存储状态if qh_his_jiaoyi_list_len > 0:qh_Log.qh_Log("请求（{}）数据成功，本次共{}条，阙辉！".format(qh_code,qh_his_jiaoyi_list_len))qh_cc_star = qh_chcan.Qh_InF_To_DB(qh_path, qh_his_jiaoyi_list, qh_spider_id="QH_Spider_00008_D",qh_canshu=False, qh_new_table=qh_new_table)else:qh_cc_star = False#判断数据是否更新成功，更新成功则if qh_cc_star:qh_Log.qh_Log("存储（{}）数据成功，本次共{}条，阙辉！".format(qh_code, qh_his_jiaoyi_list_len))qh_flow_jilu.append("Y") # #qh_flow_jilu 5 是否成功 爬取状态qh_flow_jilu.append(qh_his_jiaoyi_list_len) # #qh_flow_jilu 6 本次数量qh_flow_jilu.append(qh_his_jiaoyi_list_len + int(qh_zong_count)) # #qh_flow_jilu 7 总数量qh_flow_jilu = qh_chcan.Qh_TC_List(qh_flow_jilu, 18) # 补充不满18的字段为空 #qh_flow_jilu 8以后为空的字段if qh_chcan.Qh_InF_To_DB(qh_path, [qh_flow_jilu], qh_spider_id="QH_Spider_99999"): # 更新数据存储状态qh_Log.qh_Log("更新股票记录流成功，股票记录流为：{}，阙辉！".format(qh_flow_jilu))
