前言
各位，七夕快到了，想好要送什么礼物了吗？
昨天有朋友私信我，问我能用Python分析下网上小猫咪的数据，是想要送一只给女朋友，当做礼物。
Python从零基础入门到实战系统教程、源码、视频
网上的数据太多、太杂，而且我也不知道哪个网站的数据比较好。所以，只能找到一个猫咪交易网站的数据来分析了
地址：
/
爬虫部分
请求数据
import requestsurl = f'/index.php?/chanpinliebiao_c_2_1--24.html'headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'}response = requests.get(url=url, headers=headers)print(response.text)
解析数据
# 把获取到的 html 字符串数据转换成 selector 对象 这样调用selector = parsel.Selector(response.text)# css 选择器只要是根据标签属性内容提取数据 编程永远不看过程 只要结果href = selector.css('.content:nth-child(1) a::attr(href)').getall()areas = selector.css('.content:nth-child(1) .area .color_333::text').getall()areas = [i.strip() for i in areas] # 列表推导式
提取标签数据
for index in zip(href, areas):# /index.php?/chanpinxiangqing_224383.htmlindex_url = '' + index[0]response_1 = requests.get(url=index_url, headers=headers)selector_1 = parsel.Selector(response_1.text)area = index[1]# getall 取所有 get 取一个title = selector_1.css('.detail_text .title::text').get().strip()shop = selector_1.css('.dinming::text').get().strip() # 店名price = selector_1.css('.info1 div:nth-child(1) span.red.size_24::text').get() # 价格views = selector_1.css('.info1 div:nth-child(1) span:nth-child(4)::text').get() # 浏览次数# replace() 替换promise = selector_1.css('.info1 div:nth-child(2) span::text').get().replace('卖家承诺: ', '') # 浏览次数num = selector_1.css('.info2 div:nth-child(1) div.red::text').get() # 在售只数age = selector_1.css('.info2 div:nth-child(2) div.red::text').get() # 年龄kind = selector_1.css('.info2 div:nth-child(3) div.red::text').get() # 品种prevention = selector_1.css('.info2 div:nth-child(4) div.red::text').get() # 预防person = selector_1.css('div.detail_text .user_info div:nth-child(1) .c333::text').get() # 联系人phone = selector_1.css('div.detail_text .user_info div:nth-child(2) .c333::text').get() # 联系方式postage = selector_1.css('div.detail_text .user_info div:nth-child(3) .c333::text').get().strip() # 包邮purebred = selector_1.css('.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(1) .c333::text').get().strip() # 是否纯种sex = selector_1.css('.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(4) .c333::text').get().strip() # 猫咪性别video = selector_1.css('.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(4) .c333::text').get().strip() # 能否视频worming = selector_1.css('.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(2) .c333::text').get().strip() # 是否驱虫dit = {'地区': area,'店名': shop,'标题': title,'价格': price,'浏览次数': views,'卖家承诺': promise,'在售只数': num,'年龄': age,'品种': kind,'预防': prevention,'联系人': person,'联系方式': phone,'异地运费': postage,'是否纯种': purebred,'猫咪性别': sex,'驱虫情况': worming,'能否视频': video,'详情页': index_url,}
保存数据
import csv # 内置模块f = open('猫咪1.csv', mode='a', encoding='utf-8', newline='')csv_writer = csv.DictWriter(f, fieldnames=['地区', '店名', '标题', '价格', '浏览次数', '卖家承诺', '在售只数','年龄', '品种', '预防', '联系人', '联系方式', '异地运费', '是否纯种','猫咪性别', '驱虫情况', '能否视频', '详情页'])csv_writer.writeheader() # 写入表头csv_writer.writerow(dit)print(title, area, shop, price, views, promise, num, age,kind, prevention, person, phone, postage, purebred, sex, video, worming, index_url, sep=' | ')
得到数据
数据可视化部分
词云图
from pyecharts import options as optsfrom pyecharts.charts import WordCloudfrom pyecharts.globals import SymbolTypefrom pyecharts.globals import ThemeTypewords = [(i,1) for i in cat_info['品种'].unique()]c = (WordCloud(init_opts=opts.InitOpts(theme=ThemeType.LIGHT)).add("", words,shape=SymbolType.DIAMOND).set_global_opts(title_opts=opts.TitleOpts(title="")))c.render_notebook()
交易品种占比图
from pyecharts import options as optsfrom pyecharts.charts import TreeMappingzhong = cat_info['品种'].value_counts().reset_index()data = [{'value':i[1],'name':i[0]} for i in zip(list(pingzhong['index']),list(pingzhong['品种']))]c = (TreeMap(init_opts=opts.InitOpts(theme=ThemeType.LIGHT)).add("", data).set_global_opts(title_opts=opts.TitleOpts(title="")).set_series_opts(label_opts=opts.LabelOpts(position="inside")))c.render_notebook()
均价占比图
from pyecharts import options as optsfrom pyecharts.charts import PictorialBarfrom pyecharts.globals import SymbolTypelocation = list(price['品种'])values = list(price['价格'])c = (PictorialBar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT)).add_xaxis(location).add_yaxis("",values,label_opts=opts.LabelOpts(is_show=False),symbol_size=18,symbol_repeat="fixed",symbol_offset=[0, 0],is_symbol_clip=True,symbol=SymbolType.ROUND_RECT,).reversal_axis().set_global_opts(title_opts=opts.TitleOpts(title="均价排名"),xaxis_opts=opts.AxisOpts(is_show=False),yaxis_opts=opts.AxisOpts(axistick_opts=opts.AxisTickOpts(is_show=False),axisline_opts=opts.AxisLineOpts(linestyle_opts=opts.LineStyleOpts(opacity=0),),),).set_series_opts(label_opts=opts.LabelOpts(position='insideRight')))c.render_notebook()
猫龄柱状图
from pyecharts import options as optsfrom pyecharts.charts import Barfrom pyecharts.faker import Fakerx = ['1-3个月','3-6个月','6-9个月','9-12个月','1年以上']y = [69343,115288,18239,4139,5]c = (Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT)).add_xaxis(x).add_yaxis('', y).set_global_opts(title_opts=opts.TitleOpts(title="猫龄分布")))c.render_notebook()
