Python爬取當(dāng)網(wǎng)書籍?dāng)?shù)據(jù)并數(shù)據(jù)可視化展示

第Python爬取當(dāng)網(wǎng)書籍?dāng)?shù)據(jù)并數(shù)據(jù)可視化展示目錄一、開發(fā)環(huán)境二、模塊使用三、爬蟲代碼實(shí)現(xiàn)步驟1.導(dǎo)入所需模塊2.發(fā)送請(qǐng)求,用python代碼模擬瀏覽器發(fā)送請(qǐng)求3.解析數(shù)據(jù),提取我們想要數(shù)據(jù)內(nèi)容4.多頁爬取5.保存數(shù)據(jù),保存csv表格里面四、數(shù)據(jù)可視化1.導(dǎo)入所需模塊2.導(dǎo)入數(shù)據(jù)3.可視化

一、開發(fā)環(huán)境

Python3.8Pycharm2025.2專業(yè)版

二、模塊使用

csv模塊把爬取下來的數(shù)據(jù)保存表格里面的內(nèi)置模塊

requestspipinstallrequests數(shù)據(jù)請(qǐng)求模塊

parselpipinstallparsel數(shù)據(jù)解析模塊css選擇器去提取數(shù)據(jù)

三、爬蟲代碼實(shí)現(xiàn)步驟

導(dǎo)入所需模塊發(fā)送請(qǐng)求,用python代碼模擬瀏覽器發(fā)送請(qǐng)求解析數(shù)據(jù),提取我們想要數(shù)據(jù)內(nèi)容多頁爬取保存數(shù)據(jù),保存csv表格里面

1.導(dǎo)入所需模塊

importrequests

#數(shù)據(jù)請(qǐng)求模塊第三方模塊需要pipinstallrequests

importparsel

#數(shù)據(jù)解析模塊第三方模塊需要pipinstallparsel

importcsv

#保存csv表格數(shù)據(jù)模塊內(nèi)置模塊

importtime

#時(shí)間模塊

2.發(fā)送請(qǐng)求,用python代碼模擬瀏覽器發(fā)送請(qǐng)求

headers請(qǐng)求頭作用就是python代碼偽裝成瀏覽器對(duì)于服務(wù)器發(fā)送請(qǐng)求

User-Agent用戶代理瀏覽器的基本身份標(biāo)識(shí)

標(biāo)題中無效的返回字符或前導(dǎo)空格：User-Agent不要留有空格

通過requests模塊里面get請(qǐng)求方法，對(duì)于url地址發(fā)送請(qǐng)求，并且攜帶上面header請(qǐng)求頭參數(shù)，最后用response變量接收返回?cái)?shù)據(jù)

url=f'/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-{page}'

#headers請(qǐng)求頭字典數(shù)據(jù)類型

headers={

'User-Agent':'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/96.0.4664.45Safari/537.36'

response=requests.get(url=url,headers=headers)

3.解析數(shù)據(jù),提取我們想要數(shù)據(jù)內(nèi)容

selector=parsel.Selector(response.text)

#對(duì)于獲取到的html字符串?dāng)?shù)據(jù)進(jìn)行轉(zhuǎn)換selector對(duì)象

#css選擇器就是根據(jù)標(biāo)簽屬性提取相應(yīng)的數(shù)據(jù)

lis=selector.css('ul.bang_listli')

forliinlis:

#.name定位class類名name標(biāo)簽a標(biāo)簽attr()屬性選擇器取a標(biāo)簽里面title屬性get()獲取數(shù)據(jù)

title=li.css('.namea::attr(title)').get()

#書名

#獲取標(biāo)簽里面文本數(shù)據(jù)直接text

comment=li.css('.stara::text').get().replace('條評(píng)論','')

#評(píng)論

recommend=li.css('.star.tuijian::text').get().replace('推薦','')

#推薦

author=li.css('.publisher_infoa:nth-child(1)::attr(title)').get()

#作者

publish=li.css('div:nth-child(6)a::text').get()

#出版社

price_n=li.css('.price.price_n::text').get()

#售價(jià)

price_r=li.css('.price.price_r::text').get()

#原價(jià)

price_s=li.css('.price.price_s::text').get()

#折扣

price_e=li.css('.price.price_e.price_n::text').get()

#電子書價(jià)格

href=li.css('.namea::attr(href)').get()

#詳情頁

dit={

'書名':title,

'評(píng)論數(shù)':comment,

'推薦量':recommend,

'作者':author,

'出版社':publish,

'售價(jià)':price_n,

'原價(jià)':price_r,

'折扣':price_s,

'電子書價(jià)格':price_e,

'詳情頁':href,

}

csv_writer.writerow(dit)

#數(shù)據(jù)保存到csv

print(title,comment,recommend,author,publish,price_n,price_r,price_s,price_e,href,sep='|')

4.多頁爬取

forpageinrange(1,26):

#字符串格式化方法

print(f'正在爬取第{page}頁的數(shù)據(jù)內(nèi)容')

time.sleep(1.5)

url=f'/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-{page}'

5.保存數(shù)據(jù),保存csv表格里面

#創(chuàng)建打開一個(gè)文件進(jìn)行保存

f=open('當(dāng)當(dāng)圖書.csv',mode='a',encoding='utf-8',newline='')

csv_writer=csv.DictWriter(f,fieldnames=[

'書名',

'評(píng)論數(shù)',

'推薦量',

'作者',

'出版社',

'售價(jià)',

'原價(jià)',

'折扣',

'電子書價(jià)格',

'詳情頁',

csv_writer.writeheader()

#寫入表頭

運(yùn)行代碼，效果如下圖：

四、數(shù)據(jù)可視化

1.導(dǎo)入所需模塊

importpandasaspd

frompyecharts.chartsimport*

frompyecharts.globalsimportThemeType#設(shè)定主題

frommons.utilsimportJsCode

importpyecharts.optionsasopts

2.導(dǎo)入數(shù)據(jù)

df=pd.read_csv('書籍信息.csv',encoding='utf-8',engine='python')

df.head()

3.可視化

書籍總體價(jià)格區(qū)間:

pie1=(

Pie(init_opts=opts.InitOpts(theme='dark',width='1000px',height='600px'))

.add('',datas_pair_1,radius=['35%','60%'])

.set_series_opts(label_opts=opts.LabelOpts(formatter=":okqqyea%"))

.set_global_opts(

title_opts=opts.TitleOpts(

title="當(dāng)當(dāng)網(wǎng)書籍\n\n原價(jià)價(jià)格區(qū)間",

pos_left='center',

pos_top='center',

title_textstyle_opts=opts.TextStyleOpts(

color='#F0F8FF',

font_size=20,

font_weight='bold'

),

)

)

.set_colors(['#EF9050','#3B7BA9','#6FB27C','#FFAF34','#D8BFD8','#00BFFF','#7FFFAA'])

pie1.render_notebook()

pie1=(

Pie(init_opts=opts.InitOpts(theme='dark',width='1000px',height='600px'))

.add('',datas_pair_2,radius=['35%','60%'])

.set_series_opts(label_opts=opts.LabelOpts(formatter=":e40u44w%"))

.set_global_opts(

title_opts=opts.TitleOpts(

title="當(dāng)當(dāng)網(wǎng)書籍\n\n售價(jià)價(jià)格區(qū)間",

pos_left='center',

pos_top='center',

title_textstyle_opts=opts.TextStyleOpts(

color='#F0F8FF',

font_size=20,

font_weight='bold'

),

)

)

.set_colors(['#EF9050','#3B7BA9','#6FB27C','#FFAF34','#D8BFD8','#00BFFF','#7FFFAA'])

pie1.render_notebook()

各個(gè)出版社書籍?dāng)?shù)量柱狀圖:

bar=(

Bar(init_opts=opts.InitOpts(height='500px',width='1000px',theme='dark'))

.add_xaxis(counts.index.tolist())

.add_yaxis(

'出版社書籍?dāng)?shù)量',

counts.values.tolist(),

label_opts=opts.LabelOpts(is_show=True,position='top'),

itemstyle_opts=opts.ItemStyleOpts(

color=JsCode("""newecharts.graphic.LinearGradient(

0,0,0,1,[{offset:0,color:'rgb(255,99,71)'},{offset:1,color:'rgb(32,178,170)'}])

"""

)

)

)

.set_global_opts(

title_opts=opts.TitleOpts(

title='各個(gè)出版社書籍?dāng)?shù)量柱狀圖'),

xaxis_opts=opts.AxisOpts(name='書籍名稱',

type_='category',

axislabel_opts=opts.LabelOpts(rotate=90),

),

yaxis_opts=opts.AxisOpts(

name='數(shù)量',

min_=0,

max_=29.0,

splitline_opts=opts.SplitLineOpts(is_show=True,linestyle_opts=opts.LineStyleOpts(type_='dash'))

),

tooltip_opts=opts.TooltipOpts(trigger='axis',axis_pointer_type='cross')

)

.set_series_opts(

markline_opts=opts.MarkLineOpts(

data=[

opts.MarkLineItem(type_='average',name='均值'),

opts.MarkLineItem(type_='max',name='最大值'),

opts.MarkLineItem(type_='min',name='最小值'),

]

)

)

bar.render_notebook()

書籍評(píng)論數(shù)最高Top20:

bar=(

Bar(init_opts=opts.InitOpts(height='500px',width='1000px',theme='dark'))

.add_xaxis(price_top.index.tolist())

.add_yaxis(

'書籍單價(jià)',

price_top.values.tolist(),

label_opts=opts.LabelOpts(is_show=True,position='top'),

itemstyle_opts=opts.ItemStyleOpts(

color=JsCode("""newecharts.graphic.LinearGradient(

0,0,0,1,[{offset:0,color:'rgb(255,99,71)'},{offset:1,color:'rgb(32,178,170)'}])

"""

)

)

)

.set_global_opts(

title_opts=opts.TitleOpts(

title='單價(jià)最高的書籍詳細(xì)柱狀圖'),

xaxis_opts=opts.AxisOpts(name='書籍名稱',

type_='cate