首先 我們要爬取一下有關的數據
將數據分別存儲在不同的文件中
方便接下來的數據處理
import time
import json
import requests
from datetime import datetime
import pandas as pd
import numpy as np
def catch_data():
? url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5'
? reponse = requests.get(url=url).json()
? #返回數據字典
? data = json.loads(reponse['data'])
? return data
data = catch_data()
data.keys()
lastUpdateTime = data['lastUpdateTime']
# 數據明細，數據結構比較復雜，一步一步打印出來看，先明白數據結構
areaTree = data['areaTree']
# 國內數據
china_data = areaTree[0]['children']
china_list = []
for a in range(len(china_data)):
? province = china_data[a]['name']
? province_list = china_data[a]['children']
? for b in range(len(province_list)):
? ? ? city = province_list[b]['name']
? ? ? total = province_list[b]['total']
? ? ? today = province_list[b]['today']
? ? ? china_dict = {}
? ? ? china_dict['province'] = province
? ? ? china_dict['city'] = city
? ? ? china_dict['total'] = total
? ? ? china_dict['today'] = today
? ? ? china_list.append(china_dict)
china_data = pd.DataFrame(china_list)
china_data.head()
# 定義數據處理函數
def confirm(x):
? confirm = eval(str(x))['confirm']
? return confirm
def dead(x):
? dead = eval(str(x))['dead']
? return dead
def heal(x):
? heal =? eval(str(x))['heal']
? return heal
# 函數映射
china_data['confirm'] = china_data['total'].map(confirm)
china_data['dead'] = china_data['total'].map(dead)
china_data['heal'] = china_data['total'].map(heal)
china_data = china_data[["province","city","confirm","dead","heal"]]
china_data.head()
area_data = china_data.groupby("province")["confirm"].sum().reset_index()
area_data.column=["province","confirm"]
# print(area_data)
area_data.to_csv("confirm.csv", encoding="utf_8_sig")
area_data = china_data.groupby("province")["dead"].sum().reset_index()
area_data.column=["province","dead"]
# print(area_data)
area_data.to_csv("dead.csv", encoding="utf_8_sig")
area_data = china_data.groupby("province")["heal"].sum().reset_index()
area_data.column=["province","heal"]
# print(area_data)
area_data.to_csv("heal.csv", encoding="utf_8_sig")
效果圖
還有一些傳言的數據
import requests
import pandas as pd
class SpiderRumor(object):
?def __init__(self):
? ? ?self.url = "https://vp.fact.qq.com/loadmore?artnum=0&page=%s"
? ? ?self.header = {
? ? ? ? ?"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
}
?def spider_run(self):
? ? ?df_all = list()
? ? ?for url in [self.url % i for i in range(61)]:
? ? ? ? ?data_list = requests.get(url, headers=self.header).json()["content"]
? ? ? ? ?temp_data = [[df["title"], df["date"], df["result"], df["explain"], df["tag"]] for df in data_list]
? ? ? ? ?df_all.extend(temp_data)
? ? ? ? ?print(temp_data[0])
? ? ?pd.DataFrame(df_all, columns=["title", "date", "result", "explain", "tag"]).to_csv("冠狀病毒謠言數據.csv", encoding="utf_8_sig")
if __name__ == '__main__':
?spider = SpiderRumor()
?spider.spider_run()
數據都獲取到了
然后我們來完成數據可視化吧！
先看一下matplotlib庫做的可視化
折線圖：
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# Windows系統設置中文字體
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
data = pd.read_csv('冠狀病毒謠言數據.csv')
labels = data['date'].value_counts().index.tolist()
sizes = data['date'].value_counts().values.tolist()
plt.figure(figsize=(30, 8))
plt.plot(labels, sizes)
plt.xticks(labels, labels, rotation=45)
plt.title('每日謠言數量', fontsize=40)
plt.show()
效果圖：
柱狀圖：
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# Windows系統設置中文字體
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
data = pd.read_csv("冠狀病毒謠言數據.csv")
df = pd.Series([j for i in [eval(i) for i in data["tag"].tolist()] for j in i]).value_counts()[:20]
X = df.index.tolist()
Y = df.values.tolist()
plt.figure(figsize=(15, 8))? # 設置畫布
plt.bar(X, Y, color="blue")
plt.tight_layout()
plt.grid(ls='-.')
plt.show()
效果圖;
餅圖：
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# Windows系統設置中文字體
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
data = pd.read_csv("冠狀病毒謠言數據.csv")
labels = data["explain"].value_counts().index.tolist()? # 可以理解為每個文本
sizes = data["explain"].value_counts().values.tolist()? # 可以理解為篩選出每個文本所對應的出現次數
colors = ['lightgreen', 'gold', 'lightskyblue', 'lightcoral']
plt.figure(figsize=(18, 10))
plt.pie(sizes, labels=labels,
? ? ? colors=None, autopct='%1.1f%%', shadow=True,
? ? ? explode=(0.1, 0.1, 0, 0, 0, 0, 0, 0, 0, 0),
? ? ? textprops={'fontsize': 15, 'color': 'black'})? # shadow=True 表示陰影
plt.axis('equal')? # 設置為正的圓形
plt.legend(loc='upper right', ncol=2)
plt.show()
效果圖：
然后是pyecharts庫的可視化
折線圖：
import pandas as pd
import numpy as np
from pyecharts import Line
data = pd.read_csv("dead.csv")
x = data["province"]
y = data["dead"]
line = Line('國內死亡折線圖')
line.add('確診數', x, y, is_label_show=True)
line.render('國內死亡折線圖.html')
line.render_notebook()
效果圖：
import pandas as pd
import numpy as np
from pyecharts import Line
data = pd.read_csv("heal.csv")
x = data["province"]
y = data["heal"]
line = Line('國內治愈折線圖')
line.add('確診數', x, y, is_label_show=True)
line.render('國內治愈折線圖.html')
line.render_notebook()
from pyecharts import Line
import numpy as np
import pandas as pd
data = pd.read_csv("dead.csv")
x = data["province"]
y = data["dead"]
data1 = pd.read_csv("heal.csv")
z = data1["heal"]
line = Line("治愈死亡折線圖")
line.add("治愈", x, z, mark_point=["max", "min"], mark_line=["average"])
line.add("死亡", x, y, mark_point=["max", "min"], mark_line=["average"])
line.render("治愈死亡折線圖.html")
import pandas as pd
import numpy as np
from pyecharts import Line
data = pd.read_csv("confirm.csv")
x = data["province"]
y = data["confirm"]
line = Line('國內確診折線圖')
line.add('確診數', x, y, is_label_show=True)
line.render('國內確診折線圖.html')
line.render_notebook()
from pyecharts import Bar
import numpy as np
import pandas as pd
data = pd.read_csv("dead.csv")
x = data["province"]
y = data["dead"]
data1 = pd.read_csv("heal.csv")
z = data1["heal"]
bar = Bar("治愈死亡柱狀圖")
bar.add("治愈", x, z, is_stack=True, is_label_show=True)
bar.add("死亡", x, y, is_stack=True, is_label_show=True)
bar.render("治愈死亡柱狀圖.html")
from pyecharts import Pie
import pandas as pd
import numpy as np
data = pd.read_csv("dead.csv")
x = data["province"]
y = data["dead"]
pie = Pie("死亡環圖", title_pos='right')
pie.add(
? "",
? x,
? y,
? radius=[40, 75],
? label_text_color=None,
? is_label_show=True,
? is_more_utils=True,
? legend_orient="vertical",
? legend_pos="left",
)
pie.render(path="死亡環圖.html")
from pyecharts import Pie
import pandas as pd
import numpy as np
data = pd.read_csv("heal.csv")
x = data["province"]
y = data["heal"]
pie = Pie("治愈環圖", title_pos='right')
pie.add(
? "",
? x,
? y,
? radius=[40, 75],
? label_text_color=None,
? is_label_show=True,
? is_more_utils=True,
? legend_orient="vertical",
? legend_pos="left",
)
pie.render(path="治愈環圖.html")
from pyecharts import Pie
import pandas as pd
import numpy as np
data = pd.read_csv("confirm.csv")
x = data["province"]
y = data["confirm"]
pie = Pie("確診環圖", title_pos='right')
pie.add(
? "",
? x,
? y,
? radius=[40, 75],
? label_text_color=None,
? is_label_show=True,
? is_more_utils=True,
? legend_orient="vertical",
? legend_pos="left",
)
pie.render(path="確診環圖.html")
from pyecharts import Pie
import numpy as np
import pandas as pd
data = pd.read_csv("冠狀病毒謠言數據.csv")
df = pd.Series([j for i in [eval(i) for i in data["tag"].tolist()] for j in i]).value_counts()[:20]
X = df.index.tolist()
Y = df.values.tolist()
pie = Pie("謠言關鍵字環圖", title_pos='center')
pie.add(
? "",
? X,
? Y,
? radius=[40, 75],
? label_text_color=None,
? is_label_show=True,
? is_more_utils=True,
? legend_orient="vertical",
? legend_pos="left",
)
pie.render(path="謠言環圖.html")
import pandas as pd
from pyecharts import WordCloud
import matplotlib.pyplot as plt
# Windows系統設置中文字體
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
data = pd.read_csv("confirm.csv")
x = data["province"]
y = data["confirm"]
wordcloud = WordCloud(width=900, height=420)
wordcloud.add("", x, y, word_size_range=[20, 100])
wordcloud.render("疫情詞云圖.html")
wordcloud.render_notebook()
import numpy as np
import pandas as pd
from pyecharts import WordCloud
import matplotlib.pyplot as plt
# Windows系統設置中文字體
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
data = pd.read_csv("冠狀病毒謠言數據.csv")
df = pd.Series([j for i in [eval(i) for i in data["tag"].tolist()] for j in i]).value_counts()[:20]
X = df.index.tolist()
Y = df.values.tolist()
wordcloud = WordCloud(width=1300, height=620)
wordcloud.add("", X, Y, word_size_range=[20, 100])
wordcloud.render("謠言詞云圖.html")
wordcloud.render_notebook()
