實戰計劃0430-石頭的練習作業

練習要求

數據來源

數據處理

運行效果：

抓取數據

數據處理

實現代碼

__author__ = 'daijielei'

'''
19課時練習，為11課時練習的加強版,之前從小豬短租處抓取一堆租房詳情，19課時需要將其內容保存入DB中
'''
from bs4 import BeautifulSoup   #解析html文檔用
import requests                 #抓取html頁面用
import time


import pymongo

client = pymongo.MongoClient('localhost',27017)
xiaozuDB = client['xiaozuDB']
sheet1 = xiaozuDB['sheet1']

'''
saveToDB
將需要保存的內容寫入mongoDB
'''
def saveToDB(data):
    sheet1.insert_one(data)
    client.close()

'''
showxiaozuDB_payMore500
查看mongoDB中保存的數據，篩選出金額500以上的顯示出來
'''
def showxiaozuDB_payMore500():
    print(sheet1.count())
    for item in sheet1.find({'pay':{'$gte':500}}):#篩選條件，將金額500以上的篩選出來
        print(item)
    client.close()

'''
cleanxiaozuDB
清除mangoDB中的數據
'''
def cleanxiaozuDB():
    sheet1.delete_many({})
    print(sheet1.count())
    client.close()






'''
##################################################################################################################
########                                                                                                  ########
                                    以下內容為第11課時完成，抓取小豬短租的相關信息
########                                                                                                  ########
##################################################################################################################
'''

#urls用來制定抓取范圍，header該處無用處
header = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
    'Host':'bj.xiaozhu.com'
}


#getlist用來抓取對應網頁上的每個列表的跳轉url
#測試用url = "http://bj.xiaozhu.com/search-duanzufang-p1-0/"
def getlist(url):
    webData = requests.get(url,headers=header)
    soup = BeautifulSoup(webData.text,'lxml')

    houselists = soup.select('ul > li > a')

    for houselist in houselists:
        listUrl = str(houselist.get('href'))
        if(listUrl.find('http://bj.xiaozhu.com/fangzi/')!=-1):#確認鏈接內容為詳情頁，則進行詳情頁信息抓取
            #print(listUrl)
            getInfoPage(listUrl)


#getInfoPage用來抓取詳情頁里的相關信息，如標題、地址、金額、房屋圖片、擁有人圖片等等
#測試用url = "http://bj.xiaozhu.com/fangzi/525041101.html"
def getInfoPage(url):
    webData = requests.get(url,headers=header)
    soup = BeautifulSoup(webData.text,'lxml')
    time.sleep(2)

    titles = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
    address = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
    pays = soup.select('#pricePart > div.day_l > span')
    houseimages = soup.select('#curBigImage')
    ownerimages = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
    ownernames = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
    ownerSexs = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')

    for title,addres,pay,houseimage,ownerimage,ownername,ownerSex in zip(titles,address,pays,houseimages,ownerimages,ownernames,ownerSexs):
        data = {
            'title':title.get_text(),
            'addres':addres.get_text(),
            'pay': int(pay.get_text()) if str((pay.get_text())).isdigit() else 0,
            'houseimage':houseimage.get('src'),
            'ownerimage':ownerimage.get('src'),
            'ownername':ownername.get_text(),
            'ownerSex':'female'
        }
        if('member_boy_ico' in str(ownerSex['class'])):#該處用來判斷是男性還是女性，男性是通過標簽來區分
            data['ownerSex'] = "male"
        print(data)
        #saveToDB(data)

def startCatch():
    urls = ["http://bj.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,5,1)]
    for url in urls:
        getlist(url)






#入口，該語句只有直接執行時會被調用
if __name__ == '__main__':
    #startCatch()
    showxiaozuDB_payMore500()
    #cleanxiaozuDB()

筆記、總結、思考：

1、這節練習主要的重點有，存儲到數據庫

1、創建本地mongoDB的數據庫連接
 client = pymongo.MongoClient('localhost',27017)

2、創建名為'xiaozuDB'的數據庫
 xiaozuDB = client['xiaozuDB']

3、創建名為'sheet1'的集合（collection，和表類似，但是和表對比靈活多了）
sheet1 = xiaozuDB['sheet1']

4、往'sheet1'的表中寫入數據，這里是直接把字典寫進去
sheet1.insert_one(data)

2、篩選出金額為500以上的數據

1、從'sheet1'中篩選出數據，find函數用來尋找所有滿足條件的數據，傳入字典來包含尋找的條件

for item in sheet1.find({'pay':{'$gte':500}}):#篩選條件，將金額500以上的篩選出來
        print(item)