用Python抓取王垠的博客
直接上代码（由于他博客是异步的，所以我们要用mechanize）
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created on 2017年5月24日
@author: BirdZhang
'''
from bs4 import BeautifulSoup
import mechanize
import cookielib
from wyblog import BLOG_URL
class NoHistory(object):
def add(self, *a, **k): pass
def clear(self): pass
def getBrowers():
br = mechanize.Browser(history=NoHistory())
#options
br.set_handle_equiv(True)
#br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
cj = cookielib.LWPCookieJar()
br.set_cookiejar(cj)##关联cookies
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
br.set_debug_http(False)
br.set_debug_redirects(False)
br.set_debug_responses(False)
br.addheaders = [("User-agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36")]
return br
if __name__ == "__main__":
br = getBrowers()
r = br.open(BLOG_URL)
html = r.read()
# print html
soup = BeautifulSoup(html,"html5lib")
lis = soup.find_all(name='li',attrs={
"class":"list-group-item title"
})
for i in lis:
print i.a["href"]," ".join(i.a.contents)
剩下的自己该干嘛干嘛吧