Python爬取全书网后,对python几个模块的了解
新人,仅接触Python两天,未深入学习,代码较渣,勿喷.
直接发代码:
#!/usr/bin/python
# -*- coding: utf-8 -*-
#爬取速度较慢,主要是卡在urllib.openurl().read()上
import urllib, re, MySQLdb,time
class Sql(object): #创建SQL类来处理数据操作
# conn=MySQLdb.connect(host="localhost", user="root",passwd= "root", db="quanshuwang",charset="utf8")
conn = MySQLdb.connect(host="mysql.xxx.com", port=7150, user="noveltest", passwd="123456", db="noveltest", charset="utf8")
def addBooks(self,sort,name,imgurl,des,status,author):
cur = self.conn.cursor()
sql="INSERT INTO novel(sort,name,imgurl,description,status,author) values(%s,'%s','%s','%s','%s','%s')"%(sort,name,imgurl,des,status,author)
cur.execute(sql)
lastrowid = cur.lastrowid
self.conn.commit()
return lastrowid
def addChapters(self,novelid,title,content):
pass
cur=self.conn.cursor()
sql="insert into chapter(novelid,title,content) values(%s,'%s','%s')"%(novelid,title,content)
cur.execute(sql)
lastrowid=cur.lastrowid
self.conn.commit()
return lastrowid
def getBookList(): #取出所有书的名称和地址
print 'getbooklist:',time.strftime('%H:%M:%S')
url='http://www.quanshuwang.com/map/1.html'
html=urllib.urlopen(url).read() #获取HTML源码
html=html.decode('gbk').encode('utf-8') #先用decode()将源码的GBK转换为通用的UNICODE编码,再解压成UTF-8
reg=r'<a href="(/book/.*?)" target="_blank">(.*?)</a>' #正则,锁定(.*?) .为任意字符,*可匹配多个,?非贪婪模式,取最短那个
return re.findall(reg,html) #用RE.FINDALL取出正则匹配的文本
def getAuthor(bookurl):
print 'StartgetAuthorAndbookurl:',bookurl,time.strftime('%H:%M:%S')
bookurl=bookurl.split('/')[-2]
url = 'http://www.quanshuwang.com/book_%s.html' %bookurl
print 'startReadHtml:',time.strftime('%H:%M:%S')
html = urllib.urlopen(url).read()
print 'EndReadHtmlandStartDecode:',time.strftime('%H:%M:%S')
html=html.decode('gbk').encode('utf-8')
print 'StartgetAuthormsgbookandEndDecode:',time.strftime('%H:%M:%S')
sort=re.findall(r'<a href="/list/(\d+)_.*?" class="c009900">.*?</a> >', html)[0]
author=re.findall(r'<dt>作 者：</dt><dd> <a href=".*?">(.*?)</a></dd>',html)[0]
status=re.findall(r'dl><dt>小说状态：</dt><dd>(.*?)</dd></dl>',html)[0]
imgurl=re.findall(r'<img onerror="this.src=.*?" src="(.*?)" width="160" height="200" border="0" alt=".*?" title=".*?" />',html)[0]
des=re.findall(r'div class="infoDetail"><div id="waa" style="height:72px;width:690px;overflow:hidden;">(.*?)<br />',html)[0]
print 'EndgetAuthormsgbook:',time.strftime('%H:%M:%S')
bookmsg={'sort':sort,'author':author,'status':status,'imgurl':imgurl,'des':des}
return bookmsg
def getChapterList(bookurl):#得到每篇文章的章节
url='http://www.quanshuwang.com%s'%bookurl
html=urllib.urlopen(url).read().decode('gb2312').encode('utf-8')
reg=r'<li><a href="(.*?)" title=".*?">(.*?)</a></li>'
return re.findall(reg,html)
def getContent(bookurl,chapterurl): #取得每篇文章的内容
url='http://www.quanshuwang.com%s'%bookurl.replace(bookurl.split('/')[-1],chapterurl) #用章节地址替换书目地址里的链接
html=urllib.urlopen(url).read().decode('gbk').encode('utf-8')
reg=r'style5\(\);</script>(.*?)<script type="text/javascript">style6'
return re.findall(reg,html)[0]
mysql=Sql() #打开数据库链接
for bookname in getBookList(): #循环爬取小说内容
print 'for:',time.strftime('%H:%M:%S')
msgbook={}
bookurl=bookname[0] #书的URL
name=bookname[1] #书名
print 'Startmsgbooklen',len(msgbook)
msgbook=getAuthor(bookurl)
print 'EndgetauthorandEndmsgbook:',len(msgbook),time.strftime('%H:%M:%S')
sort=msgbook['sort']
author = msgbook['author']
imgurl = msgbook['imgurl']
des = msgbook['des']
status=msgbook['status']
urllib.urlretrieve(imgurl, "images/%s.jpg" %name.decode('utf-8'))#因为当前环境为utf-8,故要把书名转成unicode
#print type(sort),type(author),type(name),type(imgurl),type(des),type(status)
# novelid=mysql.addBooks(int(sort),name,imgurl,des,status,author)
print '正在存储小说%s'%name
# for chapter in getChapterList(bookurl):
# chapterurl=chapter[0] #章节URL
# chaptername=chapter[1] #章节名称
# content=getContent(bookurl,chapterurl) #小说内容
# print '正在存储章节%s'%chaptername
# mysql.addChapters(novelid,chaptername,content)
mysql.conn.close()