复制代码 代码如下:
#coding:utf-8

import urllib.request
import xml.dom.minidom
import sqlite3
import threading
import time

class logger(object):
def log(self,*msg):
for i in msg:
print(i)

Log = logger()
Log.log('测试下')

class downloader(object):

def __init__(self,url):
self.url = url

def download(self):
Log.log('开始下载',self.url)
try:
content = urllib.request.urlopen(self.url).read()
#req = urllib.request.Request(url)
#response = urllib.request.urlopen(req)
#content = response.read()
Log.log('下载完毕')
return(content)
except:
Log.log('下载出错')
return(None)


class parser(object):

def __init__(self,content):
#获得根节点
self.html = xml.dom.minidom.parseString(content)

def parse(self):
Log.log('开始提取数据')
contents = {'content':'','url':[]}
#获得div节点
divs = self.html.getElementsByTagName('div')
#获得content节点
for div in divs:
if div.hasAttribute('class') and \
div.getAttribute('class') == 'content':
#获得糗事百科的内容
textNode = div.childNodes[0]
qContent = textNode.data
#数据填充
contents['content'] = qContent

#获得上一糗事、下一糗事节点
spans = self.html.getElementsByTagName('span')
for span in spans:
pspan = span.parentNode
if pspan.tagName == 'a':
#pspan为对应的链接,此时需要将对应的地址加入数据库
url = pspan.getAttribute('href')
qid = url[10:][:-4]
#数据填充
contents['url'].append(qid)
Log.log('提取数据完毕')
return(contents)

def downloadPage(qid,db):
url = 'http://mit()
#关闭连接
c.close()
cn.close()
Log.log('更新状态成功')

def getList(self,unDonloaded=1):
Log.log('获得列表')
l = []
#获得连接
cn = sqlite3.connect(self.dbpath)
c = cn.cursor()
#获得数据
c.execute(Q_LIST,(unDonloaded,))
rows = c.fetchall()

for i in rows:
l.append(i[0])
#关闭连接
c.close()
cn.close()

Log.log('获得列表成功')
return(l)

class singleDownloader(object):
def __init__(self):
self.downloadList = []

def setdb(self,db):
self.db = db

def setDownloadList(self,downloadList):
self.downloadList = list(set(self.downloadList+downloadList))

def beginDownload(self):
for i in self.downloadList:
downloadPage(i,self.db)

def main():
db = dbConnect('db.sqlite')
#dp = downloaderPool()
#dp.setdb(db)
sp = singleDownloader()
sp.setdb(db)

dp=sp

unDownloadedList = db.getList()
#当还有未下载的糗事时就要继续下载
while(len(unDownloadedList)):
#使用该列表填充下载池
dp.setDownloadList(unDownloadedList)

dp.beginDownload()

time.sleep(1)
#重置参数
unDownloadedList = db.getList()

if __name__ == '__main__':
main()


代码是没问题的，可以正常运行，但是希望做到以下2方面：
1、多线程下载
2、代码分离度更高，跟面向对象