一、整体结构

二、代理IP类的构建（domain.py文件）

'''
实现_ init_ 方法, 负责初始化,包含如下字段:
ip: 代理的IP地址

port:代理IP的端口号

protocol: 代理IP支持的协议类型,http是0, https是1, https和http都支持是2

nick_ type: 代理IP的匿名程度,高匿:0,匿名: 1,透明:2

speed:代理IP的响应速度，单位s

area:代理IP所在地区

score:代理IP的评分，用于衡量代理的可用性;默认分值可以通过配置文件进行配置.在进行代理可用性检查的时候，
每遇到一次请求失败就减1份,减到0的时候从池中删除.如果检查代理可用,就恢复默认分值

disable_domains:不可用域名列表,有些代理IP在某些域名下不可用，但是在其他域名下可用在配置文件:
settings.py中定义MAX_ _SCORE = 50,表示代理IP的默认最高分数提供_ _str__方法， 返回数据字符串

'''
from settings import MAX_SCORE
#从settings模块中导入MAX_SCORE变量，这个变量的意思就是给每一个IP一个分数（分数高代表这个IP可用性强，初始化为MAX_SCORE）
class Proxy(object):

def \_\_init\_\_(self,ip,port,protocol=-1,nick\_type=-1,speed=-1,area=None,score=MAX\_SCORE,disable\_ip=\[\]):  
    self.ip=ip      #代理ip  
    self.port=port  #代理ip端口  
    self.protocol=protocol  #代表代理ip的协议类型  
    self.nick\_type=nick\_type  #匿名程度  
    self.speed=speed  #代理ip速度  
    self.area=area    #代理ip地址  
    self.score=score  #ip分数  
    self.disable\_ip=disable\_ip

def \_\_str\_\_(self):  
    return str(self.\_\_dict\_\_) #\_\_dict\_\_ : 类的属性（包含一个字典，由类的数据属性组成）

三、配置文件settings.py

可能有人会疑惑为什么要用配置文件保存变量，这样的话还要在其他模块引入，这不是麻烦吗？

因为后面我们可能还要对它们的某些初始值进行改变，这个时候如果如果不用配置文件方式引入，而是直接写上数字。这个时候改代码太麻烦了

import logging
MAX_SCORE=50 #代理ip初始分数

#下面这一部分是log.py的内容，是用来控制日志文件的
LOG_LEVEL = logging.INFO #控制日志文件报错级别
'''
报错级别一共5种（从上到下级别依次递增）
logger.debug("")
logger.info("")
logger.warning("")
logger.error("")
logger.critical("")

如果设置报错级别为INFO，那么就不会报debug的错误
'''
LOG_FMT = '%(asctime)s %(filename)s [line:%(lineno)d] %(levelname)s: %(message)s' #日志内容格式
LOG_DATEFMT = '%Y-%m-%d %H:%M:%S' #日志内容的时间部分格式（也就是”年-月-日 时:分:秒“）
LOG_FILENAME = 'log.log' #要把日志文件放在哪，就放在本目录下log.log文件

TEST_TIMEOUT = 5 #这个是requess访问链接设置的超时时间

#因为我把数据放入了mongodb数据库，所以这里配置的是mongodb数据库连接信息
MONGO_URL = 'mongodb:#127.0.0.1:27017'

'''
这几个是分别爬取几个网站上代理ip的具体爬虫路径
'''
SPIDERS=[

'IPProxyPool.core.proxy\_spider.proxy\_spiders.ip66Spider',  
'IPProxyPool.core.proxy\_spider.proxy\_spiders.KuaidailiSpider',  
'IPProxyPool.core.proxy\_spider.proxy\_spiders.ProxylistplusSpider',  
'IPProxyPool.core.proxy\_spider.proxy\_spiders.XiciSpider'  

]
#这个是多长时间爬取一次代理ip来使用
RUN_SPIDERS_INTERVEL = 12
#这个是最多开多少协程来判断ip是否可用
TEST_PROXIES_ASYNC_COUNT = 10
#这个是多少时间更新一次数据库内ip信息，因为代理ip存活时间有限
TEST_PROXIES_INTERVAL = 2

四、日志信息模块log.py

import logging
import sys

class Logger(object):

def \_\_init\_\_(self):  
    self.\_logger = logging.getLogger()  //得到一个日志处理对象  
    #传参数，告诉它写入日志内容格式  
    self.formatter = logging.Formatter(fmt=settings.LOG\_FMT,datefmt=settings.LOG\_DATEFMT)  
    #把要写入的日志加入句柄里面，到时候会输出到文件内保存  
    self.\_logger.addHandler(self.\_get\_file\_handler(settings.LOG\_FILENAME))  
    #把要写入的日志加入句柄里面，到时候会输出到控制台查看  
    self.\_logger.addHandler(self.\_get\_console\_handler())  
    #设置写入日志级别  
    self.\_logger.setLevel(settings.LOG\_LEVEL)

def \_get\_file\_handler(self,filename):  #设置一下要往那个文件里面写，和编码问题  
    filehandler = logging.FileHandler(filename=filename,encoding="utf-8")  
    filehandler.setFormatter(self.formatter)  #日志格式  
    return filehandler

def \_get\_console\_handler(self):  
    console\_handler = logging.StreamHandler(sys.stdout)  
    console\_handler.setFormatter(self.formatter)  
    return console\_handler

@property  
def logger(self):  
    return self.\_logger

logger = Logger()._logger

if __name__ == '__main__': #下面的是用来测试这个模块
logger.debug("1")
logger.info("2")
logger.warning("3")
logger.error("4")
logger.critical("5")

可能有些一些关键字不懂，可以点下面链接：

Python中“*”和“**”的用法 || yield的用法 || ‘$in’和'$nin' || python @property的含义

五、requests请求头（http.py）

import random

USER_AGENTS =[ #requests请求头
"Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Avant Browser/1.2.789rel1 (http:#www.avantbrowser.com)",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5", "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
"Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5", "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)", "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110622 Firefox/6.0a2",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0 )", "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90)", "Mozilla/5.0 (Windows; U; Windows XP) Gecko MultiZilla/1.6.1.0a", "Mozilla/2.02E (Win95; U)", "Mozilla/3.01Gold (Win95; I)", "Mozilla/4.8 [en] (Windows NT 5.1; U)", "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.4) Gecko Netscape/7.1 (ax)", "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0", "Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3", "Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1", "Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522 (KHTML, like Gecko) Safari/419.3", "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13", "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
"Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1"]

def get_requests_headers(): #这个方法就是随机获取一个请求头
headers = { #除了user-Agent变一下，其他大多都不用变
'User-Agent' : random.choice(USER_AGENTS),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
'Accept-Encoding': 'gzip,deflate'
}
return headers

if __name__ == '__main__': #模块检查
print(get_requests_headers())

六、python代理池的构建的其他链接

python代理池的构建5——对mongodb数据库里面代理ip检查

python代理池的构建4——mongdb数据库的增删改查

python代理池的构建3——爬取代理ip

python代理池的构建2——代理ip是否可用的处理和检查

七、关于代码一些问题解决链接：

协程gevent模块和猴子补丁

python中schedule模块的简单使用 || importlib.import_module动态导入模块

Python中“*”和“**”的用法 || yield的用法 || ‘$in’和'$nin' || python @property的含义