Python爬虫Scrapy框架HTTP代理的配置与调试——万能调试

# Scrapy 内置的 Downloader Middleware 为 Scrapy 供了基础的功能， 
# 定义一个类，其中(object)可以不写，效果一样 
class SimpleProxyMiddleware(object): 
    # 声明一个数组 
    proxyList = ['你购买的HTTP代理地址'] 
    
    # Downloader Middleware的核心方法，只有实现了其中一个或多个方法才算自定义了一个 Downloader Middleware def process_request(self, request, spider): 
        # 随机从其中选择一个，并去除左右两边空格 
        proxy = random.choice(self.proxyList).strip() 
        # 打印结果出来观察 
        print("this is request ip:" + proxy) 
        # 设置request的proxy属性的内容为代理ip 
        request.meta['proxy'] = proxy 
        
    # Downloader Middleware的核心方法，只有实现了其中一个或多个方法才算自定义了一个Downloader Middleware def process_response(self, request, response, spider): 
    # 请求失败不等于200 
    if response.status != 200: 
        # 重新选择一个代理ip 
        proxy = random.choice(self.proxyList).strip() 
        print("this is response ip:" + proxy) 
        # 设置新的代理ip内容 
        request.mete['proxy'] = proxy 
        return request 
    return response

#DDWNLQADER_MIDDLEWARES = {
#        *images.middlewares.ImagesDownloaderMiddleware": 543，
    #    *images middlewares，LocalProxyMiddleware*: 100
#}

#DDWNLQADER_MIDDLEWARES = {
  'scrapydownloadertest.middlewares.SimpleProxyMiddleware': 100,
|
}

# -*- coding: utf-8 -*-
import scrapy

class HttpproxyipSpider(scrapy.Spider):
    name = 'httpProxyIp'
    allowed_domains = ['icanhazip.com']
    start_urls = ['http://icanhazip.com/']
  
    def parse(self, response):
        pass

# -*- coding: utf-8 -*-
import scrapy
from scrapy.cmdline import execute

class HttpproxyipSpider(scrapy.Spider):
        # spider 任务名
    name = 'httpProxyIp'
    # 允许访问的域名
    allowed_domains = ['icanhazip.com']
    # 起始爬取的url
    start_urls = ['http://icanhazip.com/']
    
    #spider 爬虫解析的方法，关于内容的解析都在这里完成; self表示实例的引用，response爬虫的结果
    def parse(self, response):
        print('代理后的ip: ', response.text)

#这个是main函数也是整个程序入口的惯用写法
if __name__ == '__main__':
    execute(['scrapy', 'crawl', 'httpbin'])