爬虫代理池Python3WebSpider源代码测试过程解析


Posted in Python onDecember 20, 2019

这篇文章主要介绍了爬虫代理池Python3WebSpider源代码测试过程解析,文中通过示例代码介绍的非常详细,对大家的学习或者工作具有一定的参考学习价值,需要的朋友可以参考下

元类属性的使用

代码

主要关于元类的使用

通过获取由元类生成的爬虫抓取类的部分属性.这里为抓取函数,以相同的字符开头的抓取函数,生成属性列表,这样可以持续调用.目的是可以仅仅添加不同的抓取函数抓取不同的网站,而类的其他部分不用做调整.

部分代码:

class ProxyMetaclass(type):
  def __new__(cls, name, bases, attrs):
    count = 0
    attrs['__CrawlFunc__'] = []
    for k, v in attrs.items():
      if 'crawl_' in k:
        attrs['__CrawlFunc__'].append(k)
        count += 1
    attrs['__CrawlFuncCount__'] = count
    return type.__new__(cls, name, bases, attrs)


class Crawler(object, metaclass=ProxyMetaclass):
  def get_proxies(self, callback):
    proxies = []
    for proxy in eval("self.{}()".format(callback)):
      print('成功获取到代理', proxy)
      proxies.append(proxy)
    return proxies
    
  def crawl_daili66(self, page_count=4):
    """
    获取代理66
    :param page_count: 页码
    :return: 代理
    """
    start_url = 'http://www.66ip.cn/{}.html'
    urls = [start_url.format(page) for page in range(1, page_count + 1)]
    for url in urls:
      print('Crawling', url)
      html = get_page(url)
      if html:
        doc = pq(html)
        trs = doc('.containerbox table tr:gt(0)').items()
        for tr in trs:
          ip = tr.find('td:nth-child(1)').text()
          port = tr.find('td:nth-child(2)').text()
          yield ':'.join([ip, port])

测试方法

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time  : 12/19/19 4:10 PM
# @Author : yon
# @Email  : @qq.com
# @File  : test


import json
import re
from pyquery import PyQuery as pq


class ProxyMetaclass(type):
  def __new__(cls, name, bases, attrs):
    count = 0
    attrs['__CrawlFunc__'] = []
    for k, v in attrs.items():
      print("打印k")
      print(k)
      print("打印v")
      print(v)
      if 'crawl_' in k:
        attrs['__CrawlFunc__'].append(k)
        count += 1
    attrs['__CrawlFuncCount__'] = count
    return type.__new__(cls, name, bases, attrs)


class Crawler(object, metaclass=ProxyMetaclass):
  def get_proxies(self, callback):
    proxies = []
    for proxy in eval("self.{}()".format(callback)):
      print('成功获取到代理', proxy)
      proxies.append(proxy)
    return proxies

  def crawl_daili66(self, page_count=4):
    """
    获取代理66
    :param page_count: 页码
    :return: 代理
    """
    start_url = 'http://www.66ip.cn/{}.html'
    urls = [start_url.format(page) for page in range(1, page_count + 1)]
    for url in urls:
      print('Crawling', url)
      html = get_page(url)
      if html:
        doc = pq(html)
        trs = doc('.containerbox table tr:gt(0)').items()
        for tr in trs:
          ip = tr.find('td:nth-child(1)').text()
          port = tr.find('td:nth-child(2)').text()
          yield ':'.join([ip, port])

  def crawl_ip3366(self):
    for page in range(1, 4):
      start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page)
      html = get_page(start_url)
      ip_address = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
      # \s * 匹配空格,起到换行作用
      re_ip_address = ip_address.findall(html)
      for address, port in re_ip_address:
        result = address + ':' + port
        yield result.replace(' ', '')

  def crawl_kuaidaili(self):
    for i in range(1, 4):
      start_url = 'http://www.kuaidaili.com/free/inha/{}/'.format(i)
      html = get_page(start_url)
      if html:
        ip_address = re.compile('<td data-title="IP">(.*?)</td>')
        re_ip_address = ip_address.findall(html)
        port = re.compile('<td data-title="PORT">(.*?)</td>')
        re_port = port.findall(html)
        for address, port in zip(re_ip_address, re_port):
          address_port = address + ':' + port
          yield address_port.replace(' ', '')

  def crawl_xicidaili(self):
    for i in range(1, 3):
      start_url = 'http://www.xicidaili.com/nn/{}'.format(i)
      headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Cookie': '_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJWRjYzc5MmM1MTBiMDMzYTUzNTZjNzA4NjBhNWRjZjliBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUp6S2tXT3g5a0FCT01ndzlmWWZqRVJNek1WanRuUDBCbTJUN21GMTBKd3M9BjsARg%3D%3D--2a69429cb2115c6a0cc9a86e0ebe2800c0d471b3',
        'Host': 'www.xicidaili.com',
        'Referer': 'http://www.xicidaili.com/nn/3',
        'Upgrade-Insecure-Requests': '1',
      }
      html = get_page(start_url, options=headers)
      if html:
        find_trs = re.compile('<tr class.*?>(.*?)</tr>', re.S)
        trs = find_trs.findall(html)
        for tr in trs:
          find_ip = re.compile('<td>(\d+\.\d+\.\d+\.\d+)</td>')
          re_ip_address = find_ip.findall(tr)
          find_port = re.compile('<td>(\d+)</td>')
          re_port = find_port.findall(tr)
          for address, port in zip(re_ip_address, re_port):
            address_port = address + ':' + port
            yield address_port.replace(' ', '')

  def crawl_ip3366(self):
    for i in range(1, 4):
      start_url = 'http://www.ip3366.net/?stype=1&page={}'.format(i)
      html = get_page(start_url)
      if html:
        find_tr = re.compile('<tr>(.*?)</tr>', re.S)
        trs = find_tr.findall(html)
        for s in range(1, len(trs)):
          find_ip = re.compile('<td>(\d+\.\d+\.\d+\.\d+)</td>')
          re_ip_address = find_ip.findall(trs[s])
          find_port = re.compile('<td>(\d+)</td>')
          re_port = find_port.findall(trs[s])
          for address, port in zip(re_ip_address, re_port):
            address_port = address + ':' + port
            yield address_port.replace(' ', '')

  def crawl_iphai(self):
    start_url = 'http://www.iphai.com/'
    html = get_page(start_url)
    if html:
      find_tr = re.compile('<tr>(.*?)</tr>', re.S)
      trs = find_tr.findall(html)
      for s in range(1, len(trs)):
        find_ip = re.compile('<td>\s+(\d+\.\d+\.\d+\.\d+)\s+</td>', re.S)
        re_ip_address = find_ip.findall(trs[s])
        find_port = re.compile('<td>\s+(\d+)\s+</td>', re.S)
        re_port = find_port.findall(trs[s])
        for address, port in zip(re_ip_address, re_port):
          address_port = address + ':' + port
          yield address_port.replace(' ', '')

  def crawl_data5u(self):
    start_url = 'http://www.data5u.com/free/gngn/index.shtml'
    headers = {
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
      'Accept-Encoding': 'gzip, deflate',
      'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
      'Cache-Control': 'max-age=0',
      'Connection': 'keep-alive',
      'Cookie': 'JSESSIONID=47AA0C887112A2D83EE040405F837A86',
      'Host': 'www.data5u.com',
      'Referer': 'http://www.data5u.com/free/index.shtml',
      'Upgrade-Insecure-Requests': '1',
      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
    }
    html = get_page(start_url, options=headers)
    if html:
      ip_address = re.compile('<span><li>(\d+\.\d+\.\d+\.\d+)</li>.*?<li class=\"port.*?>(\d+)</li>', re.S)
      re_ip_address = ip_address.findall(html)
      for address, port in re_ip_address:
        result = address + ':' + port
        yield result.replace(' ', '')


class Getter():
  def __init__(self):
    self.crawler = Crawler()

  def run(self):
    print('获取器开始执行')
    for callback_label in range(self.crawler.__CrawlFuncCount__):
      print(callback_label)
      callback = self.crawler.__CrawlFunc__[callback_label]
      print(callback)
      # # 获取代理
      # proxies = self.crawler.get_proxies(callback)
      # sys.stdout.flush()
      # for proxy in proxies:
      #   self.redis.add(proxy)
if __name__ == '__main__':
  get = Getter()
  get.run()

测试结果

/home/baixiaoxu/PycharmProjects/pytthon-tt/venv/bin/python /home/baixiaoxu/PycharmProjects/pytthon-tt/proxypool/test.py
打印k
__module__
打印v
__main__
打印k
__qualname__
打印v
Crawler
打印k
get_proxies
打印v
<function Crawler.get_proxies at 0x7f905ca5a598>
打印k
crawl_daili66
打印v
<function Crawler.crawl_daili66 at 0x7f905ca5a620>
打印k
crawl_ip3366
打印v
<function Crawler.crawl_ip3366 at 0x7f905ca5a840>
打印k
crawl_kuaidaili
打印v
<function Crawler.crawl_kuaidaili at 0x7f905ca5a730>
打印k
crawl_xicidaili
打印v
<function Crawler.crawl_xicidaili at 0x7f905ca5a7b8>
打印k
crawl_iphai
打印v
<function Crawler.crawl_iphai at 0x7f905ca5a6a8>
打印k
crawl_data5u
打印v
<function Crawler.crawl_data5u at 0x7f905ca5a8c8>
打印k
__CrawlFunc__
打印v
['crawl_daili66', 'crawl_ip3366', 'crawl_kuaidaili', 'crawl_xicidaili', 'crawl_iphai', 'crawl_data5u']
获取器开始执行
0
crawl_daili66
1
crawl_ip3366
2
crawl_kuaidaili
3
crawl_xicidaili
4
crawl_iphai
5
crawl_data5u

进程完成,退出码 0

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
python迭代器的使用方法实例
Nov 21 Python
深入理解Python装饰器
Jul 27 Python
浅谈五大Python Web框架
Mar 20 Python
Django内容增加富文本功能的实例
Oct 17 Python
python调用Matplotlib绘制分布点并且添加标签
May 31 Python
使用 Python 写一个简易的抽奖程序
Dec 08 Python
使用tensorflow DataSet实现高效加载变长文本输入
Jan 20 Python
Python json模块与jsonpath模块区别详解
Mar 05 Python
150行Python代码实现带界面的数独游戏
Apr 04 Python
python pip如何手动安装二进制包
Sep 30 Python
详解使用python爬取抖音app视频(appium可以操控手机)
Jan 26 Python
关于Python使用turtle库画任意图的问题
Apr 01 Python
python3的UnicodeDecodeError解决方法
Dec 20 #Python
基于python调用psutil模块过程解析
Dec 20 #Python
python如何使用jt400.jar包代码实例
Dec 20 #Python
基于python使用tibco ems代码实例
Dec 20 #Python
使用python实现数组、链表、队列、栈的方法
Dec 20 #Python
python隐藏类中属性的3种实现方法
Dec 19 #Python
Python合并2个字典成1个新字典的方法(9种)
Dec 19 #Python
You might like
PHPMailer 中文使用说明小结
2010/01/22 PHP
使用php将某个目录下面的所有文件罗列出来的方法详解
2013/06/21 PHP
Laravel框架分页实现方法分析
2018/06/12 PHP
Laravel推荐使用的十个辅助函数
2019/05/10 PHP
My Desktop :) 桌面式代码
2008/12/29 Javascript
使弱类型的语言JavaScript变强势
2009/06/22 Javascript
javascript window.opener的用法分析
2010/04/07 Javascript
js中小数转换整数的方法
2014/01/26 Javascript
javascript删除一个html元素节点的方法
2014/12/20 Javascript
js实现鼠标滑过文字链接色彩变化的效果
2015/05/06 Javascript
jquery Deferred 快速解决异步回调的问题
2016/04/05 Javascript
jQuery插件jquery.kxbdmarquee.js实现无缝滚动效果
2017/02/15 Javascript
修改UA在PC中访问只能在微信中打开的链接方法
2017/11/27 Javascript
详解angular2 控制视图的封装模式
2018/12/27 Javascript
微信小程序自定义头部导航栏和导航栏背景图片 navigationStyle问题
2019/07/26 Javascript
node.js使用stream模块实现自定义流示例
2020/02/13 Javascript
vue中echarts图表大小适应窗口大小且不需要刷新案例
2020/07/19 Javascript
Element Rate 评分的使用方法
2020/07/27 Javascript
vue中如何自定义右键菜单详解
2020/12/08 Vue.js
[03:56]还原FTP电影首映式 DOTA2群星拼出遗迹世界
2014/03/26 DOTA
[03:58]2014DOTA2国际邀请赛 龙宝赛后解密DK获胜之道
2014/07/14 DOTA
[01:48]2018DOTA2亚洲邀请赛主赛事第二日五佳镜头 VG完美团战逆转TNC
2018/04/05 DOTA
python正则中最短匹配实现代码
2018/01/16 Python
python实现在cmd窗口显示彩色文字
2019/06/24 Python
python 中pyqt5 树节点点击实现多窗口切换问题
2019/07/04 Python
Pycharm远程调试原理及具体配置详解
2019/08/08 Python
基于CSS3 animation动画属性实现轮播图效果
2017/09/12 HTML / CSS
伦敦剧院门票:From The Box Office
2018/06/30 全球购物
有模特经验的简历自我评价
2013/09/19 职场文书
大学生应聘自荐信
2013/10/11 职场文书
英文留学推荐信范文
2014/01/25 职场文书
公司人事专员岗位职责
2014/08/11 职场文书
学校党的群众路线教育实践活动整改措施
2014/10/25 职场文书
匿名信格式范文
2015/05/27 职场文书
幼儿园教师教育随笔
2015/08/14 职场文书
基层医务人员三严三实心得体会
2016/01/05 职场文书