Python实现多线程抓取妹子图


Posted in Python onAugust 08, 2015

心血来潮写了个多线程抓妹子图,虽然代码还是有一些瑕疵,但是还是记录下来,分享给大家。

Pic_downloader.py

# -*- coding: utf-8 -*-
"""
Created on Fri Aug 07 17:30:58 2015
 
@author: Dreace
"""
import urllib2
import sys
import time
import os
import random
from multiprocessing.dummy import Pool as ThreadPool 
type_ = sys.getfilesystemencoding()
def rename():
  return time.strftime("%Y%m%d%H%M%S")
def rename_2(name): 
  if len(name) == 2: 
    name = '0' + name + '.jpg' 
  elif len(name) == 1: 
    name = '00' + name + '.jpg' 
  else: 
    name = name + '.jpg' 
  return name
def download_pic(i):
  global count
  global time_out
  if Filter(i):
    try: 
      content = urllib2.urlopen(i,timeout = time_out)
      url_content = content.read()
      f = open(repr(random.randint(10000,999999999)) + "_" + rename_2(repr(count)),"wb")
      f.write(url_content)
      f.close()
      count += 1
    except Exception, e:
      print i + "下载超时,跳过!".decode("utf-8").encode(type_)
def Filter(content):
  for line in Filter_list:
    line=line.strip('\n')
    if content.find(line) == -1:
      return True
def get_pic(url_address):
  global pic_list
  try:
    str_ = urllib2.urlopen(url_address, timeout = time_out).read()
    url_content = str_.split("\"")
    for i in url_content:
      if i.find(".jpg") != -1:
        pic_list.append(i)  
  except Exception, e:
    print "获取图片超时,跳过!".decode("utf-8").encode(type_)
MAX = 2
count = 0
time_out = 60
thread_num = 30
pic_list = []
page_list = []
Filter_list = ["imgsize.ph.126.net","img.ph.126.net","img2.ph.126.net"]
dir_name = "C:\Photos\\"+rename()
os.makedirs(dir_name)
os.chdir(dir_name)
start_time = time.time()
url_address = "http://sexy.faceks.com/?page="
for i in range(1,MAX + 1): 
  page_list.append(url_address + repr(i))
page_pool = ThreadPool(thread_num)
page_pool.map(get_pic,page_list)
print "获取到".decode("utf-8").encode(type_),len(pic_list),"张图片,开始下载!".decode("utf-8").encode(type_)
pool = ThreadPool(thread_num) 
pool.map(download_pic,pic_list)
pool.close() 
pool.join()
print count,"张图片保存在".decode("utf-8").encode(type_) + dir_name
print "共耗时".decode("utf-8").encode(type_),time.time() - start_time,"s"

我们来看下一个网友的作品

#coding: utf-8 #############################################################
# File Name: main.py
# Author: mylonly
# mail: mylonly@gmail.com
# Created Time: Wed 11 Jun 2014 08:22:12 PM CST
#########################################################################
#!/usr/bin/python

import re,urllib2,HTMLParser,threading,Queue,time

#各图集入口链接
htmlDoorList = []
#包含图片的Hmtl链接
htmlUrlList = []
#图片Url链接Queue
imageUrlList = Queue.Queue(0)
#捕获图片数量
imageGetCount = 0
#已下载图片数量
imageDownloadCount = 0
#每个图集的起始地址,用于判断终止
nextHtmlUrl = ''
#本地保存路径
localSavePath = '/data/1920x1080/'

#如果你想下你需要的分辨率的,请修改replace_str,有如下分辨率可供选择1920x1200,1980x1920,1680x1050,1600x900,1440x900,1366x768,1280x1024,1024x768,1280x800
replace_str = '1920x1080'

replaced_str = '960x600'

#内页分析处理类
class ImageHtmlParser(HTMLParser.HTMLParser):
def __init__(self):
self.nextUrl = ''
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
global imageUrlList
if(tag == 'img' and len(attrs) > 2 ):
if(attrs[0] == ('id','bigImg')):
url = attrs[1][1]
url = url.replace(replaced_str,replace_str)
imageUrlList.put(url)
global imageGetCount
imageGetCount = imageGetCount + 1
print url
elif(tag == 'a' and len(attrs) == 4):
if(attrs[0] == ('id','pageNext') and attrs[1] == ('class','next')):
global nextHtmlUrl
nextHtmlUrl = attrs[2][1];

#首页分析类
class IndexHtmlParser(HTMLParser.HTMLParser):
def __init__(self):
self.urlList = []
self.index = 0
self.nextUrl = ''
self.tagList = ['li','a']
self.classList = ['photo-list-padding','pic']
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
if(tag == self.tagList[self.index]):
for attr in attrs:
if (attr[1] == self.classList[self.index]):
if(self.index == 0):
#第一层找到了
self.index = 1
else:
#第二层找到了
self.index = 0
print attrs[1][1]
self.urlList.append(attrs[1][1])
break
elif(tag == 'a'):
for attr in attrs:
if (attr[0] == 'id' and attr[1] == 'pageNext'):
self.nextUrl = attrs[1][1]
print 'nextUrl:',self.nextUrl
break

#首页Hmtl解析器
indexParser = IndexHtmlParser()
#内页Html解析器
imageParser = ImageHtmlParser()

#根据首页得到所有入口链接
print '开始扫描首页...'
host = 'http://desk.zol.com.cn'
indexUrl = '/meinv/'
while (indexUrl != ''):
print '正在抓取网页:',host+indexUrl
request = urllib2.Request(host+indexUrl)
try:
m = urllib2.urlopen(request)
con = m.read()
indexParser.feed(con)
if (indexUrl == indexParser.nextUrl):
break
else:
indexUrl = indexParser.nextUrl
except urllib2.URLError,e:
print e.reason

print '首页扫描完成,所有图集链接已获得:'
htmlDoorList = indexParser.urlList

#根据入口链接得到所有图片的url
class getImageUrl(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
for door in htmlDoorList:
print '开始获取图片地址,入口地址为:',door
global nextHtmlUrl
nextHtmlUrl = ''
while(door != ''):
print '开始从网页%s获取图片...'% (host+door)
if(nextHtmlUrl != ''):
request = urllib2.Request(host+nextHtmlUrl)
else:
request = urllib2.Request(host+door)
try:
m = urllib2.urlopen(request)
con = m.read()
imageParser.feed(con)
print '下一个页面地址为:',nextHtmlUrl
if(door == nextHtmlUrl):
break
except urllib2.URLError,e:
print e.reason
print '所有图片地址均已获得:',imageUrlList

class getImage(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
global imageUrlList
print '开始下载图片...'
while(True):
print '目前捕获图片数量:',imageGetCount
print '已下载图片数量:',imageDownloadCount
image = imageUrlList.get()
print '下载文件路径:',image
try:
cont = urllib2.urlopen(image).read()
patter = '[0-9]*\.jpg';
match = re.search(patter,image);
if match:
print '正在下载文件:',match.group()
filename = localSavePath+match.group()
f = open(filename,'wb')
f.write(cont)
f.close()
global imageDownloadCount
imageDownloadCount = imageDownloadCount + 1
else:
print 'no match'
if(imageUrlList.empty()):
break
except urllib2.URLError,e:
print e.reason
print '文件全部下载完成...'

get = getImageUrl()
get.start()
print '获取图片链接线程启动:'

time.sleep(2)

download = getImage()
download.start()
print '下载图片链接线程启动:'

批量抓取指定网页上的所有图片

# -*- coding:utf-8 -*-
# coding=UTF-8
 
import os,urllib,urllib2,re
 
url = u"http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=python&oq=python&rsp=-1"
outpath = "t:\\"
 
def getHtml(url):
  webfile = urllib.urlopen(url)
  outhtml = webfile.read()
  print outhtml
  return outhtml
 
def getImageList(html):
  restr=ur'('
  restr+=ur'http:\/\/[^\s,"]*\.jpg'
  restr+=ur'|http:\/\/[^\s,"]*\.jpeg'
  restr+=ur'|http:\/\/[^\s,"]*\.png'
  restr+=ur'|http:\/\/[^\s,"]*\.gif'
  restr+=ur'|http:\/\/[^\s,"]*\.bmp'
  restr+=ur'|https:\/\/[^\s,"]*\.jpeg'  
  restr+=ur'|https:\/\/[^\s,"]*\.jpeg'
  restr+=ur'|https:\/\/[^\s,"]*\.png'
  restr+=ur'|https:\/\/[^\s,"]*\.gif'
  restr+=ur'|https:\/\/[^\s,"]*\.bmp'
  restr+=ur')'
  htmlurl = re.compile(restr)
  imgList = re.findall(htmlurl,html)
  print imgList
  return imgList
 
def download(imgList, page):
  x = 1
  for imgurl in imgList:
    filepathname=str(outpath+'pic_%09d_%010d'%(page,x)+str(os.path.splitext(urllib2.unquote(imgurl).decode('utf8').split('/')[-1])[1])).lower()
    print '[Debug] Download file :'+ imgurl+' >> '+filepathname
    urllib.urlretrieve(imgurl,filepathname)
    x+=1
 
def downImageNum(pagenum):
  page = 1
  pageNumber = pagenum
  while(page <= pageNumber):
    html = getHtml(url)#获得url指向的html内容
    imageList = getImageList(html)#获得所有图片的地址,返回列表
    download(imageList,page)#下载所有的图片
    page = page+1
 
if __name__ == '__main__':
  downImageNum(1)

以上就是给大家汇总的3款Python实现的批量抓取妹纸图片的代码了,希望对大家学习Python爬虫能够有所帮助。

Python 相关文章推荐
复制粘贴功能的Python程序
Apr 04 Python
python的描述符(descriptor)、装饰器(property)造成的一个无限递归问题分享
Jul 09 Python
python统计文本字符串里单词出现频率的方法
May 26 Python
使用Numpy读取CSV文件,并进行行列删除的操作方法
Jul 04 Python
python简单操作excle的方法
Sep 12 Python
使用selenium模拟登录解决滑块验证问题的实现
May 10 Python
python调用pyaudio使用麦克风录制wav声音文件的教程
Jun 26 Python
利用Python实现Shp格式向GeoJSON的转换方法
Jul 09 Python
python使用正则来处理各种匹配问题
Dec 22 Python
在PyCharm中实现添加快捷模块
Feb 12 Python
Python 调用 ES、Solr、Phoenix的示例代码
Nov 23 Python
Python 如何将integer转化为罗马数(3999以内)
Jun 05 Python
通过Python来使用七牛云存储的方法详解
Aug 07 #Python
Python爬虫框架Scrapy实战之批量抓取招聘信息
Aug 07 #Python
深入理解Python中命名空间的查找规则LEGB
Aug 06 #Python
举例详解Python中yield生成器的用法
Aug 05 #Python
Python中return语句用法实例分析
Aug 04 #Python
python函数形参用法实例分析
Aug 04 #Python
Python简明入门教程
Aug 04 #Python
You might like
web server使用php生成web页面的三种方法总结
2013/10/28 PHP
主流PHP框架的优缺点对比分析
2014/12/25 PHP
javascript 支持链式调用的异步调用框架Async.Operation
2009/08/04 Javascript
jQuery 源码分析笔记(6) jQuery.data
2011/06/08 Javascript
jquery struts 验证唯一标识(公用方法)
2013/03/27 Javascript
读取input:file的路径并显示本地图片的方法
2013/09/23 Javascript
javascript新建标签,判断键盘输入,以及判断焦点(示例代码)
2013/11/25 Javascript
在JavaScript应用中使用RequireJS来实现延迟加载
2015/07/01 Javascript
jQuery控制DIV层实现由大到小,由远及近动画变化效果
2015/10/09 Javascript
实例讲解避免javascript冲突的方法
2016/01/03 Javascript
谷歌showModalDialog()方法不兼容出现对话窗口的解决办法
2016/02/15 Javascript
bootstrap组件之按钮式下拉菜单小结
2017/01/19 Javascript
JS检测是否可以访问公网服务器功能代码
2017/06/19 Javascript
React + webpack 环境配置的方法步骤
2017/09/07 Javascript
VUEJS 2.0 子组件访问/调用父组件的实例
2018/02/10 Javascript
浅谈Vue2.0中v-for迭代语法的变化(key、index)
2018/03/06 Javascript
jQuery实现图片简单轮播功能示例
2018/08/13 jQuery
layui使用数据表格实现购物车功能
2019/07/26 Javascript
在vue中封装方法以及多处引用该方法详解
2020/08/14 Javascript
Vue中登录验证成功后保存token,并每次请求携带并验证token操作
2020/09/08 Javascript
[03:48]2014DOTA2 TI专访71DK夺冠不靠小组赛高排名
2014/07/11 DOTA
[01:00:26]Ti4主赛事胜者组第一天 EG vs NEWBEE 1
2014/07/19 DOTA
总结Python编程中函数的使用要点
2016/03/20 Python
基于Python和Scikit-Learn的机器学习探索
2017/10/16 Python
numpy的文件存储.npy .npz 文件详解
2018/07/09 Python
Python实现的微信红包提醒功能示例
2019/08/22 Python
Python中filter与lambda的结合使用详解
2019/12/24 Python
python使用rsa非对称加密过程解析
2019/12/28 Python
python+selenium实现12306模拟登录的步骤
2021/01/21 Python
有关HTML5页面在iPhoneX适配问题
2017/11/13 HTML / CSS
如何使用PHP session
2015/04/21 面试题
Linux机考试题
2015/07/17 面试题
服装创业计划书范文
2014/02/05 职场文书
幼儿园家长安全责任书
2014/07/22 职场文书
协议书范文
2015/01/27 职场文书
react antd实现动态增减表单
2021/06/03 Javascript