python实现爬取图书封面


Posted in Python onJuly 05, 2018

本文实例为大家分享了python实现爬取图书封面的具体代码,供大家参考,具体内容如下

kongfuzi.py

利用更换代理ip,延迟提交数据,设置请求头破解网站的反爬虫机制

import requests
import random
import time
 
 
class DownLoad():
  def __init__(self):
    self.ip_list = ['191.33.179.242:8080', '122.72.108.53:80', '93.190.142.214:80', '189.8.88.125:65301',
            '36.66.55.181:8080', '170.84.102.5:8080', '177.200.72.214:20183', '115.229.115.190:9000']
 
    self.user_agent_list = [
      'User-Agent:Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
      'User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
      'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'
    ]
 
  def get(self, url, proxy=None, timeout=20, num=5):
    print("正在请求%s" % url)
    UA = random.choice(self.user_agent_list)
    headers = {'User-Agent': UA}
 
    if proxy == None:
      try:
        return requests.get(url, headers=headers, timeout=timeout)
      except:
        if num > 0:
          time.sleep(10)
          return self.get(url, num=num - 1)
        else:
          time.sleep(10)
          IP = ''.join(random.choice(self.ip_list).strip())
          proxy = {'http': IP}
          return self.get(url, proxy=proxy, timeout=timeout)
    else:
      try:
        IP = ''.join(random.choice(self.ip_list).strip())
        proxy = {'http': IP}
        return requests.get(url, headers=headers, proxy=proxy, timeout=timeout)
      except:
        if num > 0:
          time.sleep(10)
          IP = ''.join(random.choice(self.ip_list).strip())
          proxy = {'http': IP}
          print("正在更换代理")
          print("当前代理%s" % proxy)
          return self.get(url, proxy=proxy, num=num - 1)

main.py

将爬取的图片保存到本地,然后展示到界面

import kongfuzi
import os
import requests
import bs4
from tkinter import *
from PIL import Image, ImageTk
 
 
# 下载图片,生成图片地址列表和图书信息列表
def download():
  baseUrl = "http://search.kongfz.com"
  keyword = e1.get()
  url = baseUrl + "/product_result/?select=0&key=" + keyword
  print("下载链接:" + url)
  show(url)
 
 
# bs4处理
def changesoup(html):
  htm = html.content
  html_doc = str(htm, 'utf-8')
  soup = bs4.BeautifulSoup(html_doc, "html.parser")
  return soup
 
 
# 图书信息集合
def bookinfo(soup):
  # 图书价格列表
  price = []
  soupprice = soup.select(".first-info .f_right .bold")
  for i in soupprice:
    price.append(i.string)
 
  # 书店名列表
  storename = []
  soupstorename = soup.select(".text a span")
  for each in soupstorename:
    if each.string == None:
      soupstorename.remove(each)
  for i in soupstorename:
    storename.append(i.string)
 
  # 商家地区列表
  place = []
  soupplace = soup.select(".user-place")
  for i in soupplace:
    place.append(i.string)
 
  # 书名列表
  bookname = []
  bookname1 = soup.select(
    ".search-wrap .search-main .search-main-result .result-content .result-list .item .item-info .title .link")
  # print(len(bookname1))
  # print(bookname1)
  for each in bookname1:
    print(each)
    # a = bs4.BeautifulSoup(each, "html.parser")
    a = each.get_text()
    print(a)
    # type(a)
    # a = bs4.BeautifulSoup(a, "html.parser")
    # b = a.get_text()
    bookname.append(a)
  # print(bookname)
  # print(len(bookname))
 
  return bookname, price, place, storename
 
 
# 保存图片
def imgsave(soup):
  dirName = "image"
  os.makedirs(dirName, exist_ok=True)
  filePathList = []
  imgUrl = soup.select(".search-main-result .result-content .result-list .item .item-img .img-box img")
 
  # print(imgUrl)
  if not imgUrl:
    print("没有找到当前节点下图片")
  else:
    i = 0
    for imageUrls in imgUrl:
      # 找到图片地址 获取它
      downloadUrl = imageUrls.get('src')
      # if downloadUrl == "/searchfront/img/error.jpg":
      #   downloadUrl = "http://book.kongfz.com/img/pc/error.jpg"
      print("打印要下载的图片地址:", downloadUrl)
      #   http://book.kongfz.com/img/pc/error.jpg
      # 分割字符
      split = downloadUrl.split("/")
      # 只保留最后一个元素
      fileName = str(i) + "-" + os.path.basename(split[len(split) - 1])
      print("文件名:" + fileName)
      # 建立一个新路径
      filePath = os.path.join(dirName, fileName)
      filePathList.append(filePath)
      if not os.path.exists(filePath):
        imageUrlPath = requests.get(downloadUrl)
        # 检查当前网络是否请求成功
        imageUrlPath.raise_for_status()
        # 'wb'二进制模式打开img适用
        imageFile = open(filePath, 'wb')
        for image in imageUrlPath.iter_content(10000):
          # 把每次遍历的文件图像都存储进文件夹中
          imageFile.write(image)
        # 关闭文件
        imageFile.close()
      i = i + 1
  return filePathList
 
# 图片展示
def show(url):
  xz = kongfuzi.DownLoad()
  html = xz.get(url)
 
  # 添加代理ip到ip_list
  add_ip = e2.get()
  xz.ip_list.append(add_ip)
 
  soup = changesoup(html)
  bookname, price, place, storename = bookinfo(soup)
  # print(bookname)
  # print(price)
  # print(place)
  # print(storename)
  filePathList = imgsave(soup)
  root1 = Toplevel()
  root1.geometry("1720x800")
  root1.title("孔网图片爬取")
 
  # 处理图片,转换成可以显示
  photo = []
  temp = []
  for each in filePathList:
    temp = Image.open(each)
    photo.append(ImageTk.PhotoImage(temp))
 
  canvas = Canvas(root1, width=1700, height=800, scrollregion=(0, 0, 0, 4000)) # 创建canvas
  canvas.place(x=10, y=10) # 放置canvas的位置
 
  frame = Frame(canvas) # 把frame放在canvas里
  frame.place(width=1680, height=800)
 
  for i in range(50):
    # 图片行列
    rownum = int(i / 5)
    columnnum = i % 5
 
    # photo = ImageTk.PhotoImage(Image.open(filePathList[i]))
    imgLabel1 = Label(frame, image=photo[i], width=280, height=280)
    imgLabel1.grid(row=rownum * 5, column=columnnum, padx=10, pady=5)
 
    infoLabel1 = Label(frame, text="书名:" + bookname[i], bg="#FFF8DC", justify=LEFT)
    infoLabel1.grid(row=rownum * 5 + 1, column=columnnum, padx=45, pady=2, sticky=W)
    infoLabel2 = Label(frame, text="价格:" + price[i] + "元", bg="#FFF8DC", justify=LEFT)
    infoLabel2.grid(row=rownum * 5 + 2, column=columnnum, padx=45, pady=2, sticky=W)
    infoLabel3 = Label(frame, text="发货地区:" + place[i], bg="#FFF8DC", justify=LEFT)
    infoLabel3.grid(row=rownum * 5 + 3, column=columnnum, padx=45, pady=2, sticky=W)
    infoLabel4 = Label(frame, text="书店:" + storename[i], bg="#FFF8DC", justify=LEFT)
    infoLabel4.grid(row=rownum * 5 + 4, column=columnnum, padx=45, pady=2, sticky=W)
 
  vbar = Scrollbar(canvas, orient=VERTICAL) # 竖直滚动条
  vbar.place(x=1680, width=20, height=800)
  vbar.configure(command=canvas.yview)
  canvas.config(yscrollcommand=vbar.set) # 设置
  canvas.create_window((800, 2000), window=frame)
 
  mainloop()
 
 
if __name__ == '__main__':
  # 界面
  root = Tk()
  root.title("孔网图片爬取")
  e1 = Entry(root)
  e2 = Entry(root)
  e1.grid(row=0, column=0, padx=20, pady=20)
  e2.grid(row=0, column=2, padx=20, pady=20)
  label1 = Label(root, text="关键字", width=10).grid(row=0, column=1, padx=10, pady=5)
  label2 = Label(root, text="添加代理ip", width=10).grid(row=0, column=3, padx=10, pady=5)
  btn1 = Button(root, text="搜索", width=10, command=download).grid(row=1, column=1, padx=10, pady=5)
  # print(e1.get())
  mainloop()

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
用smtplib和email封装python发送邮件模块类分享
Feb 17 Python
Python实现二叉搜索树
Feb 03 Python
Python使用tablib生成excel文件的简单实现方法
Mar 16 Python
python实现朴素贝叶斯分类器
Mar 28 Python
python 常用的基础函数
Jul 10 Python
Python常见数据类型转换操作示例
May 08 Python
Python pandas.DataFrame调整列顺序及修改index名的方法
Jun 21 Python
django数据库自动重连的方法实例
Jul 21 Python
pytorch 获取tensor维度信息示例
Jan 03 Python
django 模版关闭转义方式
May 14 Python
python如何爬取动态网站
Sep 09 Python
理解python中装饰器的作用
Jul 21 Python
Python定义二叉树及4种遍历方法实例详解
Jul 05 #Python
Python使用pyodbc访问数据库操作方法详解
Jul 05 #Python
如何优雅地处理Django中的favicon.ico图标详解
Jul 05 #Python
解决pandas中读取中文名称的csv文件报错的问题
Jul 04 #Python
解决Python pandas df 写入excel 出现的问题
Jul 04 #Python
python处理数据,存进hive表的方法
Jul 04 #Python
利用Pandas读取文件路径或文件名称包含中文的csv文件方法
Jul 04 #Python
You might like
php获取后台Job管理的实现代码
2011/06/10 PHP
php根据数据id自动生成编号的实现方法
2016/10/16 PHP
PHP基于ORM方式操作MySQL数据库实例
2017/06/21 PHP
PHP校验15位和18位身份证号的类封装
2018/11/07 PHP
PHP7新特性
2021/03/09 PHP
javascript 树控件 比较好用
2009/06/11 Javascript
JS中如何设置readOnly的值
2013/12/25 Javascript
JS自调用匿名函数具体实现
2014/02/11 Javascript
window resize和scroll事件的基本优化思路
2014/04/29 Javascript
javascript数组去重方法终极总结
2014/06/05 Javascript
Jquery代码实现图片轮播效果(一)
2015/08/12 Javascript
jQuery下拉友情链接美化效果代码分享
2015/08/26 Javascript
原生JavaScript实现Ajax的方法
2016/04/07 Javascript
Jquery on绑定的事件 触发多次实例代码
2016/12/08 Javascript
JS闭包与延迟求值用法示例
2016/12/22 Javascript
简单谈谈axios中的get,post方法
2017/06/25 Javascript
vue.js模仿京东省市区三级联动的选择组件实例代码
2017/11/22 Javascript
完美解决axios在ie下的兼容性问题
2018/03/05 Javascript
深入Vue-Router路由嵌套理解
2018/08/13 Javascript
vuejs数据超出单行显示更多,点击展开剩余数据实例
2019/05/05 Javascript
javascript实现画板功能
2020/04/12 Javascript
详解JavaScript作用域 闭包
2020/07/29 Javascript
Python多线程编程(八):使用Event实现线程间通信
2015/04/05 Python
使用pickle存储数据dump 和 load实例讲解
2019/12/30 Python
TFRecord格式存储数据与队列读取实例
2020/01/21 Python
Python中私有属性的定义方式
2020/03/05 Python
快速解决jupyter启动卡死的问题
2020/04/10 Python
python filecmp.dircmp实现递归比对两个目录的方法
2020/05/22 Python
Hertz荷兰:荷兰和全球租车
2018/01/07 全球购物
英国领先的体验日提供商:Buyagift
2019/04/19 全球购物
主持词开场白
2014/03/17 职场文书
《毛主席在花山》教学反思
2014/04/20 职场文书
个人销售励志奋斗口号
2019/12/05 职场文书
Golang之sync.Pool使用详解
2021/05/06 Golang
Python数据可视化之用Matplotlib绘制常用图形
2021/06/03 Python
一文教你快速生成MySQL数据库关系图
2022/06/28 Redis