python利用xpath爬取网上数据并存储到django模型中


Posted in Python onFebruary 26, 2021

帮朋友制作一个网站,需要一些产品数据信息,因为是代理其他公司产品,直接爬取代理公司产品数据

1.设计数据库

from django.db import models
from uuslug import slugify
import uuid
import os


def products_directory_path(instance, filename):
  ext = filename.split('.')[-1]
  filename = '{}.{}'.format(uuid.uuid4().hex[:8], ext)
  # return the whole path to the file
  return os.path.join('images', "products", instance.title, filename)


def product_relatedimage_directory_path(instance, filename):
  ext = filename.split('.')[-1]
  filename = '{}.{}'.format(uuid.uuid4().hex[:8], ext)
  # return the whole path to the file
  return os.path.join('images', "product_relatedimage", instance.product.title, filename)


class ProductsCategory(models.Model):
  """产品分类"""
  name = models.CharField('产品分类名', max_length=80, unique=True)
  description = models.TextField('产品分类描述', blank=True, null=True)
  slug = models.SlugField('slug', max_length=80, blank=True, null=True)
  parent_category = models.ForeignKey('self', verbose_name="父级分类", blank=True, null=True, on_delete=models.CASCADE)

  def save(self, *args, **kwargs):
    if not self.id or not self.slug:
      self.slug = slugify(self.name)
    super().save(*args, **kwargs)

  def __str__(self):
    return self.name

  class Meta:
    ordering = ['name']
    verbose_name = "产品分类"
    verbose_name_plural = verbose_name


class ProductsTag(models.Model):
  """产品标签"""
  name = models.CharField('产品标签名', max_length=30, unique=True)
  slug = models.SlugField('slug', max_length=40)

  def __str__(self):
    return self.name

  def save(self, *args, **kwargs):
    if not self.id or not self.slug:
      self.slug = slugify(self.name)
    super().save(*args, **kwargs)

  class Meta:
    ordering = ['name']
    verbose_name = "产品标签"
    verbose_name_plural = verbose_name


class Product(models.Model):
  title = models.CharField('标题', max_length=255, unique=True)
  slug = models.SlugField('slug', max_length=255, blank=True, null=True)
  jscs = models.TextField('技术参数', blank=True, null=True)
  image = models.ImageField(upload_to=products_directory_path, verbose_name="产品图片")
  views = models.PositiveIntegerField('浏览量', default=0)
  category = models.ForeignKey('ProductsCategory', verbose_name='分类', on_delete=models.CASCADE, blank=True, null=True)
  tags = models.ManyToManyField('ProductsTag', verbose_name='标签集合', blank=True)

  def save(self, *args, **kwargs):
    if not self.id or not self.slug:
      self.slug = slugify(self.title)
    super().save(*args, **kwargs)

  def update_views(self):
    self.views += 1
    self.save(update_fields=['views'])

  def get_pre(self):
    return Product.objects.filter(id__lt=self.id).order_by('-id').first()

  def get_next(self):
    return Product.objects.filter(id__gt=self.id).order_by('id').first()

  def __str__(self):
    return self.title

  class Meta:
    verbose_name = "产品"
    verbose_name_plural = verbose_name


class ProductAdvantage(models.Model):
  content = models.TextField('产品优势', blank=True, null=True)
  product = models.ForeignKey(Product, on_delete=models.CASCADE, blank=True, null=True)

  def __str__(self):
    return self.content

  class Meta:
    verbose_name = "产品优势"
    verbose_name_plural = verbose_name


class ProductBody(models.Model):
  body = models.CharField('产品内容', max_length=256, blank=True, null=True)
  product = models.ForeignKey(Product, on_delete=models.CASCADE, blank=True, null=True)

  def __str__(self):
    return self.product.title

  class Meta:
    verbose_name = "产品内容"
    verbose_name_plural = verbose_name

2.脚本编写

2.1编写获取网页源代码函数

def get_one_page(url):
  try:
    headers = {
      "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
    res = requests.get(url=url, headers=headers)
    res.encoding = 'utf-8'
    if res.status_code == 200:
      return res.text
    else:
      return None
  except Exception:
    return None

2.2根据base页面获取所有产品分类页面链接

if __name__ == '__main__':
  content = get_one_page(url)
  tree = etree.HTML(content)
  # 产品分类url
  catgory_urls = tree.xpath('//div[@class="fdh-01-nav"]/div/h3/a/@href')
  # 处理catgory_urls
  for url in catgory_urls:
    url = 'http://www.kexinjianji.com' + url
    print(url)

2.3根据产品分类页面链接获取对应所有产品链接

if __name__ == '__main__':
  content = get_one_page(url)
  tree = etree.HTML(content)
  # 产品分类
  catgory = tree.xpath('//div[@class="cplb-3n-ts-03 b"]/h3/span/text()')
  print("产品分类:" + catgory[0])
  # 该分类下产品url
  urls = tree.xpath('//div[@class="cplb-3n-ts-03-list"]/dl/dt/a/@href')
  # 处理url
  for url in urls:
    url = 'http://www.kexinjianji.com' + url
    print(url)
  print("=====================================================")

两者结合起来就可以打印出所有产品链接

if __name__ == '__main__':
  content = get_one_page(url)
  tree = etree.HTML(content)
  # 产品分类url
  catgory_urls = tree.xpath('//div[@class="fdh-01-nav"]/div/h3/a/@href')
  # 处理catgory_urls
  for url in catgory_urls:
    url = 'http://www.kexinjianji.com' + url
    content = get_one_page(url)
    tree = etree.HTML(content)
    # 产品分类
    catgory = tree.xpath('//div[@class="cplb-3n-ts-03 b"]/h3/span/text()')
    print("产品分类:" + catgory[0])
    # 该分类下产品url
    urls = tree.xpath('//div[@class="cplb-3n-ts-03-list"]/dl/dt/a/@href')
    # 处理url
    for url in urls:
      url = 'http://www.kexinjianji.com' + url
      print(url)
    print("=====================================================")

2.2使用xpath解析函数返回产品链接的内容

if __name__ == '__main__':
  content = get_one_page(url)
  tree = etree.HTML(content)
  # 产品名称
  title = tree.xpath('//*[@id="wrap"]//h1/text()')
  images = tree.xpath('//div[@class="sol_tj_left"]/a/img/@src')
  # 产品图片
  images_url = 'http://www.kexinjianji.com/' + images[0]
  # 性能特点
  xntd = tree.xpath('//div[@class="w"]//div/span/text()|//div[@class="w"]//div/text()')
  # 技术参数
  jscs = tree.xpath('//table')[0]
  jscs_str = etree.tostring(jscs, encoding='utf-8').decode('utf-8')
  # 产品内容
  cpnr = tree.xpath('//div[@class="describe"]/p')
  print('产品名称:' + title[0])
  print('产品图片:' + images_url)
  for td in xntd:
    print('性能特点:' + td)
  print('技术参数:' + jscs_str)
  for cp in cpnr:
    # string(.) 获取当前标签下所有文本内容
    cp = cp.xpath('string(.)')
    print('产品内容:' + cp)
  print('============================================')

将三者结合在一起就可以获取所有产品信息

if __name__ == '__main__':
  content = get_one_page(url)
  tree = etree.HTML(content)
  # 产品分类url
  catgory_urls = tree.xpath('//div[@class="fdh-01-nav"]/div/h3/a/@href')
  # 处理catgory_urls
  for url in catgory_urls:
    url = 'http://www.kexinjianji.com' + url
    content = get_one_page(url)
    tree = etree.HTML(content)
    # 产品分类
    catgory = tree.xpath('//div[@class="cplb-3n-ts-03 b"]/h3/span/text()')
    # 该分类下产品url
    urls = tree.xpath('//div[@class="cplb-3n-ts-03-list"]/dl/dt/a/@href')
    # 处理url
    for url in urls:
      url = 'http://www.kexinjianji.com' + url
      content = get_one_page(url)
      try:
        tree = etree.HTML(content)
        # 产品名称
        title = tree.xpath('//*[@id="wrap"]//h1/text()')
        images = tree.xpath('//div[@class="sol_tj_left"]/a/img/@src')
        # 产品图片
        images_url = 'http://www.kexinjianji.com' + images[0]
        # 性能特点
        xntd = tree.xpath('//div[@class="w"]//div/span/text()|//div[@class="w"]//div/text()')
        # 技术参数
        jscs = tree.xpath('//table')[0]
        jscs_str = etree.tostring(jscs, encoding='utf-8').decode('utf-8')
        # 产品内容
        cpnr = tree.xpath('//div[@class="describe"]/p')
        print("产品分类:" + catgory[0])
        print('产品链接:' + url)
        print('产品名称:' + title[0])
        print('产品图片:' + images_url)
        for td in xntd:
          print('性能特点:' + td.strip())
        # print('技术参数:' + jscs_str)
        for cp in cpnr:
          # string(.) 获取当前标签下所有文本内容
          cp = cp.xpath('string(.)')
          print('产品内容:' + cp)
        print('============================================')
      except Exception as e:
        print(e)
        print('出错url:' + url)
        pass

3.存储到django模型

import requests
from lxml.html import etree
import os
import django
import uuid
from django.core.files.base import ContentFile

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jiaobanzhan.settings")
django.setup()

from products.models import ProductBody, ProductsCategory, Product, ProductAdvantage

url = 'http://www.kexinjianji.com/product/hzshntjbz_1/'


def get_one_page(url):
  try:
    headers = {
      "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
    res = requests.get(url=url, headers=headers, timeout=10)
    res.encoding = 'utf-8'
    if res.status_code == 200:
      return res.text
    else:
      return None
  except Exception:
    print('aa')
    return None


if __name__ == '__main__':
  content = get_one_page(url)
  tree = etree.HTML(content)
  # 产品分类url
  catgory_urls = tree.xpath('//div[@class="fdh-01-nav"]/div/h3/a/@href')
  # 处理catgory_urls
  for url in catgory_urls:
    url = 'http://www.kexinjianji.com' + url
    content = get_one_page(url)
    tree = etree.HTML(content)
    # 产品分类
    p_catgory = tree.xpath('//div[@class="cplb-3n-ts-03 b"]/h3/span/text()')
    # 该分类下产品url
    urls = tree.xpath('//div[@class="cplb-3n-ts-03-list"]/dl/dt/a/@href')
    # 处理url
    for url in urls:
      url = 'http://www.kexinjianji.com' + url
      content = get_one_page(url)
      try:
        tree = etree.HTML(content)
        # 产品名称
        title = tree.xpath('//*[@id="wrap"]//h1/text()')
        images = tree.xpath('//div[@class="sol_tj_left"]/a/img/@src')
        # 产品图片
        images_url = 'http://www.kexinjianji.com' + images[0]
        # 性能特点
        xntd = tree.xpath('//div[@class="w"]//div/span/text()|//div[@class="w"]//div/text()')
        # 技术参数
        jscs = tree.xpath('//table')[0]
        jscs_str = etree.tostring(jscs, encoding='utf-8').decode('utf-8')
        # 产品内容
        cpnr = tree.xpath('//div[@class="describe"]/p')
        # 判断是否有这分类,没有则新建
        catgory = p_catgory[0]
        products_catgory = ProductsCategory.objects.filter(name=catgory).exists()
        if products_catgory:
          products_catgory = ProductsCategory.objects.get(name=catgory)
        else:
          products_catgory = ProductsCategory(name=catgory)
          products_catgory.save()
        print(products_catgory)

        # 保存产品图片
        image_content = requests.get(url=images_url)
        ext = images_url.split('.')[-1] # 获取图片类型
        filename = '{}.{}'.format(uuid.uuid4().hex[:8], ext) # 随机生成图片名字
        upload_image_file = ContentFile(image_content.content, name=filename) # 将图片保存为django类型
        product = Product(title=title[0], jscs=jscs_str, image=upload_image_file, category=products_catgory)
        product.save()
        for td in xntd:
          product_advantage = ProductAdvantage()
          product_advantage.content = td
          product_advantage.product = product
          product_advantage.save()
        for cp in cpnr:
          cp = cp.xpath('string(.)')
          product_body = ProductBody()
          product_body.body = cp
          product_body.product = product
          product_body.save()
      except Exception as e:
        print(e)
        print('出错url:' + url)

最后自己手动处理出错url(页面没有获取到技术参数,技术参数是一张图片)

4.总结

1.xpath 获取标签内容时,p标签中嵌套span标签,源码如下

<div class="describe" style="position: relative;"> 
   <p><span>板  宽:</span>1500mm</p> 
   <p><span>板  厚:</span>4.5 mm</p> 
   <p><span>出料口:</span>6口</p> 
   <p><span>重  量:</span>6000 kg</p>
</div>

使用xpath获取p标签内容
我想得到的效果如下
板 宽:1500mm
板 厚:4.5 mm
出料口:6口
重 量:6000 kg
使用以下xpath 只能分开获取,不是想要的效果

//div[@class="describe"]/p/span/text()|//div[@class="describe"]/p/text()

百度之后找到的解决办法,使用xpath(‘string(.)')
1.先获取所有p标签

cpnr = tree.xpath('//div[@class="describe"]/p')

2.使用**string(.)**获取所有标签所有文本

cp = cp.xpath('string(.)')

循环遍历所有p标签即可

到此这篇关于python利用xpath爬取网上数据并存储到django模型中的文章就介绍到这了,更多相关xpath爬取网上数据存储到django模型内容请搜索三水点靠木以前的文章或继续浏览下面的相关文章希望大家以后多多支持三水点靠木!

Python 相关文章推荐
Python序列操作之进阶篇
Dec 08 Python
vscode 远程调试python的方法
Dec 01 Python
Python实现可获取网易页面所有文本信息的网易网络爬虫功能示例
Jan 15 Python
python实现二叉查找树实例代码
Feb 08 Python
Python数据类型之Number数字操作实例详解
May 08 Python
利用python、tensorflow、opencv、pyqt5实现人脸实时签到系统
Sep 25 Python
python获取依赖包和安装依赖包教程
Feb 13 Python
python使用Thread的setDaemon启动后台线程教程
Apr 25 Python
基于python生成英文版词云图代码实例
May 16 Python
什么是Python包的循环导入
Sep 08 Python
python mongo 向数据中的数组类型新增数据操作
Dec 05 Python
Python实现socket库网络通信套接字
Jun 04 Python
用python 绘制茎叶图和复合饼图
Feb 26 #Python
python lambda的使用详解
Feb 26 #Python
python爬虫scrapy框架之增量式爬虫的示例代码
Feb 26 #Python
详解Python openpyxl库的基本应用
Feb 26 #Python
解决python的空格和tab混淆而报错的问题
Feb 26 #Python
Python Spyder 调出缩进对齐线的操作
Feb 26 #Python
使用Python制作一盏 3D 花灯喜迎元宵佳节
Feb 26 #Python
You might like
PHP+JQUERY操作JSON实例
2017/03/23 PHP
php和nginx交互实例讲解
2019/09/24 PHP
javascript 设置某DIV区域内的checkbox复选框
2009/11/30 Javascript
jquery 查找新建元素代码
2010/07/06 Javascript
JQuery中的$.getJSON 使用说明
2011/03/10 Javascript
js replace正则表达式应用案例讲解
2013/01/17 Javascript
NodeJS的url截取模块url-extract的使用实例
2013/11/18 NodeJs
JavaScript预解析及相关技巧分析
2016/04/21 Javascript
JS实现表单多文件上传样式美化支持选中文件后删除相关项
2016/09/30 Javascript
Bootstrap Table的使用总结
2016/10/08 Javascript
浅谈struts1 &amp; jquery form 文件异步上传
2017/05/25 jQuery
基于angular2 的 http服务封装的实例代码
2017/06/29 Javascript
Vue中计算属性computed的示例解读
2017/07/26 Javascript
详解如何在项目中使用jest测试react native组件
2018/02/09 Javascript
vue使用中的内存泄漏【推荐】
2018/07/10 Javascript
JS 实现获取验证码 倒计时功能
2018/10/29 Javascript
layui复选框的全选与取消实现方法
2019/09/02 Javascript
解决layui弹框失效的问题
2019/09/09 Javascript
react 原生实现头像滚动播放的示例
2020/04/21 Javascript
微信小程序中data-key属性之数据传输(经验总结)
2020/08/22 Javascript
使用Node.js和Socket.IO扩展Django的实时处理功能
2015/04/20 Python
Python对List中的元素排序的方法
2018/04/01 Python
python中单下划线(_)和双下划线(__)的特殊用法
2019/08/29 Python
python实现静态服务器
2019/09/05 Python
15行Python代码实现免费发送手机短信推送消息功能
2020/02/27 Python
python怎么提高计算速度
2020/06/11 Python
利用html5 canvas破解简单验证码及getImageData接口应用
2013/01/25 HTML / CSS
中国高端家电购物商城:顺电
2018/03/04 全球购物
项目资料员岗位职责
2013/12/10 职场文书
医学专业毕业生个人求职信
2013/12/25 职场文书
青年志愿者事迹材料
2014/02/07 职场文书
简历中个人自我评价分享
2014/03/15 职场文书
校庆标语集锦
2014/06/25 职场文书
2015年元旦联欢晚会活动总结
2014/11/28 职场文书
通知格式
2015/04/27 职场文书
同学聚会开幕词
2019/04/02 职场文书