python分析apache访问日志脚本分享


Posted in Python onFebruary 26, 2015
#!/usr/bin/env python
# coding=utf-8
 
#------------------------------------------------------
# Name:     Apache 日志分析脚本
# Purpose:   此脚本只用来分析Apache的访问日志
# Version:   2.0
# Author:    LEO
# Created:   2013-4-26
# Modified:   2013-5-4
# Copyright:  (c) LEO 2013
#------------------------------------------------------
 
import sys
import time
 
#该类是用来打印格式
class displayFormat(object):
 
  def format_size(self,size):
    '''格式化流量单位'''
    KB = 1024     
    MB = 1048576    
    GB = 1073741824  
    TB = 1099511627776
    if size >= TB :
      size = str(size / TB) + 'T'
    elif size < KB :
      size = str(size) + 'B'
    elif size >= GB and size < TB:
      size = str(size / GB) + 'G'
    elif size >= MB and size < GB :
      size = str(size / MB) + 'M'
    else :
      size = str(size / KB) + 'K'
    return size
 
  formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'
 
  def transverse_line(self) :
    '''输出横线'''
    print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10)
 
  def head(self):
    '''输出头部信息'''
    print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503')
 
  def error_print(self) :
    '''输出错误信息'''
    print
    print 'Usage : ' + sys.argv[0] + ' ApacheLogFilePath [Number]'
    print
    sys.exit(1)
 
  def execut_time(self):
    '''输出脚本执行的时间'''
    print
    print "Script Execution Time: %.3f second" % time.clock()
    print
 
#该类是用来生成主机信息的字典
class hostInfo(object):
   
  host_info = ['200','404','500','302','304','503','403','times','size']
 
  def __init__(self,host):
    self.host = host = {}.fromkeys(self.host_info,0)
 
  def increment(self,status_times_size,is_size):
    '''该方法是用来给host_info中的各个值加1'''
    if status_times_size == 'times':
      self.host['times'] += 1
    elif is_size:
      self.host['size'] = self.host['size'] + status_times_size
    else:
      self.host[status_times_size] += 1
 
  def get_value(self,value):
    '''该方法是取到各个主机信息中对应的值'''
    return self.host[value]
 
#该类是用来分析文件
class fileAnalysis(object):
  def __init__(self):
    '''初始化一个空字典'''
    self.report_dict = {}
    self.total_request_times,self.total_traffic,self.total_200, 
    self.total_404,self.total_500,self.total_403,self.total_302, 
    self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0
 
  def split_eachline_todict(self,line):
    '''分割文件中的每一行,并返回一个字典'''
    split_line = line.split()
    split_dict = {'remote_host':split_line[0],'status':split_line[-2],'bytes_sent':split_line[-1],}
    return split_dict
 
  def generate_log_report(self,logfile):
    '''读取文件,分析split_eachline_todict方法生成的字典'''
    for line in logfile:
      try:
        line_dict = self.split_eachline_todict(line)
        host = line_dict['remote_host']
        status = line_dict['status']
      except ValueError :
        continue
      except IndexError :
        continue
 
      if host not in self.report_dict :
        host_info_obj = hostInfo(host)
        self.report_dict[host] = host_info_obj
      else :
        host_info_obj = self.report_dict[host]
 
      host_info_obj.increment('times',False)  
      if status in host_info_obj.host_info : 
        host_info_obj.increment(status,False) 
      try:
        bytes_sent = int(line_dict['bytes_sent']) 
      except ValueError:
        bytes_sent = 0
      host_info_obj.increment(bytes_sent,True)
    return self.report_dict
 
  def return_sorted_list(self,true_dict):
    '''计算各个状态次数、流量总量,请求的总次数,并且计算各个状态的总量 并生成一个正真的字典,方便排序'''
    for host_key in true_dict :
      host_value = true_dict[host_key]
      times = host_value.get_value('times') 
      self.total_request_times = self.total_request_times + times 
      size = host_value.get_value('size') 
      self.total_traffic = self.total_traffic + size 
 
      o200 = host_value.get_value('200')
      o404 = host_value.get_value('404')
      o500 = host_value.get_value('500')
      o403 = host_value.get_value('403')
      o302 = host_value.get_value('302')
      o304 = host_value.get_value('304')
      o503 = host_value.get_value('503')
 
      true_dict[host_key] = {'200':o200,'404':o404,'500':o500,'403':o403,'302':o302,'304':o304, 
                  '503':o503,'times':times,'size':size}
 
      self.total_200 = self.total_200 + o200
      self.total_404 = self.total_404 + o404
      self.total_500 = self.total_500 + o500
      self.total_302 = self.total_302 + o302
      self.total_304 = self.total_304 + o304
      self.total_503 = self.total_503 + o503
 
    sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],t[1]['size']),reverse=True)
    return sorted_list
 
class Main(object):
  def main(self) :
    '''主调函数'''
    display_format = displayFormat()
    arg_length = len(sys.argv)
    if arg_length == 1 :
      display_format.error_print()
    elif arg_length == 2 or arg_length == 3:
      infile_name = sys.argv[1]
      try :
        infile = open(infile_name,'r')
        if arg_length == 3 :
          lines = int(sys.argv[2])
        else :
          lines = 0
      except IOError,e :
        print
        print e
        display_format.error_print()
      except ValueError :
        print
        print "Please Enter A Volid Number !!"
        display_format.error_print()
    else :
      display_format.error_print()
 
    fileAnalysis_obj = fileAnalysis()
    not_true_dict = fileAnalysis_obj.generate_log_report(infile)
    log_report = fileAnalysis_obj.return_sorted_list(not_true_dict)
    total_ip = len(log_report)
    if lines :
      log_report = log_report[0:lines]
    infile.close()
 
    print
    total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic)
    total_request_times = fileAnalysis_obj.total_request_times
    print 'Total IP: %s  Total Traffic: %s  Total Request Times: %d' 
       % (total_ip,total_traffic,total_request_times)
    print
    display_format.head()
    display_format.transverse_line()
 
    for host in log_report :
      times = host[1]['times']
      times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100
      print display_format.formatstring % (host[0],
                         display_format.format_size(host[1]['size']),
                         times,str(times_percent)[0:5],
                         host[1]['200'],host[1]['404'],
                         host[1]['500'],host[1]['403'],
                         host[1]['302'],host[1]['304'],host[1]['503'])
                         
    if (not lines) or total_ip == lines :
      display_format.transverse_line()
      print display_format.formatstring % (total_ip,total_traffic, 
                         total_request_times,'100%',
                         fileAnalysis_obj.total_200,
                         fileAnalysis_obj.total_404,
                         fileAnalysis_obj.total_500, 
                         fileAnalysis_obj.total_403,
                         fileAnalysis_obj.total_302, 
                         fileAnalysis_obj.total_304,
                         fileAnalysis_obj.total_503)
    display_format.execut_time()
 
if __name__ == '__main__':
  main_obj = Main()
  main_obj.main()
Python 相关文章推荐
Python实现对PPT文件进行截图操作的方法
Apr 28 Python
Python操作串口的方法
Jun 17 Python
python实现JAVA源代码从ANSI到UTF-8的批量转换方法
Aug 10 Python
Python深度优先算法生成迷宫
Jan 22 Python
详解python执行shell脚本创建用户及相关操作
Apr 11 Python
python logging模块的使用总结
Jul 09 Python
Python下应用opencv 实现人脸检测功能
Oct 24 Python
Python+OpenCV实现实时眼动追踪的示例代码
Nov 11 Python
python3 requests库实现多图片爬取教程
Dec 18 Python
Django 设置多环境配置文件载入问题
Feb 25 Python
Python如何在bool函数中取值
Sep 21 Python
使用BeautifulSoup4解析XML的方法小结
Dec 07 Python
Python构造函数及解构函数介绍
Feb 26 #Python
python中的__slots__使用示例
Feb 26 #Python
Python map和reduce函数用法示例
Feb 26 #Python
Python中运行并行任务技巧
Feb 26 #Python
Python通过递归遍历出集合中所有元素的方法
Feb 25 #Python
Python THREADING模块中的JOIN()方法深入理解
Feb 18 #Python
python持久性管理pickle模块详细介绍
Feb 18 #Python
You might like
php小偷相关截取函数备忘
2010/11/28 PHP
PHP面向对象之领域模型+数据映射器实例(分析)
2017/06/21 PHP
JavaScript在IE和Firefox浏览器下的7个差异兼容写法小结
2010/06/18 Javascript
js function定义函数的几种不错方法
2014/02/27 Javascript
JavaScript fontsize方法入门实例(按照指定的尺寸来显示字符串)
2014/10/17 Javascript
jQuery仿淘宝网产品品牌隐藏与显示效果
2015/09/01 Javascript
JavaScript中访问id对象 属性的方式访问属性(实例代码)
2016/10/28 Javascript
JavaScript基础之AJAX简单的小demo
2017/01/29 Javascript
ES6 Promise对象概念与用法分析
2017/04/01 Javascript
详解使用angular的HttpClient搭配rxjs
2017/09/01 Javascript
浅谈vue,angular,react数据双向绑定原理分析
2017/11/28 Javascript
Vue-cli中为单独页面设置背景色的实现方法
2018/02/11 Javascript
JS跨域请求的问题解析
2018/12/03 Javascript
jQuery实现模拟搜索引擎的智能提示功能简单示例
2019/01/27 jQuery
express + jwt + postMan验证实现持久化登录
2019/06/05 Javascript
微信小程序在ios下Echarts图表不能滑动的问题解决
2019/07/10 Javascript
JS中的模糊查询功能
2019/12/08 Javascript
vue相同路由跳转强制刷新该路由组件操作
2020/08/05 Javascript
Vue中关闭弹窗组件时销毁并隐藏操作
2020/09/01 Javascript
[51:06]DOTA2-DPC中国联赛 正赛 Elephant vs Aster BO3 第二场 1月26日
2021/03/11 DOTA
python 域名分析工具实现代码
2009/07/15 Python
Python运算符重载详解及实例代码
2017/03/07 Python
python实现维吉尼亚算法
2019/03/20 Python
使用coverage统计python web项目代码覆盖率的方法详解
2019/08/05 Python
Java文件和目录(IO)操作
2014/08/26 面试题
《月迹》教学反思
2014/02/19 职场文书
餐饮采购员岗位职责
2014/03/15 职场文书
小学生感恩老师演讲稿
2014/08/28 职场文书
运动会稿件100字
2014/09/24 职场文书
员工保密协议书
2014/09/27 职场文书
交警正风肃纪剖析材料
2014/10/29 职场文书
导游词之云南-元阳梯田
2019/10/08 职场文书
你真的了解redis为什么要提供pipeline功能
2021/06/22 Redis
python图片灰度化处理的几种方法
2021/06/23 Python
基于PyQt5制作一个群发邮件工具
2022/04/08 Python
清空 Oracle 安装记录并重新安装
2022/04/26 Oracle