python分析apache访问日志脚本分享


Posted in Python onFebruary 26, 2015
#!/usr/bin/env python
# coding=utf-8
 
#------------------------------------------------------
# Name:     Apache 日志分析脚本
# Purpose:   此脚本只用来分析Apache的访问日志
# Version:   2.0
# Author:    LEO
# Created:   2013-4-26
# Modified:   2013-5-4
# Copyright:  (c) LEO 2013
#------------------------------------------------------
 
import sys
import time
 
#该类是用来打印格式
class displayFormat(object):
 
  def format_size(self,size):
    '''格式化流量单位'''
    KB = 1024     
    MB = 1048576    
    GB = 1073741824  
    TB = 1099511627776
    if size >= TB :
      size = str(size / TB) + 'T'
    elif size < KB :
      size = str(size) + 'B'
    elif size >= GB and size < TB:
      size = str(size / GB) + 'G'
    elif size >= MB and size < GB :
      size = str(size / MB) + 'M'
    else :
      size = str(size / KB) + 'K'
    return size
 
  formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'
 
  def transverse_line(self) :
    '''输出横线'''
    print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10)
 
  def head(self):
    '''输出头部信息'''
    print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503')
 
  def error_print(self) :
    '''输出错误信息'''
    print
    print 'Usage : ' + sys.argv[0] + ' ApacheLogFilePath [Number]'
    print
    sys.exit(1)
 
  def execut_time(self):
    '''输出脚本执行的时间'''
    print
    print "Script Execution Time: %.3f second" % time.clock()
    print
 
#该类是用来生成主机信息的字典
class hostInfo(object):
   
  host_info = ['200','404','500','302','304','503','403','times','size']
 
  def __init__(self,host):
    self.host = host = {}.fromkeys(self.host_info,0)
 
  def increment(self,status_times_size,is_size):
    '''该方法是用来给host_info中的各个值加1'''
    if status_times_size == 'times':
      self.host['times'] += 1
    elif is_size:
      self.host['size'] = self.host['size'] + status_times_size
    else:
      self.host[status_times_size] += 1
 
  def get_value(self,value):
    '''该方法是取到各个主机信息中对应的值'''
    return self.host[value]
 
#该类是用来分析文件
class fileAnalysis(object):
  def __init__(self):
    '''初始化一个空字典'''
    self.report_dict = {}
    self.total_request_times,self.total_traffic,self.total_200, 
    self.total_404,self.total_500,self.total_403,self.total_302, 
    self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0
 
  def split_eachline_todict(self,line):
    '''分割文件中的每一行,并返回一个字典'''
    split_line = line.split()
    split_dict = {'remote_host':split_line[0],'status':split_line[-2],'bytes_sent':split_line[-1],}
    return split_dict
 
  def generate_log_report(self,logfile):
    '''读取文件,分析split_eachline_todict方法生成的字典'''
    for line in logfile:
      try:
        line_dict = self.split_eachline_todict(line)
        host = line_dict['remote_host']
        status = line_dict['status']
      except ValueError :
        continue
      except IndexError :
        continue
 
      if host not in self.report_dict :
        host_info_obj = hostInfo(host)
        self.report_dict[host] = host_info_obj
      else :
        host_info_obj = self.report_dict[host]
 
      host_info_obj.increment('times',False)  
      if status in host_info_obj.host_info : 
        host_info_obj.increment(status,False) 
      try:
        bytes_sent = int(line_dict['bytes_sent']) 
      except ValueError:
        bytes_sent = 0
      host_info_obj.increment(bytes_sent,True)
    return self.report_dict
 
  def return_sorted_list(self,true_dict):
    '''计算各个状态次数、流量总量,请求的总次数,并且计算各个状态的总量 并生成一个正真的字典,方便排序'''
    for host_key in true_dict :
      host_value = true_dict[host_key]
      times = host_value.get_value('times') 
      self.total_request_times = self.total_request_times + times 
      size = host_value.get_value('size') 
      self.total_traffic = self.total_traffic + size 
 
      o200 = host_value.get_value('200')
      o404 = host_value.get_value('404')
      o500 = host_value.get_value('500')
      o403 = host_value.get_value('403')
      o302 = host_value.get_value('302')
      o304 = host_value.get_value('304')
      o503 = host_value.get_value('503')
 
      true_dict[host_key] = {'200':o200,'404':o404,'500':o500,'403':o403,'302':o302,'304':o304, 
                  '503':o503,'times':times,'size':size}
 
      self.total_200 = self.total_200 + o200
      self.total_404 = self.total_404 + o404
      self.total_500 = self.total_500 + o500
      self.total_302 = self.total_302 + o302
      self.total_304 = self.total_304 + o304
      self.total_503 = self.total_503 + o503
 
    sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],t[1]['size']),reverse=True)
    return sorted_list
 
class Main(object):
  def main(self) :
    '''主调函数'''
    display_format = displayFormat()
    arg_length = len(sys.argv)
    if arg_length == 1 :
      display_format.error_print()
    elif arg_length == 2 or arg_length == 3:
      infile_name = sys.argv[1]
      try :
        infile = open(infile_name,'r')
        if arg_length == 3 :
          lines = int(sys.argv[2])
        else :
          lines = 0
      except IOError,e :
        print
        print e
        display_format.error_print()
      except ValueError :
        print
        print "Please Enter A Volid Number !!"
        display_format.error_print()
    else :
      display_format.error_print()
 
    fileAnalysis_obj = fileAnalysis()
    not_true_dict = fileAnalysis_obj.generate_log_report(infile)
    log_report = fileAnalysis_obj.return_sorted_list(not_true_dict)
    total_ip = len(log_report)
    if lines :
      log_report = log_report[0:lines]
    infile.close()
 
    print
    total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic)
    total_request_times = fileAnalysis_obj.total_request_times
    print 'Total IP: %s  Total Traffic: %s  Total Request Times: %d' 
       % (total_ip,total_traffic,total_request_times)
    print
    display_format.head()
    display_format.transverse_line()
 
    for host in log_report :
      times = host[1]['times']
      times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100
      print display_format.formatstring % (host[0],
                         display_format.format_size(host[1]['size']),
                         times,str(times_percent)[0:5],
                         host[1]['200'],host[1]['404'],
                         host[1]['500'],host[1]['403'],
                         host[1]['302'],host[1]['304'],host[1]['503'])
                         
    if (not lines) or total_ip == lines :
      display_format.transverse_line()
      print display_format.formatstring % (total_ip,total_traffic, 
                         total_request_times,'100%',
                         fileAnalysis_obj.total_200,
                         fileAnalysis_obj.total_404,
                         fileAnalysis_obj.total_500, 
                         fileAnalysis_obj.total_403,
                         fileAnalysis_obj.total_302, 
                         fileAnalysis_obj.total_304,
                         fileAnalysis_obj.total_503)
    display_format.execut_time()
 
if __name__ == '__main__':
  main_obj = Main()
  main_obj.main()
Python 相关文章推荐
tensorflow实现加载mnist数据集
Sep 08 Python
Python OpenCV读取png图像转成jpg图像存储的方法
Oct 28 Python
python requests.post带head和body的实例
Jan 02 Python
对python3中, print横向输出的方法详解
Jan 28 Python
Python利用heapq实现一个优先级队列的方法
Feb 03 Python
python ChainMap的使用和说明详解
Jun 11 Python
centos+nginx+uwsgi+Django实现IP+port访问服务器
Nov 15 Python
解决tensorflow打印tensor有省略号的问题
Feb 04 Python
Python中猜拳游戏与猜筛子游戏的实现方法
Sep 04 Python
工程师必须了解的LRU缓存淘汰算法以及python实现过程
Oct 15 Python
使用Python制作一个数据预处理小工具(多种操作一键完成)
Feb 07 Python
Django后端按照日期查询的方法教程
Feb 28 Python
Python构造函数及解构函数介绍
Feb 26 #Python
python中的__slots__使用示例
Feb 26 #Python
Python map和reduce函数用法示例
Feb 26 #Python
Python中运行并行任务技巧
Feb 26 #Python
Python通过递归遍历出集合中所有元素的方法
Feb 25 #Python
Python THREADING模块中的JOIN()方法深入理解
Feb 18 #Python
python持久性管理pickle模块详细介绍
Feb 18 #Python
You might like
php后台多用户权限组思路与实现程序代码分享
2012/02/13 PHP
php读取mysql的简单实例
2014/01/15 PHP
PHP命名空间(Namespace)简明教程
2014/06/11 PHP
Yii2下session跨域名共存的解决方案
2017/02/04 PHP
php抽象方法和普通方法的区别点总结
2019/10/13 PHP
javascript实现轮显新闻标题链接
2007/08/13 Javascript
html中的input标签的checked属性jquery判断代码
2012/09/19 Javascript
我的NodeJs学习小结(一)
2014/07/06 NodeJs
js实现简单折叠、展开菜单的方法
2015/08/28 Javascript
JavaScript+CSS实现的可折叠二级菜单实例
2016/02/29 Javascript
JavaScript核心语法总结(推荐)
2016/06/02 Javascript
深入理解JavaScript单体内置对象
2016/06/06 Javascript
JS操作JSON方法总结(推荐)
2016/06/14 Javascript
jQuery生成假加载动画效果
2016/12/01 Javascript
纯js实现悬浮按钮组件
2016/12/17 Javascript
VUE axios发送跨域请求需要注意的问题
2017/07/06 Javascript
angular-tree-component的使用详解
2018/07/30 Javascript
webpack 如何同时输出压缩和未压缩的文件的实现步骤
2020/06/05 Javascript
[01:14:34]DOTA2上海特级锦标赛C组资格赛#2 LGD VS Newbee第一局
2016/02/28 DOTA
python中使用pyhook实现键盘监控的例子
2014/07/18 Python
浅谈Python中的zip()与*zip()函数详解
2018/02/24 Python
Django 忘记管理员或忘记管理员密码 重设登录密码的方法
2018/05/30 Python
Numpy将二维数组添加到空数组的实现
2019/12/05 Python
通过案例解析python鸭子类型相关原理
2020/10/10 Python
Python内置函数及功能简介汇总
2020/10/13 Python
pycharm中leetcode插件使用图文详解
2020/12/07 Python
ECHT官方网站:男女健身服
2020/02/14 全球购物
瑞典多品牌连锁店:Johnells
2021/01/13 全球购物
八年级生物教学反思
2014/01/22 职场文书
安全标准化实施方案
2014/02/20 职场文书
关于青春的演讲稿
2014/05/05 职场文书
2014年内部审计工作总结
2014/12/09 职场文书
2015年秋季开学典礼校长致辞
2015/07/16 职场文书
退伍军人感言
2015/08/01 职场文书
python开发飞机大战游戏
2021/07/15 Python
MySQL中的引号和反引号的区别与用法详解
2021/10/24 MySQL