python分析nignx访问日志脚本分享


Posted in Python onFebruary 26, 2015
#!/usr/bin/env python 
# coding=utf-8 
 
#------------------------------------------------------ 
# Name:     nginx 日志分析脚本 
# Purpose:   此脚本只用来分析nginx的访问日志 
# Version:   1.0 
# Author:    LEO 
# Created:   2013-05-07 
# Modified:   2013-05-07 
# Copyright:  (c) LEO 2013 
#------------------------------------------------------ 
 
import sys 
import time 
 
#该类是用来打印格式 
class displayFormat(object): 
 
  def format_size(self,size): 
    '''''格式化流量单位''' 
    KB = 1024      #KB -> B B是字节 
    MB = 1048576    #MB -> B 
    GB = 1073741824   #GB -> B 
    TB = 1099511627776 #TB -> B 
    if size >= TB : 
      size = str(size / TB) + 'T' 
    elif size < KB : 
      size = str(size) + 'B' 
    elif size >= GB and size < TB: 
      size = str(size / GB) + 'G' 
    elif size >= MB and size < GB : 
      size = str(size / MB) + 'M' 
    else : 
      size = str(size / KB) + 'K' 
    return size 
 
  #定义字符串格式化 
  formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s' 
 
  def transverse_line(self) : 
    '''''输出横线''' 
    print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10) 
 
  def head(self): 
    '''''输出头部信息''' 
    print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503') 
 
  def error_print(self) : 
    '''''输出错误信息''' 
    print 
    print 'Usage : ' + sys.argv[0] + ' NginxLogFilePath [Number]' 
    print 
    sys.exit(1) 
 
  def execut_time(self): 
    '''''输出脚本执行的时间''' 
    print 
    print "Script Execution Time: %.3f second" % time.clock() 
    print 
 
#该类是用来生成主机信息的字典 
class hostInfo(object): 
  host_info = ['200','404','500','302','304','503','403','times','size'] 
 
  def __init__(self,host): 
    self.host = host = {}.fromkeys(self.host_info,0) 
 
  def increment(self,status_times_size,is_size): 
    '''''该方法是用来给host_info中的各个值加1''' 
    if status_times_size == 'times': 
      self.host['times'] += 1 
    elif is_size: 
      self.host['size'] = self.host['size'] + status_times_size 
    else: 
      self.host[status_times_size] += 1 
 
  def get_value(self,value): 
    '''''该方法是取到各个主机信息中对应的值''' 
    return self.host[value] 
 
#该类是用来分析文件 
class fileAnalysis(object): 
  def __init__(self): 
    '''''初始化一个空字典''' 
    self.report_dict = {} 
    self.total_request_times,self.total_traffic,self.total_200, 
    self.total_404,self.total_500,self.total_403,self.total_302, 
    self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0 
 
  def split_eachline_todict(self,line): 
    '''''分割文件中的每一行,并返回一个字典''' 
    split_line = line.split() 
    split_dict = {'remote_host':split_line[0],'status':split_line[8], 
           'bytes_sent':split_line[9],} 
    return split_dict 
 
  def generate_log_report(self,logfile): 
    '''''读取文件,分析split_eachline_todict方法生成的字典''' 
    for line in logfile: 
      try: 
        line_dict = self.split_eachline_todict(line) 
        host = line_dict['remote_host'] 
        status = line_dict['status'] 
      except ValueError : 
        continue 
      except IndexError : 
        continue 
 
      if host not in self.report_dict : 
        host_info_obj = hostInfo(host) 
        self.report_dict[host] = host_info_obj 
      else : 
        host_info_obj = self.report_dict[host] 
 
      host_info_obj.increment('times',False) 
      if status in host_info_obj.host_info : 
        host_info_obj.increment(status,False) 
      try: 
        bytes_sent = int(line_dict['bytes_sent']) 
      except ValueError: 
        bytes_sent = 0 
      host_info_obj.increment(bytes_sent,True) 
    return self.report_dict 
 
  def return_sorted_list(self,true_dict): 
    '''''计算各个状态次数、流量总量,请求的总次数,并且计算各个状态的总量 并生成一个正真的字典,方便排序''' 
    for host_key in true_dict : 
      host_value = true_dict[host_key] 
      times = host_value.get_value('times')            
      self.total_request_times = self.total_request_times + times 
      size = host_value.get_value('size')            
      self.total_traffic = self.total_traffic + size  
 
      o200 = host_value.get_value('200') 
      o404 = host_value.get_value('404') 
      o500 = host_value.get_value('500') 
      o403 = host_value.get_value('403') 
      o302 = host_value.get_value('302') 
      o304 = host_value.get_value('304') 
      o503 = host_value.get_value('503') 
 
      true_dict[host_key] = {'200':o200,'404':o404,'500':o500, 
                  '403':o403,'302':o302,'304':o304, 
                  '503':o503,'times':times,'size':size} 
 
      self.total_200 = self.total_200 + o200 
      self.total_404 = self.total_404 + o404 
      self.total_500 = self.total_500 + o500 
      self.total_302 = self.total_302 + o302 
      self.total_304 = self.total_304 + o304 
      self.total_503 = self.total_503 + o503 
 
    sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],
                               t[1]['size']),reverse=True) 
 
    return sorted_list 
 
class Main(object): 
  def main(self) : 
    '''''主调函数''' 
    display_format = displayFormat() 
    arg_length = len(sys.argv) 
    if arg_length == 1 : 
      display_format.error_print() 
    elif arg_length == 2 or arg_length == 3: 
      infile_name = sys.argv[1] 
      try : 
        infile = open(infile_name,'r') 
        if arg_length == 3 : 
          lines = int(sys.argv[2]) 
        else : 
          lines = 0 
      except IOError,e : 
        print 
        print e 
        display_format.error_print() 
      except ValueError : 
        print 
        print "Please Enter A Volid Number !!" 
        display_format.error_print() 
    else : 
      display_format.error_print() 
 
    fileAnalysis_obj = fileAnalysis() 
    not_true_dict = fileAnalysis_obj.generate_log_report(infile) 
    log_report = fileAnalysis_obj.return_sorted_list(not_true_dict) 
    total_ip = len(log_report) 
    if lines : 
      log_report = log_report[0:lines] 
    infile.close() 
 
    print 
    total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic) 
    total_request_times = fileAnalysis_obj.total_request_times 
    print 'Total IP: %s  Total Traffic: %s  Total Request Times: %d' 
       % (total_ip,total_traffic,total_request_times) 
    print 
    display_format.head() 
    display_format.transverse_line() 
 
    for host in log_report : 
      times = host[1]['times'] 
      times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100 
      print display_format.formatstring % (host[0],
                         display_format.format_size(host[1]['size']),
                         times,str(times_percent)[0:5],
                         host[1]['200'],host[1]['404'],
                         host[1]['500'],host[1]['403'],
                         host[1]['302'],host[1]['304'],host[1]['503']) 
                         
    if (not lines) or total_ip == lines : 
      display_format.transverse_line() 
      print display_format.formatstring % (total_ip,total_traffic, 
                         total_request_times,'100%',
                         fileAnalysis_obj.total_200,
                         fileAnalysis_obj.total_404,
                         fileAnalysis_obj.total_500, 
                         fileAnalysis_obj.total_403,
                         fileAnalysis_obj.total_302, 
                         fileAnalysis_obj.total_304,
                         fileAnalysis_obj.total_503) 
 
    display_format.execut_time() 
 
if __name__ == '__main__': 
  main_obj = Main() 
  main_obj.main()
Python 相关文章推荐
python生成器的使用方法
Nov 21 Python
python中enumerate函数用法实例分析
May 20 Python
深入讲解Python中的迭代器和生成器
Oct 26 Python
使用Python读写及压缩和解压缩文件的示例
Jul 08 Python
python实现上传下载文件功能
Nov 19 Python
利用Django内置的认证视图实现用户密码重置功能详解
Nov 24 Python
Python实现按中文排序的方法示例
Apr 25 Python
python中sys.argv函数精简概括
Jul 08 Python
Django 内置权限扩展案例详解
Mar 04 Python
Django models文件模型变更错误解决
May 11 Python
Pycharm新手使用教程(图文详解)
Sep 17 Python
一个入门级python爬虫教程详解
Jan 27 Python
python分析apache访问日志脚本分享
Feb 26 #Python
Python构造函数及解构函数介绍
Feb 26 #Python
python中的__slots__使用示例
Feb 26 #Python
Python map和reduce函数用法示例
Feb 26 #Python
Python中运行并行任务技巧
Feb 26 #Python
Python通过递归遍历出集合中所有元素的方法
Feb 25 #Python
Python THREADING模块中的JOIN()方法深入理解
Feb 18 #Python
You might like
php面向对象的方法重载两种版本比较
2008/09/08 PHP
php压缩HTML函数轻松实现压缩html/js/Css及注意事项
2013/01/27 PHP
PHP获取MSN好友列表类的实现代码
2013/06/23 PHP
如何在Ubuntu下启动Apache的Rewrite功能
2013/07/05 PHP
基于php实现七牛抓取远程图片
2015/12/01 PHP
PHP验证码生成原理和实现
2016/01/24 PHP
php实现图片按比例截取的方法
2017/02/06 PHP
Laravel 5.5基于内置的Auth模块实现前后台登陆详解
2017/12/21 PHP
极酷的javascirpt,让你随意编辑任何网页
2007/02/25 Javascript
基于dom编程中 动态创建与删除元素的使用
2013/04/17 Javascript
JQuery对表单元素的基本操作使用总结
2014/07/18 Javascript
jQuery+jRange实现滑动选取数值范围特效
2015/03/14 Javascript
jquery实现多条件筛选特效代码分享
2015/08/28 Javascript
全面解析JavaScript的Backbone.js框架中的Router路由
2016/05/05 Javascript
jQuery封装的屏幕居中提示信息代码
2016/06/08 Javascript
jQuery Ajax全解析
2017/02/13 Javascript
详解vue后台系统登录态管理
2019/04/02 Javascript
JS控制只能输入数字并且最多允许小数点两位
2019/11/24 Javascript
js实现烟花特效
2020/03/02 Javascript
微信小程序自定义弹出层效果
2020/05/26 Javascript
python操作MySQL 模拟简单银行转账操作
2017/09/27 Python
python3+PyQt5图形项的自定义和交互 python3实现page Designer应用程序
2020/07/20 Python
详解通过API管理或定制开发ECS实例
2018/09/30 Python
Python 静态方法和类方法实例分析
2019/11/21 Python
解决Tensorflow 使用时cpu编译不支持警告的问题
2020/02/03 Python
python实现处理mysql结果输出方式
2020/04/09 Python
python如何查看网页代码
2020/06/07 Python
python动态规划算法实例详解
2020/11/22 Python
CSS3实现伪类hover离开时平滑过渡效果示例
2017/08/10 HTML / CSS
银行简历自我评价
2014/02/11 职场文书
幼儿教师求职信
2014/05/24 职场文书
2015年党风廉政建设目标责任书
2015/05/08 职场文书
2015年反腐倡廉工作总结
2015/05/14 职场文书
侵犯商业秘密的律师函
2015/05/27 职场文书
Html5新增了哪些功能
2021/04/16 HTML / CSS
MySQL 执行数据库更新update操作的时候数据库卡死了
2022/05/02 MySQL