python分析nignx访问日志脚本分享

    #!/usr/bin/env python 
    # coding=utf-8 

    #------------------------------------------------------ 
    # Name:     nginx 日志分析脚本 
    # Purpose:   此脚本只用来分析nginx的访问日志 
    # Version:   1.0 
    # Author:    LEO 
    # Created:   2013-05-07 
    # Modified:   2013-05-07 
    # Copyright:  (c) LEO 2013 
    #------------------------------------------------------ 

    import sys 
    import time 

    #该类是用来打印格式 
    class displayFormat(object): 

      def format_size(self,size): 
        '''''格式化流量单位''' 
        KB = 1024      #KB -> B B是字节 
        MB = 1048576    #MB -> B 
        GB = 1073741824   #GB -> B 
        TB = 1099511627776 #TB -> B 
        if size >= TB : 
          size = str(size / TB) + 'T' 
        elif size < KB : 
          size = str(size) + 'B' 
        elif size >= GB and size < TB: 
          size = str(size / GB) + 'G' 
        elif size >= MB and size < GB : 
          size = str(size / MB) + 'M' 
        else : 
          size = str(size / KB) + 'K' 
        return size 

      #定义字符串格式化 
      formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s' 

      def transverse_line(self) : 
        '''''输出横线''' 
        print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10) 

      def head(self): 
        '''''输出头部信息''' 
        print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503') 

      def error_print(self) : 
        '''''输出错误信息''' 
        print 
        print 'Usage : ' + sys.argv[0] + ' NginxLogFilePath [Number]' 
        print 
        sys.exit(1) 

      def execut_time(self): 
        '''''输出脚本执行的时间''' 
        print 
        print "Script Execution Time: %.3f second" % time.clock() 
        print 

    #该类是用来生成主机信息的字典 
    class hostInfo(object): 
      host_info = ['200','404','500','302','304','503','403','times','size'] 

      def __init__(self,host): 
        self.host = host = {}.fromkeys(self.host_info,0) 

      def increment(self,status_times_size,is_size): 
        '''''该方法是用来给host_info中的各个值加1''' 
        if status_times_size == 'times': 
          self.host['times'] += 1 
        elif is_size: 
          self.host['size'] = self.host['size'] + status_times_size 
        else: 
          self.host[status_times_size] += 1 

      def get_value(self,value): 
        '''''该方法是取到各个主机信息中对应的值''' 
        return self.host[value] 

    #该类是用来分析文件 
    class fileAnalysis(object): 
      def __init__(self): 
        '''''初始化一个空字典''' 
        self.report_dict = {} 
        self.total_request_times,self.total_traffic,self.total_200, 
        self.total_404,self.total_500,self.total_403,self.total_302, 
        self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0 

      def split_eachline_todict(self,line): 
        '''''分割文件中的每一行，并返回一个字典''' 
        split_line = line.split() 
        split_dict = {'remote_host':split_line[0],'status':split_line[8], 
               'bytes_sent':split_line[9],} 
        return split_dict 

      def generate_log_report(self,logfile): 
        '''''读取文件，分析split_eachline_todict方法生成的字典''' 
        for line in logfile: 
          try: 
            line_dict = self.split_eachline_todict(line) 
            host = line_dict['remote_host'] 
            status = line_dict['status'] 
          except ValueError : 
            continue 
          except IndexError : 
            continue 

          if host not in self.report_dict : 
            host_info_obj = hostInfo(host) 
            self.report_dict[host] = host_info_obj 
          else : 
            host_info_obj = self.report_dict[host] 

          host_info_obj.increment('times',False) 
          if status in host_info_obj.host_info : 
            host_info_obj.increment(status,False) 
          try: 
            bytes_sent = int(line_dict['bytes_sent']) 
          except ValueError: 
            bytes_sent = 0 
          host_info_obj.increment(bytes_sent,True) 
        return self.report_dict 

      def return_sorted_list(self,true_dict): 
        '''''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序''' 
        for host_key in true_dict : 
          host_value = true_dict[host_key] 
          times = host_value.get_value('times')            
          self.total_request_times = self.total_request_times + times 
          size = host_value.get_value('size')            
          self.total_traffic = self.total_traffic + size  

          o200 = host_value.get_value('200') 
          o404 = host_value.get_value('404') 
          o500 = host_value.get_value('500') 
          o403 = host_value.get_value('403') 
          o302 = host_value.get_value('302') 
          o304 = host_value.get_value('304') 
          o503 = host_value.get_value('503') 

          true_dict[host_key] = {'200':o200,'404':o404,'500':o500, 
                      '403':o403,'302':o302,'304':o304, 
                      '503':o503,'times':times,'size':size} 

          self.total_200 = self.total_200 + o200 
          self.total_404 = self.total_404 + o404 
          self.total_500 = self.total_500 + o500 
          self.total_302 = self.total_302 + o302 
          self.total_304 = self.total_304 + o304 
          self.total_503 = self.total_503 + o503 

        sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],
                                   t[1]['size']),reverse=True) 

        return sorted_list 

    class Main(object): 
      def main(self) : 
        '''''主调函数''' 
        display_format = displayFormat() 
        arg_length = len(sys.argv) 
        if arg_length == 1 : 
          display_format.error_print() 
        elif arg_length == 2 or arg_length == 3: 
          infile_name = sys.argv[1] 
          try : 
            infile = open(infile_name,'r') 
            if arg_length == 3 : 
              lines = int(sys.argv[2]) 
            else : 
              lines = 0 
          except IOError,e : 
            print 
            print e 
            display_format.error_print() 
          except ValueError : 
            print 
            print "Please Enter A Volid Number !!" 
            display_format.error_print() 
        else : 
          display_format.error_print() 

        fileAnalysis_obj = fileAnalysis() 
        not_true_dict = fileAnalysis_obj.generate_log_report(infile) 
        log_report = fileAnalysis_obj.return_sorted_list(not_true_dict) 
        total_ip = len(log_report) 
        if lines : 
          log_report = log_report[0:lines] 
        infile.close() 

        print 
        total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic) 
        total_request_times = fileAnalysis_obj.total_request_times 
        print 'Total IP: %s  Total Traffic: %s  Total Request Times: %d' 
           % (total_ip,total_traffic,total_request_times) 
        print 
        display_format.head() 
        display_format.transverse_line() 

        for host in log_report : 
          times = host[1]['times'] 
          times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100 
          print display_format.formatstring % (host[0],
                             display_format.format_size(host[1]['size']),
                             times,str(times_percent)[0:5],
                             host[1]['200'],host[1]['404'],
                             host[1]['500'],host[1]['403'],
                             host[1]['302'],host[1]['304'],host[1]['503']) 

        if (not lines) or total_ip == lines : 
          display_format.transverse_line() 
          print display_format.formatstring % (total_ip,total_traffic, 
                             total_request_times,'100%',
                             fileAnalysis_obj.total_200,
                             fileAnalysis_obj.total_404,
                             fileAnalysis_obj.total_500, 
                             fileAnalysis_obj.total_403,
                             fileAnalysis_obj.total_302, 
                             fileAnalysis_obj.total_304,
                             fileAnalysis_obj.total_503) 

        display_format.execut_time() 

    if __name__ == '__main__': 
      main_obj = Main() 
      main_obj.main()