python3 log文件处理获取某天需要的数据
时间:2022-05-08
本文章向大家介绍python3 log文件处理获取某天需要的数据,主要内容包括其使用实例、应用技巧、基本知识点总结和需要注意事项,具有一定的参考价值,需要的朋友可以参考一下。
问题描述:需要读取log文件,并获取出两天的日志做对比,查询增加或者减少量,规则化输出 完成心得:python的字典,很好用,读写时间快,用法自由
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 17/8/18 下午5:08
# @Author : lee
# @File : read.py
# @Software: PyCharm
# 说明: code后有'#'的时测试时加的或者需要修改的code
# 用法 传入两个值 day1是前一天的日期 day2是后一天的内容
import sys
def log_data(address, day1, day2):
try:
# 打开文件
log = open("%s" % address)
# 定义:
dict_yesterday = {} # 昨天的数据
dict_the_day_before_yesterday = {} # 前天的数据
dict_yesterday_difference = {} # 记录昨天中有的数据,前天没有
dict_the_day_before_yesterday_difference = {} # 和上一条相反
dict_identical = {} # 记录两天内相同的数据
dict_result = {} # 记录结果,输出时读取
ii = jj = z = x = 0
#遍历 将两个日期的日志筛选出并装入对应的字典
for line in log:
log_data = ','.join(filter(lambda x: x,line.split(' ')))
date = (log_data.split(',')[0])[-10:].replace('.','-')
if date == day2:
dict_yesterday[ii] = log_data
dict_yesterday_difference[ii] = log_data
ii += 1
elif date == day1:
dict_the_day_before_yesterday[jj] = log_data
dict_the_day_before_yesterday_difference[jj] = log_data
jj += 1
if len(dict_yesterday) >= len(dict_the_day_before_yesterday):
for i in dict_yesterday:
object_list1 = (dict_yesterday[i]).split(',')
entry_name_yesterday = ((object_list1[0]).split(',')[0])[:-10]+object_list1[1]+'-'+object_list1[2]
for j in dict_the_day_before_yesterday:
object_list2 = (dict_the_day_before_yesterday[j]).split(',')
entry_name_tdat = ((object_list2[0]).split(',')[0])[:-10]+object_list2[1]+'-'+object_list2[2]
if entry_name_yesterday == entry_name_tdat:
dict_identical[x] = entry_name_yesterday
docs_result = int(((dict_yesterday[i]).split(','))[4]) - int(((dict_the_day_before_yesterday[j]).split(','))[4])
capacity1 = (dict_yesterday[i].split(',')[5])[-2:]
capacity2 = (dict_the_day_before_yesterday[j].split(',')[5])[-2:]
if capacity1 == capacity2:
store_result = str(round(float((dict_yesterday[i].split(',')[5])[:-2]) - float((dict_the_day_before_yesterday[j].split(',')[5])[:-2]),2)) +(dict_yesterday[i].split(',')[5])[-2:]
# 1gb = 1024mb = 1024kb
elif capacity1 != capacity2:
n_y = round(float((dict_yesterday[i].split(',')[5])[:-2]), 2)
n_t = float((dict_the_day_before_yesterday[j].split(',')[5])[:-2])
if capacity1 == 'gb'and capacity2 == 'mb':
if n_y * 1024 > n_t:
store_result = str(abs(round(n_y * 1024 - n_t), 2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
else:
store_result = str(round(n_y * 1024 - n_t),) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
elif capacity1 == 'gb'and capacity2 == 'kb':
if n_y * 1024 * 1024 > n_t:
# round(,2)
store_result = str(abs(round(n_y * 1024 * 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
else:
store_result = str(round(n_y * 1024 * 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
elif capacity1 == 'mb'and capacity2 == 'gb':
if n_y / 1024 > n_t:
store_result = str(abs(round(n_y / 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
else:
store_result = str(round(n_y / 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
elif capacity1 == 'mb'and capacity2 == 'kb':
if n_y * 1024 > n_t:
store_result = str(abs(round(n_y * 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
else:
store_result = str(round(n_y * 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
elif capacity1 == 'kb'and capacity2 == 'gb':
store_result = str(round(n_t - n_t * 1024 * 1024,2)) +(dict_yesterday[i].split(',')[5])[-2:]
elif capacity1 == 'kb'and capacity2 == 'mb':
if n_y / 1024 > n_t:
store_result = str(abs(round(n_t - n_t * 1024,2))) +(dict_yesterday[i].split(',')[5])[-2:]
else:
store_result = str(round(n_t - n_t * 1024,2)) +(dict_yesterday[i].split(',')[5])[-2:]
dict_result[z] = entry_name_tdat + ' docs: ' + str(docs_result) + ' store: ' +store_result
z += 1
x += 1
# else:
# print("当前日期下没有相关日志")
# 筛选出昨天出现 前天没有的数据
if len(dict_yesterday) >= len(dict_identical):
for i in dict_yesterday:
object_list_1 = (dict_yesterday[i]).split(',')
entry_name_yesterday_1 = ((object_list_1[0]).split(',')[0])[:-10]+object_list_1[1]+'-'+object_list_1[2]
for j in dict_identical:
entry_name_identical = dict_identical[j]
if entry_name_yesterday_1 == entry_name_identical:
dict_yesterday_difference.pop(i)
# 筛选出前天出现 昨天没有的数据
if len(dict_the_day_before_yesterday) >= len(dict_identical):
for i in dict_the_day_before_yesterday:
object_list_1 = (dict_the_day_before_yesterday[i]).split(',')
entry_name_yesterday_1 = ((object_list_1[0]).split(',')[0])[:-10]+object_list_1[1]+'-'+object_list_1[2]
for j in dict_identical:
entry_name_identical = dict_identical[j]
if entry_name_yesterday_1 == entry_name_identical:
dict_the_day_before_yesterday_difference.pop(i)
# 输出两天都有的数据
print("下面是 %s 和 %s 内%s中 docs、store差值"%(day1,day2,address))
print('')
print("++++++++++++++++两天都有的数据++++++++++++++++")
for i in dict_result:
print(dict_result[i])
# 输出两天中前者中有的数据而在后者没有出现,取正值(+)区别
print('')
if len(dict_yesterday) != 0 :
print("++++++++++++++++只有%s有的数据取正值(+)予以区别++++++++++++++++"%day2)
for i in dict_yesterday_difference:
object = (dict_yesterday_difference[i]).split(',')
name = ((object[0]).split(',')[0])[:-10]+object[1]+'-'+object[2]
docs = object[4]
store = object[5]
print(name + ' docs: +' + str(docs) + ' store: ' + store)
else:
print("%s日期下没有相关日志"%day2)
# 输出两天中后者中有的数据而在前者没有出现,取负值(-)区别
print('')
if len(dict_the_day_before_yesterday) != 0:
print("++++++++++++++++只有%s有的数据,都取负值(-)予以区别++++++++++++++++" % day1)
for i in dict_the_day_before_yesterday_difference:
object = (dict_the_day_before_yesterday_difference[i]).split(',')
name = ((object[0]).split(',')[0])[:-10]+object[1]+'-'+object[2]
docs = object[4]
store = object[5]
print(name+ ' docs: -' + str(docs) + ' store: -' + store)
else:
print("%s日期下没有相关日志"%day1)
except:
print("输入的参数可能不对,三个参数,地址,day1 day2")
# log_data('/Users/lee/Desktop/py交易/es.log','2017-08-11','2017-08-12')
log_data(sys.argv[1],sys.argv[2],sys.argv[3])
# /Users/lee/Desktop/py交易/test.log
- JavaScript 教程
- JavaScript 编辑工具
- JavaScript 与HTML
- JavaScript 与Java
- JavaScript 数据结构
- JavaScript 基本数据类型
- JavaScript 特殊数据类型
- JavaScript 运算符
- JavaScript typeof 运算符
- JavaScript 表达式
- JavaScript 类型转换
- JavaScript 基本语法
- JavaScript 注释
- Javascript 基本处理流程
- Javascript 选择结构
- Javascript if 语句
- Javascript if 语句的嵌套
- Javascript switch 语句
- Javascript 循环结构
- Javascript 循环结构实例
- Javascript 跳转语句
- Javascript 控制语句总结
- Javascript 函数介绍
- Javascript 函数的定义
- Javascript 函数调用
- Javascript 几种特殊的函数
- JavaScript 内置函数简介
- Javascript eval() 函数
- Javascript isFinite() 函数
- Javascript isNaN() 函数
- parseInt() 与 parseFloat()
- escape() 与 unescape()
- Javascript 字符串介绍
- Javascript length属性
- javascript 字符串函数
- Javascript 日期对象简介
- Javascript 日期对象用途
- Date 对象属性和方法
- Javascript 数组是什么
- Javascript 创建数组
- Javascript 数组赋值与取值
- Javascript 数组属性和方法
- PHP实现微信商户支付企业付款到零钱功能
- PHP调用微博接口实现微博登录的办法示例
- thinkphp3.2同时连接两个数据库的简单方法
- php实现微信企业付款到个人零钱功能
- php中对象引用和复制实例分析
- php中上传文件的的解决方案
- PHP实现与java 通信的插件使用教程
- thinkPHP5框架接口写法简单示例
- php实现数组重复数字统计实例
- php提取微信账单的有效信息
- php使用pecl方式安装扩展操作示例
- RSA实现JS前端加密与PHP后端解密功能示例
- Laravel源码解析之路由的使用和示例详解
- Linux中crontab输出重定向不生效问题的解决办法
- PHP swoole和redis异步任务实现方法分析