92 lines
2.7 KiB
Python
92 lines
2.7 KiB
Python
|
#!/usr/bin/env python
|
|||
|
|
# -*- encoding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
Topic: 一个简单的日志解析工具
|
|||
|
|
Desc :
|
|||
|
|
日志格式如下：
|
|||
|
|
* 客户端IP地址
|
|||
|
|
* 客户标识：通常不可靠，可以不记录
|
|||
|
|
* 认证用户名：如果无需认证也没有
|
|||
|
|
* 请求接受时间：包括日期、时间、地区
|
|||
|
|
* 请求内容：进一步划分为：方法、资源、请求参数、协议
|
|||
|
|
* 状态码：HTTP状态码
|
|||
|
|
* 返回对象大小：byte为单位
|
|||
|
|
* 提交方Referrer：通常是连接到Web页面或资源的URI或URL
|
|||
|
|
* 用户代理User Agent：客户端程序比如Mozilla、Chrome等
|
|||
|
|
"""
|
|||
|
|
import re
|
|||
|
|
import inspect
|
|||
|
|
from pymongo import Connection
|
|||
|
|
|
|||
|
|
LINE_REGEX = re.compile(r'(\d+\.\d+\.\d+\.\d+) ([^ ]*) ([^ ]*) '
|
|||
|
|
r'\[([^\]]*)\] "([^"]*)" (\d+) ([^ ]*) '
|
|||
|
|
r'"([^"]*)" "([^"]*)"')
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ApacheLogRecord():
|
|||
|
|
def __init__(self, *rgroups):
|
|||
|
|
self.ip, self.ident, \
|
|||
|
|
self.http_user, self.time, \
|
|||
|
|
self.request_line, self.http_response_code, \
|
|||
|
|
self.http_response_size, self.referrer, \
|
|||
|
|
self.user_agent = rgroups
|
|||
|
|
self.http_method, self.url, self.http_vers = self.request_line.split()
|
|||
|
|
|
|||
|
|
def __str__(self):
|
|||
|
|
return ' '.join([self.ip, self.ident, self.time, self.request_line,
|
|||
|
|
self.http_response_code, self.http_response_size,
|
|||
|
|
self.referrer, self.user_agent])
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ApacheLogFile():
|
|||
|
|
def __init__(self, logfile):
|
|||
|
|
self.filename = logfile
|
|||
|
|
|
|||
|
|
def my_generator(self):
|
|||
|
|
_match = LINE_REGEX.match
|
|||
|
|
print(self.filename)
|
|||
|
|
with open(self.filename, encoding='utf-8') as f:
|
|||
|
|
for line in f:
|
|||
|
|
m = _match(line)
|
|||
|
|
if m:
|
|||
|
|
print(line)
|
|||
|
|
try:
|
|||
|
|
log_line = ApacheLogRecord(*m.groups())
|
|||
|
|
yield log_line
|
|||
|
|
except GeneratorExit:
|
|||
|
|
pass
|
|||
|
|
except Exception as e:
|
|||
|
|
print('NON_COMPLIANT_FORMAT: ', line, 'Exception: ', e)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def props(ob):
|
|||
|
|
pr = {}
|
|||
|
|
for name in dir(ob):
|
|||
|
|
val = getattr(ob, name)
|
|||
|
|
if not name.startswith('__') and not inspect.ismethod(val):
|
|||
|
|
pr[name] = val
|
|||
|
|
return pr
|
|||
|
|
|
|||
|
|
|
|||
|
|
def insert_log():
|
|||
|
|
connection = Connection('localhost', 27017)
|
|||
|
|
db = connection.mydb
|
|||
|
|
collection = db.logdata
|
|||
|
|
alf = ApacheLogFile(r'D:\work\gitproject\python3-cookbook\configs\app.log')
|
|||
|
|
for lg_line in alf.my_generator():
|
|||
|
|
collection.insert(props(lg_line))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def query_log():
|
|||
|
|
connection = Connection('localhost', 27017)
|
|||
|
|
db = connection.mydb
|
|||
|
|
collection = db.logdata
|
|||
|
|
for doc in collection.find():
|
|||
|
|
print(doc)
|
|||
|
|
connection.close()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
""""""
|
|||
|
|
query_log()
|