You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
hbyd_ueba/utils/dashboard_summary_data.py

247 lines
9.0 KiB

#!/usr/bin/python
# encoding=utf-8
# author: tangwy
from __future__ import division
import json
import os, re
import codecs
import traceback
from datetime import datetime, timedelta
from collections import defaultdict
from dashboard_data_conversion import adjust_times
from dataInterface.functions import CFunction
from dataInterface.db.params import CPgSqlParam
from ext_logging import logger
TABLE_NAME = "ueba_analysis_schema.logs"
DATA_TYPE = {
"IP": 1,
"ACCOUNT": 2,
"INTERFACE": 3,
"MENU": 4,
}
#安全除
def safe_divide(numerator, denominator):
if denominator == 0:
return
else:
return numerator / denominator
#ip维度
def get_ip_summary_data(startTime, endTime):
"""
IP维度查询
:param startTime: 开始时间,
:param endTime: 结束时间,
"""
result = {}
sql = """ select company, sum(count) as count from {TABLE_NAME}
where logdate >= %s and logdate <= %s and data_type = %s
group by company""".format(TABLE_NAME=TABLE_NAME)
res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["IP"]))))
if res:
for item in res:
result[item[0]]=item[1]
return result
#账号维度
def get_account_summary_data(startTime, endTime):
"""
IP维度查询
:param startTime: 开始时间,
:param endTime: 结束时间,
"""
result = {}
sql = """ select company, sum(count) as count from {TABLE_NAME}
where logdate >= %s and logdate <= %s and data_type = %s
group by company""".format(TABLE_NAME=TABLE_NAME)
res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["ACCOUNT"]))))
if res:
for item in res:
result[item[0]]=item[1]
return result
#接口维度
def get_interface_summary_data(startTime, endTime):
"""
IP维度查询
:param startTime: 开始时间,
:param endTime: 结束时间,
"""
result = {}
sql = """select interface, sum(count) as count from {TABLE_NAME}
where logdate >= %s and logdate <= %s and data_type = %s
group by interface order by count desc limit 20""".format(TABLE_NAME=TABLE_NAME)
res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["INTERFACE"]))))
if res:
for item in res:
result[item[0]]=item[1]
return result
#菜单维度
def get_menu_summary_data(startTime, endTime):
"""
IP维度查询
:param startTime: 开始时间,
:param endTime: 结束时间,
"""
result = {}
sql = """select menu, sum(count) as count from {TABLE_NAME}
where logdate >= %s and logdate <= %s and data_type = %s
group by menu""".format(TABLE_NAME=TABLE_NAME)
res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["MENU"]))))
if res:
for item in res:
result[item[0]]=item[1]
return result
#获取IP count
def get_ip_count(startTime, endTime):
result = {}
sql = """select company, count(distinct ip) as count from {TABLE_NAME}
where logdate >= %s and logdate <= %s and data_type = %s
group by company """.format(TABLE_NAME=TABLE_NAME)
res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["IP"]))))
if res:
for item in res:
result[item[0]]=item[1]
return result
#获取account count
def get_account_count(startTime, endTime):
result = {}
sql = """select company ,count(distinct account) as count from {TABLE_NAME}
where logdate >= %s and logdate <= %s and data_type = %s
group by company """.format(TABLE_NAME=TABLE_NAME)
res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["ACCOUNT"]))))
if res:
for item in res:
result[item[0]]=item[1]
return result
#获取前一个周期数据
def get_pre_date(startTime,endTime):
date_format = "%Y-%m-%d %H:%M:%S"
start = datetime.strptime(startTime, date_format)
end = datetime.strptime(endTime, date_format)
start = start.strftime('%Y-%m-%d')
end = end.strftime('%Y-%m-%d')
old_start,old_end = adjust_times(start, end)
return old_start,old_end
#ip维度汇总数据计算
def ip_summary_calcule(startTime, endTime):
logger.info("begin")
old_start,old_end = get_pre_date(startTime,endTime)
pre_data = get_ip_summary_data(startTime=old_start,endTime=old_end)
logger.info("完成pre_data查询")
res_data = []
data = get_ip_summary_data(startTime=startTime,endTime=endTime)
ip_count_data = get_ip_count(startTime, endTime)
total_ip_count = sum(ip_count_data.itervalues())
total_frequency = sum(data.itervalues())
for key, value in data.iteritems():
tmp={}
tmp["company"]=key
tmp["req_frequency"]=value
tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
tmp["ip_rate"]=round(safe_divide(ip_count_data[key],total_ip_count),4)
tmp["ip_count"]=ip_count_data[key]
tmp["ip_avg"]=round(safe_divide(value,ip_count_data[key]),4)
if key in pre_data:
tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
else:
tmp["trend"]=0
res_data.append(tmp)
result = {"summary": {"ip": res_data}, "detail": {"ip": {}}}
return result
#account维度汇总数据计算
def account_summary_calcule(startTime, endTime):
old_start,old_end = get_pre_date(startTime,endTime)
pre_data = get_account_summary_data(startTime=old_start,endTime=old_end)
res_data = []
data = get_account_summary_data(startTime=startTime,endTime=endTime)
account_count_data = get_account_count(startTime, endTime)
total_account_count = sum(account_count_data.itervalues())
total_frequency = sum(data.itervalues())
for key, value in data.iteritems():
tmp={}
tmp["company"]=key
tmp["req_frequency"]=value
tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
tmp["account_rate"]=round(safe_divide(account_count_data[key],total_account_count),4)
tmp["account_count"]=account_count_data[key]
tmp["account_avg"]=round(safe_divide(value,account_count_data[key]),4)
if key in pre_data:
tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
else:
tmp["trend"]=0
res_data.append(tmp)
result = {"summary": {"account": res_data}, "detail": {"account": {}}}
return result
#接口维度汇总数据计算
def interface_summary_calcule(startTime, endTime):
old_start,old_end = get_pre_date(startTime,endTime)
pre_data = get_interface_summary_data(startTime=old_start,endTime=old_end)
res_data = []
data = get_interface_summary_data(startTime=startTime,endTime=endTime)
total_frequency = sum(data.itervalues())
for key, value in data.iteritems():
tmp={}
tmp["interface_addr"]=key
tmp["req_frequency"]=value
tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
tmp["frequency_avg"]=round(safe_divide(value,20),4)
if key in pre_data:
tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
else:
tmp["trend"]=0
res_data.append(tmp)
result = {"summary": {"interface": res_data}, "detail": {"interface": {}}}
return result
#菜单维度汇总数据计算
def menu_summary_calcule(startTime, endTime):
logger.info("begin")
old_start,old_end = get_pre_date(startTime,endTime)
pre_data = get_menu_summary_data(startTime=old_start,endTime=old_end)
logger.info("完成pre_data查询")
res_data = []
data = get_menu_summary_data(startTime=startTime,endTime=endTime)
logger.info("完成data查询")
total_frequency = sum(data.itervalues())
logger.info("完成合计计算")
for key, value in data.iteritems():
tmp={}
tmp["menu_name"]=key
tmp["req_frequency"]=value
tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
tmp["frequency_avg"]=round(safe_divide(value,len(data)),4)
if key in pre_data:
tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
else:
tmp["trend"]=0
res_data.append(tmp)
logger.info("完成数据处理")
result = {"summary": {"menu": res_data}, "detail": {"menu": {}}}
return result
#入口
def summary_data_entry(startTime, endTime,data_type):
data = {}
if data_type == "1":
data=ip_summary_calcule(startTime=startTime,endTime=endTime)
if data_type == "2":
data=account_summary_calcule(startTime=startTime,endTime=endTime)
if data_type == "3":
data=interface_summary_calcule(startTime=startTime,endTime=endTime)
if data_type == "4":
data=menu_summary_calcule(startTime=startTime,endTime=endTime)
return data