hbyd_ueba/utils/dashboard_summary_data.py

#!/usr/bin/python
# encoding=utf-8
# author: tangwy
from __future__ import division
import json
import os, re
import codecs
import traceback
from datetime import datetime, timedelta
from collections import defaultdict
from dashboard_data_conversion import adjust_times
from dataInterface.functions import CFunction
from dataInterface.db.params import CPgSqlParam
from ext_logging import logger

TABLE_NAME = "ueba_analysis_schema.logs"

DATA_TYPE = {
    "IP": 1,
    "ACCOUNT": 2,
    "INTERFACE": 3,
    "MENU": 4,
}

#安全除
def safe_divide(numerator, denominator):
    if denominator == 0:
        return 
    else:
        return numerator / denominator
#ip维度
def get_ip_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = {}
    sql = """ select company, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["IP"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result
#账号维度
def get_account_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = {}
    sql = """ select company, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["ACCOUNT"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#接口维度
def get_interface_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = {}
    sql = """select interface, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by interface order by count desc limit 20""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["INTERFACE"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#菜单维度
def get_menu_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = {}
    sql = """select menu, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by menu""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["MENU"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#获取IP count
def get_ip_count(startTime, endTime):
    result = {}
    sql = """select company, count(distinct ip) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company """.format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["IP"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#获取account count
def get_account_count(startTime, endTime):
    result = {}
    sql = """select  company ,count(distinct account) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company """.format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["ACCOUNT"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#获取前一个周期数据
def get_pre_date(startTime,endTime):
    date_format = "%Y-%m-%d %H:%M:%S"
    start = datetime.strptime(startTime, date_format)
    end = datetime.strptime(endTime, date_format)
    start = start.strftime('%Y-%m-%d')
    end = end.strftime('%Y-%m-%d')
    old_start,old_end = adjust_times(start, end)
    return old_start,old_end

#ip维度汇总数据计算
def ip_summary_calcule(startTime, endTime):
    logger.info("begin")
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_ip_summary_data(startTime=old_start,endTime=old_end)
    logger.info("完成pre_data查询")
    res_data = []
    data = get_ip_summary_data(startTime=startTime,endTime=endTime)
    ip_count_data = get_ip_count(startTime, endTime)
    total_ip_count = sum(ip_count_data.itervalues()) 
    total_frequency = sum(data.itervalues())
    
    for key, value in data.iteritems():
        tmp={}
        tmp["company"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["ip_rate"]=round(safe_divide(ip_count_data[key],total_ip_count),4)
        tmp["ip_count"]=ip_count_data[key]
        tmp["ip_avg"]=round(safe_divide(value,ip_count_data[key]),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    result = {"summary": {"ip": res_data}, "detail": {"ip": {}}}
    return result

#account维度汇总数据计算
def account_summary_calcule(startTime, endTime):
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_account_summary_data(startTime=old_start,endTime=old_end)

    res_data = []
    data = get_account_summary_data(startTime=startTime,endTime=endTime)
    account_count_data = get_account_count(startTime, endTime)
    total_account_count = sum(account_count_data.itervalues()) 
    total_frequency = sum(data.itervalues())
    
    for key, value in data.iteritems():
        tmp={}
        tmp["company"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["account_rate"]=round(safe_divide(account_count_data[key],total_account_count),4)
        tmp["account_count"]=account_count_data[key]
        tmp["account_avg"]=round(safe_divide(value,account_count_data[key]),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    result = {"summary": {"account": res_data}, "detail": {"account": {}}}
    return result

#接口维度汇总数据计算
def interface_summary_calcule(startTime, endTime):
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_interface_summary_data(startTime=old_start,endTime=old_end)

    res_data = []
    data = get_interface_summary_data(startTime=startTime,endTime=endTime)
    total_frequency = sum(data.itervalues())
    for key, value in data.iteritems():
        tmp={}
        tmp["interface_addr"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["frequency_avg"]=round(safe_divide(value,20),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    result = {"summary": {"interface": res_data}, "detail": {"interface": {}}}
    return result

#菜单维度汇总数据计算
def menu_summary_calcule(startTime, endTime):
    logger.info("begin")
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_menu_summary_data(startTime=old_start,endTime=old_end)
    logger.info("完成pre_data查询")
    res_data = []
    data = get_menu_summary_data(startTime=startTime,endTime=endTime)
    logger.info("完成data查询")
    total_frequency = sum(data.itervalues())
    logger.info("完成合计计算")
    for key, value in data.iteritems():
        tmp={}
        tmp["menu_name"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["frequency_avg"]=round(safe_divide(value,len(data)),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    logger.info("完成数据处理")
    result = {"summary": {"menu": res_data}, "detail": {"menu": {}}}
    return result

#入口
def summary_data_entry(startTime, endTime,data_type):
    data = {}
    if data_type == "1":
        data=ip_summary_calcule(startTime=startTime,endTime=endTime)
    if data_type == "2":
        data=account_summary_calcule(startTime=startTime,endTime=endTime)
    if data_type == "3":
        data=interface_summary_calcule(startTime=startTime,endTime=endTime)
    if data_type == "4":
        data=menu_summary_calcule(startTime=startTime,endTime=endTime)
    
    return data