hbyd_ueba/utils/dashboard_summary_data.py

#!/usr/bin/python
# encoding=utf-8
# author: tangwy
from __future__ import division
import json
import os, re
import codecs
import traceback
from datetime import datetime, timedelta
from collections import defaultdict
from dashboard_data_conversion import adjust_times
from dataInterface.functions import CFunction
from dataInterface.db.params import CPgSqlParam
from ext_logging import logger
from config import read_json_config
from collections import OrderedDict
from urlparse import urlparse
from collections import defaultdict
from appsUtils import env
from ext_logging import logger_trace,APPFOLDERNAME


TABLE_NAME = "ueba_analysis_schema.logs"

DATA_TYPE = {
    "IP": 1,
    "ACCOUNT": 2,
    "INTERFACE": 3,
    "MENU": 4,
}

#安全除
def safe_divide(numerator, denominator):
    if denominator == 0:
        return 
    else:
        return numerator / denominator
#ip维度
def get_ip_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = OrderedDict()

    sql = """ select company, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company order by count desc""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["IP"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result
#账号维度
def get_account_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = OrderedDict()
    sql = """ select company, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company  order by count desc""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["ACCOUNT"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#接口维度
def get_interface_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = OrderedDict()
    sql = """select interface, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by interface order by count desc limit 500""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["INTERFACE"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#菜单维度
def get_menu_summary_data(startTime, endTime):
    """
        IP维度查询
        :param startTime: 开始时间,
        :param endTime: 结束时间,
    """
    result = OrderedDict()
    sql = """select menu, sum(count) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by menu order by count desc""".format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["MENU"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#获取IP count
def get_ip_count(startTime, endTime):
    result = {}
    sql = """select company, count(distinct ip) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company """.format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["IP"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#获取account count
def get_account_count(startTime, endTime):
    result = {}
    sql = """select  company ,count(distinct account) as count from {TABLE_NAME} 
                where logdate >= %s and logdate <= %s and data_type = %s 
                group by company """.format(TABLE_NAME=TABLE_NAME)
    res = json.loads(CFunction.execute(CPgSqlParam(sql, params=(startTime, endTime, DATA_TYPE["ACCOUNT"]))))
    if res:
        for item in res:
            result[item[0]]=item[1]
    return result

#获取前一个周期数据
def get_pre_date(startTime,endTime):
    date_format = "%Y-%m-%d %H:%M:%S"
    start = datetime.strptime(startTime, date_format)
    end = datetime.strptime(endTime, date_format)
    start = start.strftime('%Y-%m-%d')
    end = end.strftime('%Y-%m-%d')
    old_start,old_end = adjust_times(start, end)
    return old_start,old_end

#ip维度汇总数据计算
def ip_summary_calcule(startTime, endTime):
    logger.info("begin")
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_ip_summary_data(startTime=old_start,endTime=old_end)
    logger.info("完成pre_data查询")
    res_data = []
    data = get_ip_summary_data(startTime=startTime,endTime=endTime)
    ip_count_data = get_ip_count(startTime, endTime)
    total_ip_count = sum(ip_count_data.itervalues()) 
    total_frequency = sum(data.itervalues())
    
    for key, value in data.iteritems():
        tmp={}
        tmp["company"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["ip_rate"]=round(safe_divide(ip_count_data[key],total_ip_count),4)
        tmp["ip_count"]=ip_count_data[key]
        tmp["ip_avg"]=round(safe_divide(value,ip_count_data[key]),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    result = {"summary": {"ip": res_data}, "detail": {"ip": {}}}
    return result

#account维度汇总数据计算
def account_summary_calcule(startTime, endTime):
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_account_summary_data(startTime=old_start,endTime=old_end)

    res_data = []
    data = get_account_summary_data(startTime=startTime,endTime=endTime)
    account_count_data = get_account_count(startTime, endTime)
    total_account_count = sum(account_count_data.itervalues()) 
    total_frequency = sum(data.itervalues())
    
    for key, value in data.iteritems():
        tmp={}
        tmp["company"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["account_rate"]=round(safe_divide(account_count_data[key],total_account_count),4)
        tmp["account_count"]=account_count_data[key]
        tmp["account_avg"]=round(safe_divide(value,account_count_data[key]),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    result = {"summary": {"account": res_data}, "detail": {"account": {}}}
    return result

#接口维度汇总数据计算
def interface_summary_calcule(startTime, endTime):
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_interface_summary_data(startTime=old_start,endTime=old_end)

    res_data = []
    data = get_interface_summary_data(startTime=startTime,endTime=endTime)
    #获取黑名单url
    APPHOME = env.get_isop_root() + "/apps/" + APPFOLDERNAME
    config_path = os.path.normpath(APPHOME + "/conf/sys_config.json")
    rule_data = read_json_config(config_path)
    black_url = rule_data["black_url"]

    total_frequency = sum(data.itervalues())
    for key, value in data.iteritems():
        tmp={}
        tmp["interface_addr"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["frequency_avg"]=round(safe_divide(value,20),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        
        parsed_url = urlparse(key)
        url_path = parsed_url.path
        if url_path in black_url:
            continue
        res_data.append(tmp)

    sub_list_length = min(len(res_data), 20)
    sub_list = res_data[:sub_list_length]
    result = {"summary": {"interface": sub_list}, "detail": {"interface": {}}}
    return result

#菜单维度汇总数据计算
def menu_summary_calcule(startTime, endTime):
    logger.info("begin")
    old_start,old_end = get_pre_date(startTime,endTime)
    pre_data = get_menu_summary_data(startTime=old_start,endTime=old_end)
    logger.info("完成pre_data查询")
    res_data = []
    data = get_menu_summary_data(startTime=startTime,endTime=endTime)
    logger.info("完成data查询")
    total_frequency = sum(data.itervalues())
    logger.info("完成合计计算")
    for key, value in data.iteritems():
        tmp={}
        tmp["menu_name"]=key
        tmp["req_frequency"]=value
        tmp["frequency_rate"]=round(safe_divide(value,total_frequency),4)
        tmp["frequency_avg"]=round(safe_divide(value,len(data)),4)
        if key in pre_data:
            tmp["trend"]= round(safe_divide((value-pre_data[key]),pre_data[key]),4)
        else:
            tmp["trend"]=0
        res_data.append(tmp)
    logger.info("完成数据处理")

    sub_list_length = min(len(res_data), 50)
    sub_list = res_data[:sub_list_length]
    result = {"summary": {"menu": sub_list}, "detail": {"menu": {}}}
    return result

#入口
def summary_data_entry(startTime, endTime,data_type):
    data = {}
    if data_type == "1":
        data=ip_summary_calcule(startTime=startTime,endTime=endTime)
    if data_type == "2":
        data=account_summary_calcule(startTime=startTime,endTime=endTime)
    if data_type == "3":
        data=interface_summary_calcule(startTime=startTime,endTime=endTime)
    if data_type == "4":
        data=menu_summary_calcule(startTime=startTime,endTime=endTime)
    
    return data