hbyd_ueba/utils/dashboard_data_conversion.py

# coding=utf-8
from __future__ import division
import json
from datetime import timedelta,datetime
from collections import defaultdict

jobnum_region_dict = {
    "10": "省公司",
    "110": "武汉分公司",
    "170": "襄阳分公司",
    "130": "鄂州分公司",
    "260": "孝感分公司",
    "250": "黄冈分公司",
    "120": "黄石分公司",
    "190": "咸宁分公司",
    "200": "荆州分公司",
    "140": "宜昌分公司",
    "150": "恩施分公司",
    "160": "十堰分公司",
    "240": "随州分公司",
    "230": "荆门分公司",
    "1801": "江汉分公司",
    "1802": "潜江分公司",
    "1803": "天门分公司"
}


def keep_digits_filter(code):
    """
    工号可能有字面，去掉字面，以数字起算
    """
    return ''.join(filter(str.isdigit, str(code)))

#安全除
def safe_divide(numerator, denominator):
    if denominator == 0:
        return 
    else:
        return numerator / denominator
def find_region_by_code(code, region_dict):
    """
    查询工号对应公司
    未查询到 返回错误工号
    """
    code_str = keep_digits_filter(code)
    # 使用生成器表达式和next函数尝试找到匹配的前缀
    company = next(
        (region_dict.get(code_str[:i]) for i in range(2, min(len(code_str), 5)) if code_str[:i] in region_dict),
        "错误工号")
    return company


def ip_summary_data_format(ip_summary_data):
    """
        ip维度数据转换方法
    """
    result = {"summary": {"ip": []}, "detail": {"ip": {}}}
    grouped_data = defaultdict(lambda: {"reqs": 0, "ips": set()})
    ip_detail_dict = defaultdict(lambda: defaultdict(lambda: {"req_frequency": 0}))
    # 全部账号元组
    ips_total = set()
    for ip_data in ip_summary_data:
        company = find_region_by_code(ip_data["jobnum"], jobnum_region_dict)
        count = ip_data["count"]
        ip = ip_data["ip"]
        jobnum = ip_data["jobnum"]
        ip_detail_dict_key = "{}{}".format(ip, jobnum)

        # 更新统计数据
        grouped_data[company]["reqs"] += count
        grouped_data[company]["ips"].add(ip)
        ips_total.add(ip)
        # 构建下钻详情
        ip_detail_dict[company][ip_detail_dict_key]["req_ip"] = ip
        ip_detail_dict[company][ip_detail_dict_key]["req_jobnum"] = jobnum
        ip_detail_dict[company][ip_detail_dict_key]["req_frequency"] += count

    # 统计总请求次数和独立IP数
    reqs_total = sum(data["reqs"] for data in grouped_data.values())
    # 请求为0抛出
    if reqs_total == 0:
        return result

    # 构建summary部分
    ip_data_list = [
        {
            "company": company,
            "req_frequency": data["reqs"],
            # 本公司的 请求次数/所有公司 请求次数的合计
            "frequency_rate": round(data["reqs"] / reqs_total, 4),
            "ip_count": len(data["ips"]),
            # 本公司的 ip个数/所有公司 ip个数的合计
            "ip_rate": round(len(data["ips"]) / len(ips_total), 4),
            # 本公司的 请求次数/本公司 ip个数的合计
            "ip_avg": safe_divide(data["reqs"],len(data["ips"])),
        }
        for company, data in grouped_data.items()
    ]

    result["summary"]["ip"] = sorted(ip_data_list, key=lambda x: x["req_frequency"], reverse=True)

    # 构建detail部分
    result["detail"]["ip"] = {
        company: sorted(data.values(), key=lambda x: x['req_frequency'], reverse=True)[:500]
        for company, data in ip_detail_dict.items()
    }

    return result


def account_summary_data_format(account_summary_data):
    """
            账号维度数据转换方法
    """
    result = {"summary": {"account": []}, "detail": {"account": {}}}
    grouped_data = defaultdict(lambda: {"reqs": 0, "accounts": set()})
    account_detail_dict = defaultdict(lambda: defaultdict(lambda: {"req_frequency": 0}))
    accounts_total = set()
    for account_data in account_summary_data:
        company = find_region_by_code(account_data["jobnum"], jobnum_region_dict)
        count = account_data["count"]
        account = account_data["account"]
        jobnum = account_data["jobnum"]
        account_detail_dict_key = "{}{}".format(account, jobnum)

        # 更新统计数据
        grouped_data[company]["reqs"] += count
        grouped_data[company]["accounts"].add(account)
        accounts_total.add(account)

        # 更新下钻详情
        account_detail_dict[company][account_detail_dict_key]["req_account"] = account
        account_detail_dict[company][account_detail_dict_key]["req_jobnum"] = jobnum
        account_detail_dict[company][account_detail_dict_key]["req_frequency"] += count

    # 统计总请求次数和独立账号数
    reqs_total = sum(data["reqs"] for data in grouped_data.values())
    
    # 请求为0抛出
    if reqs_total == 0:
        return result

    # 构建summary部分
    account_data_list = [
        {
            "company": company,
            "req_frequency": data["reqs"],
            # 本公司的 请求次数/所有公司 请求次数的合计
            "frequency_rate": round(data["reqs"] / reqs_total, 4),
            "account_count": len(data["accounts"]),
            # 本公司的 账号次数/所有公司 账号次数的合计
            "account_rate": round(len(data["accounts"]) / len(accounts_total), 4),
            # 本公司的 请求次数/本公司 账号次数的合计
            "account_avg": safe_divide(data["reqs"],len(data["accounts"])),
        }
        for company, data in grouped_data.items()
    ]
    result["summary"]["account"] = sorted(account_data_list, key=lambda x: x["req_frequency"], reverse=True)

    # 构建detail部分
    result["detail"]["account"] = {company: sorted(data.values(), key=lambda x: x['req_frequency'], reverse=True)[:500]
                                   for company, data in account_detail_dict.items()}

    return result


def interface_summary_data_format(interface_summary_data):
    """
        接口维度数据转换方法
    """
    result = {"summary": {"interface": []}, "detail": {"interface": {}}}
    grouped_data = defaultdict(lambda: {"reqs": 0})
    interface_detail_dict = defaultdict(lambda: defaultdict(lambda: {"req_frequency": 0}))

    for interface_data in interface_summary_data:
        count = interface_data["count"]
        interface = interface_data["interface"]
        jobnum = interface_data["jobnum"]
        account = interface_data["account"]
        ip = interface_data["ip"]
        interface_detail_dict_key = "{}{}".format(ip, account, jobnum)

        # 更新统计数据
        grouped_data[interface]["reqs"] += count
        # 构建下钻详情
        interface_detail_dict[interface][interface_detail_dict_key]["interface_addr"] = interface
        interface_detail_dict[interface][interface_detail_dict_key]["req_ip"] = ip
        interface_detail_dict[interface][interface_detail_dict_key]["req_account"] = account
        interface_detail_dict[interface][interface_detail_dict_key]["req_jobnum"] = jobnum
        interface_detail_dict[interface][interface_detail_dict_key]["req_frequency"] += count

    # 统计总请求次数
    reqs_total = sum(data["reqs"] for data in grouped_data.values())

    # 请求为0抛出
    if reqs_total == 0:
        return result

    # 构建summary部分
    interface_data_list = [
        {
            "interface_addr": interface,
            "req_frequency": data["reqs"],
            # 本接口的 请求次数/所有接口 请求次数的合计
            "frequency_rate": round(data["reqs"] / reqs_total, 4),
            # 本接口的 请求次数/ 20  查询top20接口
            "frequency_avg": safe_divide(data["reqs"],20),
        }
        for interface, data in grouped_data.items()
    ]
    result["summary"]["interface"] = sorted(interface_data_list, key=lambda x: x["req_frequency"], reverse=True)[:500]

    # 构建detail部分
    result["detail"]["interface"] = {
        company: sorted(data.values(), key=lambda x: x["req_frequency"], reverse=True)
        for company, data in interface_detail_dict.items()
    }

    return result


def menu_summary_data_format(menu_summary_data):
    """
        菜单维度数据转换方法
    """
    result = {"summary": {"menu": []}, "detail": {"menu": {}}}
    grouped_data = defaultdict(lambda: {"reqs": 0, "menu": set()})
    menu_detail_dict = defaultdict(lambda: defaultdict(lambda: {"req_frequency": 0}))
    menu_total = set()

    for menu_data in menu_summary_data:
        count = menu_data["count"]
        menu = menu_data["menu"]
        jobnum = menu_data["jobnum"]
        account = menu_data["account"]
        ip = menu_data["ip"]
        menu_detail_dict_key = "{}{}{}".format(ip, account, jobnum)

        # 更新统计数据和独立菜单数
        grouped_data[menu]["reqs"] += count
        grouped_data[menu]["menu"].add(menu)
        menu_total.add(menu)
        # 构建下钻详情
        menu_detail_dict[menu][menu_detail_dict_key]["menu_name"] = menu
        menu_detail_dict[menu][menu_detail_dict_key]["req_ip"] = ip
        menu_detail_dict[menu][menu_detail_dict_key]["req_account"] = account
        menu_detail_dict[menu][menu_detail_dict_key]["req_jobnum"] = jobnum
        menu_detail_dict[menu][menu_detail_dict_key]["req_frequency"] += count

    # 统计总请求次数
    reqs_total = sum(data["reqs"] for data in grouped_data.values())

    # 请求为0抛出
    if reqs_total == 0 or menu_total == 0:
        return result

    # 构建summary部分
    menu_data_list = [
        {
            "menu_name": menu,
            "req_frequency": data["reqs"],
            # 本菜单的 请求次数 /所有菜单 请求次数的合计
            "frequency_rate": round(data["reqs"] / reqs_total, 4),
            # 本菜单的 请求次数 /所有菜单 个数的合计
            "frequency_avg": safe_divide(data["reqs"],len(menu_total)),
        }
        for menu, data in grouped_data.items()
    ]
    result["summary"]["menu"] = sorted(menu_data_list, key=lambda x: x["req_frequency"], reverse=True)

    # 构建detail部分
    result["detail"]["menu"] = {company: sorted(data.values(), key=lambda x: x["req_frequency"], reverse=True)[:500]
                                for company, data in menu_detail_dict.items()}

    return result

def adjust_times(start_time, end_time):
    start_time = datetime.strptime(start_time, "%Y-%m-%d")
    end_time = datetime.strptime(end_time, "%Y-%m-%d")

    delta_days = (end_time - start_time).days
    if delta_days == 0:
        pre_date = start_time-timedelta(1)
        pre_date = start_time-timedelta(1)
        return pre_date.strftime("%Y-%m-%d"),pre_date.strftime("%Y-%m-%d")
    if delta_days > 0:
        pre_start_date = start_time-timedelta(delta_days+1)
        pre_end_date = end_time-timedelta(delta_days+1)
        return pre_start_date.strftime("%Y-%m-%d"),pre_end_date.strftime("%Y-%m-%d")
    return start_time, end_time