hbyd_ueba/utils/dashboard_data.py

#!/usr/bin/python
#encoding=utf-8
# author: tangwy

import json
import os,re
import codecs
import traceback
from isoc.utils.esUtil import EsUtil
from dashboard_data_conversion import ip_summary_data_format, account_summary_data_format, \
    interface_summary_data_format, menu_summary_data_format
from ext_logging import logger
## IP维度
def es_get_ip_group_data(index,startTime,endTime):
    page_size = 9000  #可以根据实际情况调整
    query_body={
  "query": {
    "bool": {
      "filter": [
        { "term": { "data_type": "ip" } },
        {"range":{
            "date_time": {
              "gte": startTime,
              "lte": endTime
            }
        }}
      ]
    }
  },
  "aggs": {
    "composite_buckets": {
      "composite": {
        "size" : page_size,
        "sources": [
          { "req_ip": { "terms": { "field": "req_ip" } } },
          { "req_jobnum": { "terms": { "field": "req_jobnum" } } }
        ]
      },
      "aggregations": {
        "total_count": {
          "sum": {
            "field": "req_frequency"
          }
        }
      }
    }
  }
}
    after_key = None
    es_util_instance = EsUtil()
    datas = []
    while True:
        if after_key:
            query_body["aggs"]["composite_buckets"]["composite"]["after"] = after_key
        response = es_util_instance.search(index,query_body)
        buckets = response.get("aggregations", {}).get("composite_buckets", {}).get("buckets", [])
        for bucket in buckets:
            data= {
                    "ip":bucket['key']['req_ip'],
                    "jobnum":bucket['key']['req_jobnum'],
                    "count":bucket['total_count']['value']
                }
            datas.append(data)
            after_key = bucket["key"]
        if not response["aggregations"]["composite_buckets"].get("after_key"):
            break
        after_key = response["aggregations"]["composite_buckets"]["after_key"]
    return datas


## 账号维度
def es_get_account_group_data(index,startTime,endTime):
    page_size = 9000  #可以根据实际情况调整
    query_body={
  "size": 0,
  "query": {
    "bool": {
      "filter": [
        { "term": { "data_type": "account" } },
        {"range":{
            "date_time": {
              "gte": startTime,
              "lte": endTime
            }
        }}
      ]
    }
  },
  "aggs": {
    "composite_buckets": {
      "composite": {
        "size" : page_size,
        "sources": [
          { "req_account": { "terms": { "field": "req_account" } } },
          { "req_jobnum": { "terms": { "field": "req_jobnum" } } }
        ]
      },
      "aggregations": {
        "total_count": {
          "sum": {
            "field": "req_frequency"
          }
        }
      }
    }
  }
}
    after_key = None
    es_util_instance = EsUtil()
    datas = []
    while True:
        if after_key:
            query_body["aggs"]["composite_buckets"]["composite"]["after"] = after_key
        response = es_util_instance.search(index,query_body)
        buckets = response.get("aggregations", {}).get("composite_buckets", {}).get("buckets", [])
        for bucket in buckets:
            data= {
                    "account":bucket['key']['req_account'],
                    "jobnum":bucket['key']['req_jobnum'],
                    "count":bucket['total_count']['value']
                }
            datas.append(data)
            after_key = bucket["key"]
        if not response["aggregations"]["composite_buckets"].get("after_key"):
            break
        after_key = response["aggregations"]["composite_buckets"]["after_key"]
    return datas


## 菜单维度
def es_get_menu_group_data(index,startTime,endTime):
    page_size = 9000  #可以根据实际情况调整
    query_body={
  "size": 0,
  "query": {
    "bool": {
      "filter": [
        { "term": { "data_type": "menu" } },
        {"range":{
            "date_time": {
              "gte": startTime,
              "lte": endTime
            }
        }}
      ]
    }
  },
  "aggs": {
    "composite_buckets": {
      "composite": {
          "size" : page_size,
        "sources": [
          { "menu_name": { "terms": { "field": "menu_name" } } },
          { "req_account": { "terms": { "field": "req_account" } } },
          { "req_ip": { "terms": { "field": "req_ip" } } },
          { "req_jobnum": { "terms": { "field": "req_jobnum" } } }
        ]
      },
      "aggregations": {
        "total_count": {
          "sum": {
            "field": "req_frequency"
          }
        }
      }
    }
  }
}
    after_key = None
    es_util_instance = EsUtil()
    datas = []
    while True:
        if after_key:
            query_body["aggs"]["composite_buckets"]["composite"]["after"] = after_key
        response = es_util_instance.search(index,query_body)
        buckets = response.get("aggregations", {}).get("composite_buckets", {}).get("buckets", [])
        for bucket in buckets:
            data= {
                    "menu":bucket['key']['menu_name'],
                    "ip":bucket['key']['req_ip'],
                    "account":bucket['key']['req_account'],
                    "jobnum":bucket['key']['req_jobnum'],
                    "count":bucket['total_count']['value']
                }
            datas.append(data)
            after_key = bucket["key"]
        if not response["aggregations"]["composite_buckets"].get("after_key"):
            break
        after_key = response["aggregations"]["composite_buckets"]["after_key"]
    return datas


## 接口维度
def es_get_interface_group_data(index,startTime,endTime):
    page_size = 9999  #可以根据实际情况调整
    query_body={
  "query": {
    "bool": {
      "filter": [
        { "term": { "data_type": "interface" } },
        {"range":{
            "date_time": {
              "gte": startTime,
              "lte": endTime
            }
        }}
      ]
    }
  },
  "aggs": {
    "group_by_menu": {
      "composite": {
        "size" : page_size,
        "sources": [
          { "interface_addr": { "terms": { "field": "interface_addr" } } },
          { "req_account": { "terms": { "field": "req_account" } } },
          { "req_ip": { "terms": { "field": "req_ip" } } },
          { "req_jobnum": { "terms": { "field": "req_jobnum" } } }
        ]
      },
      "aggregations": {
        "total_count": {
          "sum": {
            "field": "req_frequency"
          }
        }
      }
    }
  }
}
    after_key = None
    es_util_instance = EsUtil()
    datas = []
    while True:
        if after_key:
            query_body["aggs"]["composite_buckets"]["composite"]["after"] = after_key
        response = es_util_instance.search(index,query_body)
        buckets = response.get("aggregations", {}).get("composite_buckets", {}).get("buckets", [])
        for bucket in buckets:
            data= {
                    "interface":bucket['key']['interface_addr'],
                    "ip":bucket['key']['req_ip'],
                    "account":bucket['key']['req_account'],
                    "jobnum":bucket['key']['req_jobnum'],
                    "count":bucket['total_count']['value']
                }
            datas.append(data)
            after_key = bucket["key"]
        if not response["aggregations"]["composite_buckets"].get("after_key"):
            break
        after_key = response["aggregations"]["composite_buckets"]["after_key"]
    return datas

def entry(data_type,start,end):
    base_index = 'c_ueba_001'
    # es_util_instance = EsUtil()
    # res=es_util_instance.get_available_index_name(start,end,base_index)
    # if len(res)==0:
    #     return
    # index =",".join(res)

    index=base_index

    try:
        data = {}
        if data_type == "1":
            ip_summary_data = es_get_ip_group_data(index, start, end)
            data = ip_summary_data_format(ip_summary_data)
        if data_type == "2":
            account_summary_data = es_get_account_group_data(index, start, end)
            data = account_summary_data_format(account_summary_data)
        if data_type == "3":
            interface_summary_data = es_get_interface_group_data(index, start, end)
            data = interface_summary_data_format(interface_summary_data)
        if data_type == "4":
            menu_summary_data = es_get_menu_group_data(index, start, end)
            data = menu_summary_data_format(menu_summary_data)
        return data
    except Exception, e:
        logger.error(traceback.format_exc())
        raise e