You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
90 lines
2.6 KiB
90 lines
2.6 KiB
4 months ago
|
#!/usr/bin/python
|
||
|
|
#encoding=utf-8
|
||
|
|
# author: tangwy
|
||
|
|
|
||
|
|
import json
|
||
|
|
import os,re
|
||
|
|
import codecs
|
||
|
|
import csv
|
||
|
|
import ConfigParser
|
||
|
|
from isoc.utils.esUtil import EsUtil
|
||
|
|
|
||
|
|
|
||
|
|
print json.dumps(es_host_list)
|
||
|
|
# conf_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'conf')
|
||
|
|
# ini_path = os.path.join(conf_path, 'conf.ini')
|
||
|
|
# config = ConfigParser.ConfigParser()
|
||
|
|
# config.read(ini_path)
|
||
|
|
|
||
|
|
# ES_HOST = config.get('COMMON', 'es_host')
|
||
|
|
# ES_PER_COUNT = config.get('COMMON', 'es_per_count')
|
||
|
|
# ES_INDEX_NAME = config.get('COMMON', 'es_index_name')
|
||
|
|
# CSV_FILE_PATH = config.get('COMMON', 'csv_file_path')
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def createIndex():
|
||
|
|
es = Elasticsearch(es_host_list)
|
||
|
|
es.create(index="urba_analyse_2024_06", ignore=400)
|
||
|
|
|
||
|
|
map={
|
||
|
|
"ip1": "text",
|
||
|
|
"ip2": "text",
|
||
|
|
"ip3": "text",
|
||
|
|
"ip4": "text",
|
||
|
|
}
|
||
|
|
es_instance = EsUtil()
|
||
|
|
res = es_instance.create_index_simple("urba_analyse_2024_06")
|
||
|
|
return res
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# def generate_ip_range(start_ip, end_ip):
|
||
|
|
# start_parts = list(map(int, start_ip.split('.')))
|
||
|
|
# end_parts = list(map(int, end_ip.split('.')))
|
||
|
|
# ip_range = []
|
||
|
|
|
||
|
|
# while start_parts < end_parts:
|
||
|
|
# ip_range.append('.'.join(map(str, start_parts)))
|
||
|
|
# start_parts[3] += 1
|
||
|
|
# for i in range(3, 0, -1):
|
||
|
|
# if start_parts[i] == 256:
|
||
|
|
# start_parts[i] = 0
|
||
|
|
# start_parts[i-1] += 1
|
||
|
|
|
||
|
|
# ip_range.append('.'.join(map(str, start_parts))) # 添加结束IP地址
|
||
|
|
# return ip_range
|
||
|
|
|
||
|
|
# # scroll查询数据
|
||
|
|
# def get_ip_summary_data(start_time,end_time,query_body):
|
||
|
|
# es = Elasticsearch(ES_HOST)
|
||
|
|
# msg = es.search(index=ES_INDEX_NAME,scroll="3m",size=ES_PER_COUNT,_source_includes= ["cookies","url","sip","dip"], query=query_body)
|
||
|
|
|
||
|
|
# result = msg['hits']['hits']
|
||
|
|
# total = msg['hits']['total']
|
||
|
|
# scroll_id = msg['_scroll_id']
|
||
|
|
|
||
|
|
# for i in range(0,int(total["value"]/ES_PER_COUNT)+1):
|
||
|
|
# query_scroll = es.scroll(scroll_id=scroll_id, scroll='3m')["hits"]["hits"]
|
||
|
|
# result += query_scroll
|
||
|
|
# return result
|
||
|
|
|
||
|
|
# # 读取csv文件 获取ip归属地
|
||
|
|
# def get_ip_area_relation(csv_file_path):
|
||
|
|
# iprange_map = {}
|
||
|
|
# with codecs.open(csv_file_path, mode='r',encoding='utf-8') as file:
|
||
|
|
# csv_reader = csv.reader(file)
|
||
|
|
# for row in csv_reader:
|
||
|
|
# headers = next(csv_reader)
|
||
|
|
# ip_start = headers[0]
|
||
|
|
# ip_end = headers[1]
|
||
|
|
# ip_range = generate_ip_range(ip_start, ip_end)
|
||
|
|
# ip_area = headers[5]
|
||
|
|
# print (ip_area)
|
||
|
|
# for ip in ip_range:
|
||
|
|
# iprange_map[ip] = ip_area
|
||
|
|
# return iprange_map
|
||
|
|
|
||
|
|
# get_ip_area_relation("/tmp/data/ip_area_relation.csv")
|