hbyd_ueba/utils/file_merge.py

#!/usr/bin/python
#encoding=utf-8
# author: tangwy
import re,os,json,time
import codecs
from db2json import DBUtils
from datetime import datetime, timedelta
from base_dataclean_pg import TRACE_KEY
from ext_logging import logger_cron,get_clean_file_path,merge_large_file_path,logger_trace
from file_helper import read_large_json_file,write_large_file,get_file_content,delete_frile,is_file_larger_than_500mb,merge_data_new
from collections import defaultdict

date_pattern = re.compile(r'^\d{8}-\d{4}\.json$')

def get_all_files(path):
    # 列出所有包含匹配模式的文件名
    files = []
    for filename in os.listdir(path):
        if date_pattern.search(filename):
            files.append(filename)
    return files

#对待合并文件进行分组
def get_file_merge_array(filenames):
    # 创建一个defaultdict来存储分组的文件
    file_dict = defaultdict(list)
    for filename in filenames:
        date = filename[:8]
        file_dict[date].append(filename)

    file_dict = dict(file_dict)
    return file_dict

# 将 2024-08-08.json 移动大文件目录下 等待入库前的合并【只移动前一天的文件】
def mv_root_file_to_current(base_path,new_path):
    new_file_name = (datetime.now()-timedelta(days=1)).strftime("%Y%m%d-2359.json")
    old_file_name = (datetime.now()-timedelta(days=1)).strftime("%Y-%m-%d.json")

    old_full_path=os.path.join(base_path,old_file_name)
    new_full_path = os.path.join(new_path,new_file_name)
    if os.path.exists(old_full_path):
        if os.path.exists(new_full_path):
            logger_cron.error("MERGE_LARG_EFILE: 文件 "+new_full_path+"已经存在，任然移动会被覆盖")
        os.rename(old_full_path,new_full_path)

#合并大于>500M的文件
def merge_large_files(file_dict,base_path):
    # 遍历字典中的每一个键值对
    for date_str, files in file_dict.items():
        #20240721
        root_file_path = "{}-{}-{}.json".format(date_str[:4], date_str[4:6], date_str[6:])
        full_root_file_path = os.path.join(base_path,root_file_path)
        if len(files)>0:
            file_objs=[]
            file_full_path = []
            # 合并的数据存储变量
            merge_tmp_data = {}
            for filename in files:
                #20240721-0170.json
                full_path = os.path.join(base_path,filename)
                file_full_path.append(full_path)
                logger_cron.info("INSERT: 准备读取文件做合并"+full_path)
                tmp_data =read_large_json_file(full_path)
                logger_cron.info("INSERT: 数据量"+str(len(tmp_data)))

                file_objs.append(tmp_data)
                file_objs.append(merge_tmp_data)
                merge_tmp_data = merge_data_new(file_objs)
                logger_cron.info("INSERT: 合并完成"+full_path)
                #移除已经合并的数据
                del file_objs[:]
        
            #判断files目录是否存在结果文件 (2024-08-08.json)
            if os.path.exists(full_root_file_path):
                logger_cron.info("INSERT: 准备读取文件做合并"+full_root_file_path)
                root_data = read_large_json_file(full_root_file_path)
                logger_cron.info("INSERT: 数据量"+str(len(root_data)))
                file_objs.append(root_data)
                file_objs.append(merge_tmp_data)
                merge_tmp_data = merge_data_new(file_objs)
                logger_cron.info("INSERT: 合并完成"+full_root_file_path)
            logger_cron.info("INSERT: 准备写入合并的文件")

            ######################问题排查
            key=get_file_content()
            if key in merge_tmp_data:
                logger_trace.info("largefilemerge:"+full_root_file_path+":"+key+":"+str(merge_tmp_data[key]))

            write_large_file(full_root_file_path,json.dumps(merge_tmp_data))
            logger_cron.info("INSERT: 写入合并文件完成")
            #准备删除合并文件
            for del_file in file_full_path:
                logger_cron.info("INSERT: 准备删除 "+del_file)
                delete_frile(del_file)
                #os.rename(del_file,del_file+".cmp")
                logger_cron.info("INSERT: 完成删除 "+del_file)

#2024-08-23.json 换成20240823-1410 时分为当前时间的时分
def get_new_file_name(old_file_path):
    file_name_with_ext = os.path.basename(old_file_path)
    file_name, file_extension = os.path.splitext(file_name_with_ext)
    sf=time.strftime("%H%M.json", time.localtime())
    new_name = file_name.replace("-","")
    res_name = new_name+"-"+sf

    return res_name

#合并所有文件
def merge_all_files(file_dict,base_path):
    # 遍历字典中的每一个键值对
    for date_str, files in file_dict.items():
        #20240721
        root_file_path = "{}-{}-{}.json".format(date_str[:4], date_str[4:6], date_str[6:])
        full_root_file_path = os.path.join(base_path,root_file_path)
        if len(files)>0:
            file_objs=[]
            file_full_path = []
            # 合并的数据存储变量
            merge_tmp_data = {}

            file_full_path = []
            for filename in files:
                #20240721-0170.json
                full_path = os.path.join(base_path,filename)
                file_full_path.append(full_path)
                logger_cron.info("FILE_MERGE: 准备读取文件做合并"+full_path)

                tmp_data =read_large_json_file(full_path)
                file_objs.append(tmp_data)
                file_objs.append(merge_tmp_data)
                merge_tmp_data = merge_data_new(file_objs)
                logger_cron.info("FILE_MERGE: 文件合并完成"+full_path)
                #移除已经合并的数据
                del file_objs[:]

            if os.path.exists(full_root_file_path):
                flag=is_file_larger_than_500mb(full_root_file_path)
                if flag:
                    logger_cron.info("FILE_MERGE: 文件超过500M需要移动到merge_file目录"+full_root_file_path)
                    large_file_root_path = merge_large_file_path()

                    #新的文件名
                    new_file_name = get_new_file_name(full_root_file_path)
                    logger_cron.info("FILE_MERGE: 新文件名 "+new_file_name)
                    large_file_path = os.path.join(large_file_root_path,new_file_name)
                    logger_cron.info("FILE_MERGE: oldpath "+full_root_file_path)
                    if os.path.exists(large_file_path):
                        logger_cron.error("FILE_MERGE: 文件 "+large_file_path+"已经存在，任然移动会被覆盖")

                    os.rename(full_root_file_path, large_file_path)
                    logger_cron.info("FILE_MERGE: newpath "+large_file_path+"移动成功")
                else:
                    logger_cron.info("FILE_MERGE: 文件小于500M需要参与合并"+full_root_file_path)
                    root_data = read_large_json_file(full_root_file_path)
                    file_objs.append(root_data)
                    file_objs.append(merge_tmp_data)
                    merge_tmp_data = merge_data_new(file_objs)

            ###################问题排查
            key=get_file_content()
            if key in merge_tmp_data:
                logger_trace.info("filemerge:"+full_root_file_path+":"+key+":"+str(merge_tmp_data[key]))

            logger_cron.info("FILE_MERGE: 所有文件合并完成")
            write_large_file(full_root_file_path,json.dumps(merge_tmp_data))
            logger_cron.info("FILE_MERGE: 写入合并文件完成")
            #准备删除合并文件
            for del_file in file_full_path:
                logger_cron.info("FILE_MERGE: 准备删除 "+del_file)
                delete_frile(del_file)
                #os.rename(del_file,del_file+".cmp")
                logger_cron.info("FILE_MERGE: 完成删除 "+del_file)

#每半小时执行的合并
def entry():
    #清洗目录
    base_path = get_clean_file_path()
    #匹配待清洗的文件
    files = get_all_files(base_path)
    #对待清洗的文件进行分组
    file_dict =get_file_merge_array(files)
    #合并所有文件
    logger_cron.info("FILE_MERGE: 准备执行文件合并")
    merge_all_files(file_dict,base_path)

#入库前执行的大文件合并
def merge_large_entry():
    base_path = get_clean_file_path()
    #清洗目录
    new_base_path = merge_large_file_path()
    #将 2024-08-08.json 移动到merge_file目录下 等待入库前的合并
    mv_root_file_to_current(base_path,new_base_path)
    #匹配待清洗的文件
    files = get_all_files(new_base_path)
    logger_cron.info("INSERT: 待合并的文件"+json.dumps(files))
    #对待清洗的文件进行分组
    file_dict =get_file_merge_array(files)
    #合并所有文件
    logger_cron.info("INSERT: 准备执行文件合并")
    merge_large_files(file_dict,new_base_path)