From 0dfc4dcac40297f283bae075d3cc415655ef1134 Mon Sep 17 00:00:00 2001 From: TANGWY Date: Fri, 7 Jun 2024 14:32:48 +0800 Subject: [PATCH] 'init' --- .idea/.gitignore | 8 + .idea/demo.iml | 27 ++ .idea/deployment.xml | 14 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/modules.xml | 8 + __init__.py | 0 bin/__init__.py | 1 + bin/start.py | 31 +++ bin/stop.py | 21 ++ bin/uninstall.py | 40 +++ business/es_query.py | 65 +++++ conf/__init__.py | 1 + conf/conf.ini | 5 + install.py | 145 ++++++++++ lib/__init__.py | 0 lib/result.py | 82 ++++++ mock/mock_data.json | 194 ++++++++++++++ package.json | 17 ++ right_config.json | 23 ++ target/compare_dependency.py | 110 ++++++++ target/encrypt_conf | 5 + target/encrypt_tool/__init__.py | 0 target/encrypt_tool/encrypt_app.py | 23 ++ target/encrypt_tool/melon/__init__.py | 1 + target/encrypt_tool/melon/cipher.txt | Bin 0 -> 1259 bytes target/encrypt_tool/melon/melon.py | 245 +++++++++++++++++ target/encrypt_tool/melon/rsa.priv | 51 ++++ target/encrypt_tool/melon/rsa.pub | 51 ++++ target/encrypt_tool/pyprotect.py | 149 +++++++++++ target/package.py | 249 ++++++++++++++++++ urls.py | 14 + utils/__init__.py | 0 utils/ext_logging.py | 38 +++ views/__init__.py | 7 + views/dashboard_views.py | 53 ++++ 36 files changed, 1691 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/demo.iml create mode 100644 .idea/deployment.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 __init__.py create mode 100644 bin/__init__.py create mode 100644 bin/start.py create mode 100644 bin/stop.py create mode 100644 bin/uninstall.py create mode 100644 business/es_query.py create mode 100644 conf/__init__.py create mode 100644 conf/conf.ini create mode 100644 install.py create mode 100644 lib/__init__.py create mode 100644 lib/result.py create mode 100644 mock/mock_data.json create mode 100644 package.json create mode 100644 right_config.json create mode 100644 target/compare_dependency.py create mode 100644 target/encrypt_conf create mode 100644 target/encrypt_tool/__init__.py create mode 100644 target/encrypt_tool/encrypt_app.py create mode 100644 target/encrypt_tool/melon/__init__.py create mode 100644 target/encrypt_tool/melon/cipher.txt create mode 100644 target/encrypt_tool/melon/melon.py create mode 100644 target/encrypt_tool/melon/rsa.priv create mode 100644 target/encrypt_tool/melon/rsa.pub create mode 100644 target/encrypt_tool/pyprotect.py create mode 100644 target/package.py create mode 100644 urls.py create mode 100644 utils/__init__.py create mode 100644 utils/ext_logging.py create mode 100644 views/__init__.py create mode 100644 views/dashboard_views.py diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/demo.iml b/.idea/demo.iml new file mode 100644 index 0000000..5d64b93 --- /dev/null +++ b/.idea/demo.iml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/deployment.xml b/.idea/deployment.xml new file mode 100644 index 0000000..214040a --- /dev/null +++ b/.idea/deployment.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..96b7298 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..c95e899 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bin/__init__.py b/bin/__init__.py new file mode 100644 index 0000000..677570a --- /dev/null +++ b/bin/__init__.py @@ -0,0 +1 @@ +# -*-coding:utf-8 -*- diff --git a/bin/start.py b/bin/start.py new file mode 100644 index 0000000..3ac429e --- /dev/null +++ b/bin/start.py @@ -0,0 +1,31 @@ +# -*-coding:utf-8 -*- +# 组件启动脚本，调用组件管理提供的批量启用接口启用 jobs/jobmeta.json里描述的组件任务，以及组件自定义的动作 +import json +import os + +from appManager.module.global_api import ComponentHelper + +DIR_PATH = os.path.split(os.path.realpath(__file__))[0] +BASE_PATH = os.path.split(DIR_PATH)[0] + + +def start(): + with open(os.path.join(BASE_PATH, "package.json"), "r+") as f: + pkgConfig = json.load(f) + app_name = pkgConfig["name"] + componentHelper = ComponentHelper(app_name) + # 调用组件管理接口启动任务 + componentHelper.start_task() + # 创建kafka topic + # createTopic() + # 创建数据源 + # createDataSource() + return True + + +if __name__ == '__main__': + # exit 为 0，组件启动成功，为1，组件启动失败 + if start(): + exit(0) + else: + exit(1) diff --git a/bin/stop.py b/bin/stop.py new file mode 100644 index 0000000..734b37d --- /dev/null +++ b/bin/stop.py @@ -0,0 +1,21 @@ +# -*-coding:utf-8 -*- +import json +import os +from appManager.module.global_api import ComponentHelper + +DIR_PATH = os.path.split(os.path.realpath(__file__))[0] +BASE_PATH = os.path.split(DIR_PATH)[0] + + +# 组件停止脚本 +def stop(): + with open(os.path.join(BASE_PATH, "package.json"), "r+") as f: + pkgConfig = json.load(f) + app_name = pkgConfig["name"] + componentHelper = ComponentHelper(app_name) + # 调用组件管理接口停用任务 + componentHelper.stop_task() + + +if __name__ == '__main__': + stop() diff --git a/bin/uninstall.py b/bin/uninstall.py new file mode 100644 index 0000000..2265823 --- /dev/null +++ b/bin/uninstall.py @@ -0,0 +1,40 @@ +# -*-coding:utf-8 -*- +# 组件卸载时执行的脚本 +from dataInterface.functions import CFunction +from dataInterface.db.params import CPgSqlParam + + +def uninstall(): + # 组件卸载动作 + # 1. 删除任务 + remove_task() + # 2.删除数据库schema + uninstall_db_schema() + # 3. 删除kafka topic + delete_kafka_topic() + # 4. 其他操作等 + + +def remove_task(): + """ + 组件批量注销任务 + """ + from appManager.module.global_api import ComponentHelper + ch = ComponentHelper('demo') + ch_ret = ch.remove_task() + + +def uninstall_db_schema(): + ''' 删除数据库schema ''' + try: + sql = CPgSqlParam("drop schema if exists demo cascade") + CFunction.execute(sql) + except Exception as e: + raise e + # logger.error("delete data schema fail! reason: %s" % e) + +def delete_kafka_topic(): + from appsUtils.kafkaUtil import KAFKA_API + api = KAFKA_API() + api. delete_topic('topic_name') + diff --git a/business/es_query.py b/business/es_query.py new file mode 100644 index 0000000..e0fa749 --- /dev/null +++ b/business/es_query.py @@ -0,0 +1,65 @@ +#!/usr/bin/python +#encoding=utf-8 +# author: tangwy + +import json +import os,re +import codecs +import csv +import ConfigParser +from ipaddr import IPRange +from elasticsearch import Elasticsearch + +conf_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'conf') +ini_path = os.path.join(conf_path, 'conf.ini') +config = ConfigParser.ConfigParser() +config.read(ini_path) + +ES_HOST = config.get('COMMON', 'es_host') +ES_PER_COUNT = config.get('COMMON', 'es_per_count') +ES_INDEX_NAME = config.get('COMMON', 'es_index_name') + +CSV_FILE_PATH = config.get('COMMON', 'csv_file_path') + +# scroll查询数据 +def get_es_data(start_time,end_time): + es = Elasticsearch(ES_HOST) + msg = es.search(index=ES_INDEX_NAME,scroll="3m",size=ES_PER_COUNT,_source_includes= ["cookies","url","sip","dip"], query={ + "bool": { + "filter": { + "range": { + "timestamp": { + "gte": start_time, + "lte": end_time + } + } + } + } + }) + + result = msg['hits']['hits'] + total = msg['hits']['total'] + scroll_id = msg['_scroll_id'] + + for i in range(0,int(total["value"]/ES_PER_COUNT)+1): + query_scroll = es.scroll(scroll_id=scroll_id, scroll='3m')["hits"]["hits"] + result += query_scroll + return result + +# 读取csv文件 获取ip归属地 +def get_ip_area_relation(csv_file_path): + iprange_map = {} + with codecs.open(csv_file_path, mode='r',encoding='utf-8') as file: + csv_reader = csv.reader(file) + for row in csv_reader: + headers = next(csv_reader) + ip_start = headers[0] + ip_end = headers[1] + ip_range = IPRange(ip_start, ip_end) + ip_area = headers[5] + print (ip_area) + for ip in ip_range: + iprange_map[ip] = ip_area + return iprange_map + +get_ip_area_relation("/tmp/data/ip_area_relation.csv") \ No newline at end of file diff --git a/conf/__init__.py b/conf/__init__.py new file mode 100644 index 0000000..677570a --- /dev/null +++ b/conf/__init__.py @@ -0,0 +1 @@ +# -*-coding:utf-8 -*- diff --git a/conf/conf.ini b/conf/conf.ini new file mode 100644 index 0000000..4cfd7ea --- /dev/null +++ b/conf/conf.ini @@ -0,0 +1,5 @@ +[COMMON] +es_index_name = 'bsa_traffic*' +es_host = 'http://10.65.74.3:19399' +es_pre_count = 100 +csv_file_path = "/tmp/data/ip_area_relation.csv" \ No newline at end of file diff --git a/install.py b/install.py new file mode 100644 index 0000000..2b90485 --- /dev/null +++ b/install.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*-coding:utf-8 -*- + +import sys +import os +import json +import shutil +import traceback +import logging +from appManager.module.global_api import ComponentHelper +from appsUtils.confutil import ConfUtil + + +BASE_PATH = os.path.split(os.path.realpath(__file__))[0] +DSTPATH = sys.argv[1] +ISUPDATE = sys.argv[2] +CUR_PATH = os.path.normpath(os.path.dirname(os.path.abspath(__file__))) +APP_NAME = 'UebaMetricsAnalysis' + +def get_logger(logger_name=APP_NAME, logger_level=logging.INFO): + """日志""" + import logging.handlers + logger = logging.getLogger(logger_name) + formatter = logging.Formatter( + '%(asctime)s %(levelname)s %(filename)s %(funcName)s:%(lineno)d %(message)s') + logger.setLevel(logger_level) + logger_file = os.path.normpath(os.path.join('/home/master', 'logs')) + if not os.path.exists(logger_file): + os.mkdir(logger_file) + if not logger.handlers: + file_handler = logging.handlers.TimedRotatingFileHandler( + os.path.normpath(logger_file + os.sep + logger_name + '.log'), 'midnight') + file_handler.suffix = "%Y-%m-%d" + file_handler.setLevel(logger_level) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + return logger + + +logger = get_logger("UebaMetricsAnalysis") + + +def installDBSchema(exec_sql): + try: + confutil = ConfUtil() + curPgConf = confutil.getPostgresqlConf() + sqlPath = os.path.join(CUR_PATH, APP_NAME, 'sql') + + for sqlfile in exec_sql: + if sqlfile == "pg_update.sql": + continue + sqlPath = os.path.join(CUR_PATH, APP_NAME, 'sql', sqlfile) + cmd = 'psql -U %s %s<"%s"' % (curPgConf["username"], curPgConf["database"], sqlPath,) + logger.info(cmd) + r = os.popen(cmd) + logger.info(sqlfile + ' ' + str(r.readlines())) + logger.info('create schema successfully.') + except Exception as e: + logger.info(str(e)) + + +def add_task(): + # 组件批量注册任务 + ch = ComponentHelper(APP_NAME) + logger.info("开始注册定时任务") + ch_ret = ch.add_task() + if ch_ret['status'] != 200: + logger.info('add task failed, reason is %s' % ch_ret['msg']) + else: + logger.info('add task successfully.') + logger.info("定时任务注册成功") + + +class Install(): + def __init__(self): + self.app_name = self.__get_app_name() + + def install(self): + try: + installDBSchema(["pg_struct.sql"]) + add_task() + logger.info('>>>安装结束！！！') + except Exception as e: + logger.error("install failure! reason: %s" % traceback.format_exc()) + self.install_rollback() + # 此处写自定义的状态码，如100 + code = 100 + sys.exit(code) + + def upgrade(self): + try: + # 第一步，拷贝目录 + shutil.rmtree(DSTPATH) + self.copy_files() + # 第二步，写自己升级的逻辑 + except Exception as e: + logger.info(traceback.format_exc()) + self.upgrade_rollback() + sys.exit(1) + + def copy_files(self): + src_path = os.path.join(CUR_PATH, self.app_name) + shutil.copytree(src_path, DSTPATH) + + def install_rollback(self): + """安装回滚操作""" + try: + logger.error("rollback app folder") + except Exception as e: + logger.error("roll back failed") + logger.error(traceback.format_exc()) + + def upgrade_rollback(self): + """升级回滚操作""" + try: + logger.error("rollback app folder") + except Exception as e: + logger.error("roll back failed") + logger.error(traceback.format_exc()) + + def __get_app_name(self): + with open(os.path.join(CUR_PATH, list(os.walk(CUR_PATH))[0][1][0], "package.json"), "r+") as f: + pkgConfig = json.load(f) + return pkgConfig["name"] + + +def main(): + try: + flag = sys.argv[2] + install_instance = Install() + if flag == '0': + logger.info("=" * 20 + "开始安装" + "=" * 20) + install_instance.install() + if flag == '1': + logger.info("=" * 20 + "开始升级" + "=" * 20) + install_instance.upgrade() + logger.info("main 执行完成") + sys.exit(0) + except Exception as e: + logger.info("install failure, reason: %s" % traceback.format_exc()) + raise e + + +if __name__ == '__main__': + main() diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/result.py b/lib/result.py new file mode 100644 index 0000000..3793dbb --- /dev/null +++ b/lib/result.py @@ -0,0 +1,82 @@ +# coding=utf-8 + +""" +根据nsfocus rest要求返回对应对象 +""" +from django.http import JsonResponse + + +class Result(object): + """ + 结果对象 用于返回给 JsonResponse + """ + SUCCESS_CODE = 0 # 成功的状态码 + FAILED_CODE = 1 # 失败的状态码 + SUCCESS_MESSAGE = 'success' # 返回单个数据的message + SUCCESS_MESSAGE_LIST = '请求成功' # 返回list的message + + @classmethod + def ok(cls, data): + """ + 返回单个对象 + :param data: 对象 dict + :return: + """ + return JsonResponse({ + 'code': cls.SUCCESS_CODE, + 'message': cls.SUCCESS_MESSAGE, + 'data': data + }) + + @classmethod + def list(cls, total, page, limit, data): + """ + 返回list + :param total: + :param page: + :param limit: + :param data: + :return: + """ + return JsonResponse({ + 'code': cls.SUCCESS_CODE, + 'message': cls.SUCCESS_MESSAGE_LIST, + 'data': { + 'total': total, + 'page': page, + 'limit': limit, + 'list': data + } + }) + + @classmethod + def failed(cls, message, detail=''): + """ + 请求失败 + :param message: 错误信息 + :param detail: 详细描述 + :return: + """ + return JsonResponse({ + 'code': cls.FAILED_CODE, + 'message': message, + 'data': { + 'detail': detail + } + }) + + @classmethod + def failed_list(cls, message, data): + """ + 部分请求失败 + :param message: 失败描述 + :param data: 失败内容的list + :return: + """ + return JsonResponse({ + 'code': cls.FAILED_CODE, + 'message': message, + 'data': { + 'list': data + } + }) diff --git a/mock/mock_data.json b/mock/mock_data.json new file mode 100644 index 0000000..f5312d2 --- /dev/null +++ b/mock/mock_data.json @@ -0,0 +1,194 @@ +{ + "summary": { + "ip": [ + { + "company": "湖北公司", + "req_frequency": 122, + "frequency_rate": 0.2, + "ip_count": 323, + "ip_reat": 0.3, + "ip_avg": 0.43, + "trend": 0.3 + }, + { + "company": "宜昌公司", + "req_frequency": 122, + "frequency_rate": 0.2, + "ip_count": 323, + "ip_reat": 0.3, + "ip_avg": 0.43, + "trend": 0.3 + } + ], + "account": [ + { + "company": "湖北公司", + "req_frequency": 122, + "frequency_rate": 0.2, + "account_count": 323, + "account_reat": 0.3, + "account_avg": 0.43, + "trend": 0.3 + }, + { + "company": "宜昌公司", + "req_frequency": 122, + "frequency_rate": 0.2, + "account_count": 323, + "account_reat": 0.3, + "account_avg": 0.43, + "trend": 0.3 + } + ], + "interface": [ + { + "interface_addr": "/getuser", + "req_frequency": 122, + "frequency_rate": 0.2, + "frequency_avg": 0.43, + "trend": 0.3 + }, + { + "interface_addr": "/getcpminfo", + "req_frequency": 122, + "frequency_rate": 0.2, + "frequency_avg": 0.43, + "trend": 0.3 + } + ], + "menu": [ + { + "menu_name": "接口地址", + "req_frequency": 122, + "frequency_rate": 0.2, + "frequency_avg": 0.43, + "trend": 0.3 + }, + { + "menu_name": "接口地址", + "req_frequency": 122, + "frequency_rate": 0.2, + "frequency_avg": 0.43, + "trend": 0.3 + } + ] + }, + "detail": { + "ip": { + "湖北公司": [ + { + "req_ip": "xxx.xx.xx.x", + "req_frequency": 22 + }, + { + "req_ip": "xx1x.xx.xx.x", + "req_frequency": 21 + } + ], + "宜昌公司": [ + { + "req_ip": "xxx.xx.xx.x", + "req_frequency": 22 + }, + { + "req_ip": "xx1x.xx.xx.x", + "req_frequency": 21 + } + ] + }, + "account": { + "湖北公司": [ + { + "req_account": "admin", + "req_frequency": 22, + "req_jobnum": 98799 + }, + { + "req_account": "admin", + "req_frequency": 22, + "req_jobnum": 98799 + } + ], + "宜昌公司": [ + { + "req_account": "admin", + "req_frequency": 22, + "req_jobnum": 98799 + }, + { + "req_account": "admin", + "req_frequency": 22, + "req_jobnum": 98799 + } + ] + }, + "interface": { + "接口1": [ + { + "interface_addr": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + }, + { + "interface_addr": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + } + ], + "接口2": [ + { + "interface_addr": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + }, + { + "interface_addr": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + } + ] + }, + "menu": { + "菜单1": [ + { + "menu_name": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + }, + { + "menu_name": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + } + ], + "菜单2": [ + { + "menu_name": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + }, + { + "menu_name": "接口地址", + "req_frequency": 122, + "req_ip": "xxx.xx.xx.x", + "req_account": 0.2, + "req_jobnum": 0.2 + } + ] + } + } +} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..d0db477 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "UebaMetricsAnalysis", + "version": "V3.0R01F00", + "menuurl": "/UebaMetricsAnalysis", + "menuregx": "^/UebaMetricsAnalysis", + "menuname": "指标晾晒统计", + "summary": "指标晾晒统计", + "platform_mode": "simple_mode", + "base_platform_version": "V2.0R01F00", + "component_type": "application-component", + "module_dependencies":{}, + "common_dependencies": { + "component": {}, + "storage_compute_denpencies": [] + }, + "is_licensed": 0 +} diff --git a/right_config.json b/right_config.json new file mode 100644 index 0000000..36a9101 --- /dev/null +++ b/right_config.json @@ -0,0 +1,23 @@ +{ + "name": "指标晾晒统计", + "key": "UebaMetricsAnalysis", + "prefix": "UebaMetricsAnalysis", + "allfix": [ + "^/UebaMetricsAnalysis" + ], + "fix": [ + "^/UebaMetricsAnalysis" + ], + "link": "/WebApi/UebaMetricsAnalysis/static/dist/#/", + "pinyin": "zhibiaoliangshaitongji", + "app_name": "UebaMetricsAnalysis", + "children": [ + + ], + "role_code": [ + 1 + ], + "role_menu_register":{ + "指标晾晒统计": "UebaMetricsAnalysis" + } +} diff --git a/target/compare_dependency.py b/target/compare_dependency.py new file mode 100644 index 0000000..ce0724e --- /dev/null +++ b/target/compare_dependency.py @@ -0,0 +1,110 @@ +#coding:utf-8 +import traceback +import os +import json +from zipfile import ZipFile +from xml.dom.minidom import parse +import xml.dom.minidom +import shutil +'''用于比对出APP的依赖与平台的commmonlib的脚本''' + +class CompareDependency(object): + #appname:app名称，如bsa_ata + #apppath:打包平台上此时app的绝对路径 + #jarpath:需比对的app jar文件的绝对路径 + #depath:平台de文件所在的绝对路径 + #name:文件名；启动job/进程时的依赖需要从中读取 + def __init__(self, appname, apppath, jarpath, depath, name): + self.bsaroot = "${BSA_HOME}" + self.platform_libpath = self.bsaroot + "/libs/Java/commonLibs" + self.jarpath = jarpath + self.depath = depath + self.name = name + self.pom_path = os.path.dirname(os.path.dirname(self.jarpath)) + "/pom.xml" + self.lib_path = os.path.dirname(self.jarpath) + "/lib" + self.apppath = apppath + self.appname = appname + + def parse_manifest(self, cpath): + '''解析MANIFEST.MF文件''' + ls = list() + + f = ZipFile(cpath) + try: + content = f.read("META-INF/MANIFEST.MF").split("\n") + content_ls = list() + start = False + for line in content: + if line.startswith("Class-Path:"): + content_ls.append(line.replace("Class-Path:", "").replace("\r", "").replace("\n", "")) + start = True + else: + if start: + if line.find(':') != -1: + break + else: + content_ls.append(line.replace("\r", "").replace("\n", "")[1:]) + ls = ("".join(content_ls)).strip().split(" ") + except Exception, e: + traceback.print_exc() + raise e + finally: + f.close() + return ls + + def get_de_fullpath(self): + '''根据依赖的dm版本得到de.jar的版本，从而得到绝对路径''' + de_path = None + try: + if os.path.exists(self.pom_path): + domtree = xml.dom.minidom.parse(self.pom_path) + root = domtree.documentElement + node1 = root.getElementsByTagName("dependencyManagement") + node2 = node1[0].getElementsByTagName("dependencies") + node3 = node2[0].getElementsByTagName("dependency") + for item in node3: + group = item.getElementsByTagName("groupId")[0].childNodes[0].nodeValue + artifact = item.getElementsByTagName("artifactId")[0].childNodes[0].nodeValue + if group == 'com.nsfocus.bsa' and artifact == 'dm': + version = item.getElementsByTagName("version")[0].childNodes[0].nodeValue + de_path = self.depath + "/de-%s.jar"%(version,) + break + except Exception, e: + traceback.print_exc() + raise e + return de_path + + def compare(self): + '''进行比对，并且输出结果;用于spark job和普通java进程''' + app_de = self.parse_manifest(self.jarpath) + platform_de_fullpath = self.get_de_fullpath() + if platform_de_fullpath is not None: + confdic = {"common":[], "privacy":[]} + platform_de = self.parse_manifest(platform_de_fullpath) + #可以使用平台的jar,取交集 + common_jars = list(set(app_de) & set(platform_de)) + #只能使用app自己的jar，取差集 + diff_jars = list(set(app_de).difference(set(platform_de))) + #app jar依赖的私有jar路径 + privacy_jar_path = os.path.normpath(self.apppath + "/bin/lib/" + \ + os.path.basename(self.jarpath).replace(".jar", "").replace("-","_").replace(".","")) + if not os.path.exists(self.apppath + "/bin/lib"): + os.mkdir(self.apppath + "/bin/lib") + if not os.path.exists(privacy_jar_path): + os.mkdir(privacy_jar_path) + for item in diff_jars: + items = item.split("@") + srcpath = os.path.normpath(self.lib_path + "/" + items[0] + "." \ + + items[1] + "-" + items[2] + ".jar") + dstpath = os.path.normpath(privacy_jar_path + "/" + items[0] + "." \ + + items[1] + "-" + items[2] + ".jar") + shutil.copy(srcpath, dstpath) + confdic["privacy"].append(os.path.normpath(dstpath.replace(os.path.normpath(self.apppath), \ + self.bsaroot + "/apps/" + self.appname)).replace("\\", "/")) + for item in common_jars: + items = item.split("@") + confdic["common"].append(self.platform_libpath + "/" + items[0] + "." \ + + items[1]) + confpath = self.apppath + "/conf/" + self.name + open(confpath, "w").write(json.dumps(confdic)) + print "write conf file:", confpath diff --git a/target/encrypt_conf b/target/encrypt_conf new file mode 100644 index 0000000..16e4303 --- /dev/null +++ b/target/encrypt_conf @@ -0,0 +1,5 @@ +[encrypt] +is_dat = 1 +is_pyc = 1 +is_front_pac = 1 +is_front_enc = 0 \ No newline at end of file diff --git a/target/encrypt_tool/__init__.py b/target/encrypt_tool/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/target/encrypt_tool/encrypt_app.py b/target/encrypt_tool/encrypt_app.py new file mode 100644 index 0000000..c2be7ac --- /dev/null +++ b/target/encrypt_tool/encrypt_app.py @@ -0,0 +1,23 @@ +#- coding:utf-8 -*- +import melon +import sys +import os +import traceback + +class CryptApp(): + + def __init__(self,input_name,output_name): + input_name = input_name.strip() + self.input_name = input_name + if output_name.endswith('.dat'): + self.output_name = output_name + else: + print 'output_file type is error' + sys.exit(1) + + + def encrypt_app(self): + try: + melon.encrypt(self.input_name,self.output_name) + except Exception,e: + traceback.print_exc() diff --git a/target/encrypt_tool/melon/__init__.py b/target/encrypt_tool/melon/__init__.py new file mode 100644 index 0000000..407e2cd --- /dev/null +++ b/target/encrypt_tool/melon/__init__.py @@ -0,0 +1 @@ +from .melon import encrypt, decrypt \ No newline at end of file diff --git a/target/encrypt_tool/melon/cipher.txt b/target/encrypt_tool/melon/cipher.txt new file mode 100644 index 0000000000000000000000000000000000000000..0fd730f2b164591ffdf17c77fd0425d4d169404c GIT binary patch literal 1259 zcmV_P1jXUGrSQ#@Pz48j|a7cFX%sh3V5H<8UjOfWTt}DEZXP?xuO!q zYYRRSpM8(?@tmY=d=qi%D!L-K*dky>Cy0X~@|s*WDUvUaGL&7&LRZL3xllWDpKe(E z;8BLQ{*yJccZ-~E*A!2B)ClGu7ox_&71AGZNyuT(dzCFVMuRxDaSoN0g2s10e`+mO zk}l9M^dnmX23zOyP@%5_Q*FIGg?7mxv>o8LpYMh6G}(E02%79=Abv4`H48F;GzR?G zX2)*d$A(<%X+VN#IPgukbY~Va9TapM{pq&O@mXw?`g&^eu&U2Yl#MAXJetfMHC#Ea za^sSG{Z%vC6}`vDk`A@{I{#ksNSw_{#Lofhf2L_>><%6CV%fJ(CTw5#xM=V3E;shTA%i9z4Tiewj}I%V~2FH6{(p5UcC-G?a)W@c6bpvlUAyl1sR7K_5G`9jPcjZEhl#91y#oIa!Ume7CkHDb=3KL<#xf zid?XnSc3nGFk~hN>jt?Y61G#+Mx-!%zew z%3<*e+47`raXiaKGymFnPp=f#=~Kc&F83O0yqG@*tnw8@~tDTN#A z@tu6dzh7{9?mll!H3{puQ76j?pL;ffcA02Xa1sgQq_>h29?TtF0zcV-%k zx6nLL> 8) & 0xff)) + f.write(chr((x >> 16) & 0xff)) + f.write(chr((x >> 24) & 0xff)) + + +def protect_file(file, cfile=None): + """ Protect a pyfile """ + if os.path.islink(file) and not m_symlink: + print 'Protect: ignore symlink file:', file + return True + + if not os.path.isfile(file): + return False + + if file.endswith(".pyc") or file.endswith(".pyo"): + return False + + if not file.endswith(".py"): + print 'Protect: ignore non-py file:', file + return True + + f = open(file, 'U') + try: + timestamp = long(os.fstat(f.fileno()).st_mtime) + except AttributeError: + timestamp = long(os.stat(file).st_mtime) + codestring = f.read() + f.close() + if codestring and codestring[-1] != '\n': + codestring = codestring + '\n' + + try: + os.putenv('PYTHONOPTIMIZE', '2') + codeobject = compile(codestring, file, 'exec') + except: + traceback.print_exc() + return False + + _pyprotect.protect(codeobject) + + if cfile is None: + cfile = file + (__debug__ and 'c' or 'o') + + fc = open(cfile, 'wb') + fc.write('\0\0\0\0') + wr_long(fc, timestamp) + marshal.dump(codeobject, fc) + fc.flush() + fc.seek(0, 0) + fc.write(MAGIC) + fc.close() + + if m_delete: + try: + os.unlink(file) + except Exception, e: + print 'Protect: failed to delete file: %s, reason: %s' % (file, + str(e)) + + return True + + +def protect_dir(dirname): + """ Protect py files in a dir + + """ + for dirname, dirnames, filenames in os.walk(dirname): + for filename in filenames: + fullname = os.path.join(dirname, filename) + protect_file(fullname) + + if not m_recurse: + break + + +def protect(filenames): + """ Protect file or dirs """ + + for filename in filenames: + if os.path.isfile(filename): + protect_file(filename) + elif os.path.isdir(filename): + protect_dir(filename) + else: + print 'Unknown file type:', filename + + +def parse_cmdline(): + """ Parse command line + + """ + from optparse import OptionParser + global m_recurse, m_delete, m_symlink + + parser = OptionParser(usage="usage: %prog [options]") + + parser.add_option('-R', dest='recurse', default=None, action='store_true', + help='Protect all .py in the dir and subdirs') + parser.add_option('--delete', default=None, action='store_true', + help='Delete origin .py file after protect') + parser.add_option('--symlink', default=None, action='store_true', + help='Follow symlink file') + + opts, args = parser.parse_args() + + m_recurse = opts.recurse + m_delete = opts.delete + m_symlink = opts.symlink + + return args + + +if __name__ == '__main__': + filenames = parse_cmdline() + + if not filenames: + print 'Usage: pyprotect [-R] [--delete] file|dir' + sys.exit(0) + + protect(filenames) + diff --git a/target/package.py b/target/package.py new file mode 100644 index 0000000..7149b47 --- /dev/null +++ b/target/package.py @@ -0,0 +1,249 @@ +# -*- coding: UTF-8 -*- +import ConfigParser +import json +import os +import re +import sys +import time + +# from compare_dependency import CompareDependency +from encrypt_tool.encrypt_app import CryptApp + +DIR_PATH = os.path.split(os.path.realpath(__file__))[0] +BASE_PATH = os.path.split(DIR_PATH)[0] + + +class Package(object): + def __init__(self): + # 第一步：初始化一些变量，获取app fold name 版本等等 + self.__svn_up() + self.app_name = self.__get_app_name() + self.version = '1.0.0' + self.work_path_base = '/tmp' + # 输出的dat包放置的位置，默认与package.py 放在同级目录 + self.package_out_path = DIR_PATH + # 打包的临时目录 + self.app_work_path = os.path.normpath(os.path.join(self.work_path_base, self.app_name)) + print self.app_work_path + # 清理tmp目录下面的已有的 + os.system("rm -rf " + self.app_work_path) + # 不加密的py目录 + self.py_not_encnrypt_list = self.__get_py_not_encrypt() + # 第二步：选择编译前端代码,npm run build + self.packageFrontEnd() + # 第三步：获取svn版本号 + # self.build_version = self.__get_svn() + self.build_version = 23000 + self.package_name = "%s.%s.%s.tar.gz" % (self.app_name, self.version, self.build_version) + # 第五步：拷贝需要打包进组件的目录和文件到临时目录 + self.do_copy_work_dir() + # 第六步：检查加密的包配置 + self.__get_encrypt_conf() + # 第七步：把build_version 写到文件里面 + self.__set_build_version() + # 第八步：检查加密py的配置 + self.__do_encrypt_file() + # 第九步：加密py + self.run_encrypt() + + def __get_app_name(self): + with open(os.path.join(BASE_PATH, "package.json"), "r+") as f: + pkgConfig = json.load(f) + return pkgConfig["name"] + + def __get_py_not_encrypt(self): + return { + 'files': [ + # os.path.normpath(APP_WORK_PATH + os.sep + 'install.py'), + # os.path.normpath(APP_WORK_PATH + os.sep + "bin" + os.sep + "start.py"), + # os.path.normpath(APP_WORK_PATH + os.sep + "bin" + os.sep + "check.py"), + ], + 'paths': [ + os.path.normpath(self.app_work_path + os.sep + 'target'), + os.path.normpath(self.app_work_path + os.sep + 'static'), + ], + } + + def __svn_up(self): + # 更新svn到最新 + # cmd = "cd %s && git pull " % BASE_PATH + # os.system(cmd) + pass + + def __get_svn(self): + return 10000 + # svnPath = BASE_PATH + # try: + # cmd = 'cd %s && svn info' % svnPath + # svnContent = os.popen(cmd).read() + # lang = os.popen('echo $LANG').read() + # if 'en_US' in lang: + # revision = 'Rev' + # elif 'zh_CN' in lang: + # revision = '最后修改的版本' + # else: + # print 'system language is not found' + # sys.exit(1) + # svnRe = re.compile('%s: (\d*?)\n' % revision) + # svn_revision = svnRe.findall(svnContent)[0] + # print 'svn_revision : %s cmd : %s' % (svn_revision, cmd) + # except Exception, e: + # print 'get SVN revision Failed. %s' % (str(e)) + # sys.exit(1) + # return svn_revision + + def do_copy_work_dir(self): + print '拷贝数据到临时目录' + os.chdir(BASE_PATH) + cmd = 'rsync -az * --include="static/dist" --include="static/icon" --exclude="static/*" %s' % ( + self.app_work_path) + print cmd + os.system(cmd) + print '拷贝完成' + + def __get_encrypt_conf(self): + sections_dict = dict() + conf_path = os.path.normpath(os.path.join(self.app_work_path, 'target', 'encrypt_conf')) + if not os.path.exists(conf_path): + print 'conf file is not exists...' + return + try: + cf = ConfigParser.ConfigParser() + cf.read(conf_path) + sections = cf.sections() + for section in sections: + options = cf.options(section) + options_dic = dict() + for option in options: + try: + value = cf.get(section, option).decode('utf-8', 'ignore') + except Exception, e: + value = cf.get(section, option) + options_dic[option] = value + sections_dict[section] = options_dic + except Exception, e: + print 'error', str(e) + try: + self.is_dat = int(sections_dict['encrypt'].get('is_dat', 0)) + except: + pass + try: + self.is_pyc = int(sections_dict['encrypt'].get('is_pyc', 0)) + except: + pass + try: + self.is_front_pac = int(sections_dict['encrypt'].get('is_front_pac', 0)) + except: + pass + try: + self.is_front_enc = int(sections_dict['encrypt'].get('is_front_enc', 0)) + except: + pass + + def __set_build_version(self): + print "set package name ..." + try: + app_version_file = os.path.normpath(os.path.join(self.app_work_path, 'conf', 'buildversion')) + with open(app_version_file, 'w') as fw: + fw.write(self.build_version) + except Exception, e: + print e + print "set package name done" + + def __do_encrypt_file(self): + encrypt_pt_path = os.path.normpath(os.path.join(DIR_PATH, 'encrypt_tool', 'pyprotect.py')) + print "start encrypt python file ..." + for root, dirs, files in os.walk(self.app_work_path): + if root in self.py_not_encnrypt_list['paths']: + continue + for fname in files: + pyPath = os.path.normpath(os.path.join(root, fname)) + if not fname.endswith('.py'): + continue + if pyPath in self.py_not_encnrypt_list['files']: + continue + cmd = "python '%s' '%s'" % (encrypt_pt_path, pyPath) + os.system(cmd) + # print cmd + os.remove(pyPath) + print "encrypt python file done" + + def packageFrontEnd(self): + st = time.time() + print "package static folder start ..." + os.system('cd %s/static && npm install && npm run build' % BASE_PATH) + et = time.time() + print "package static folder done" + print 'take time for %sS' % (et - st) + + def run_encrypt(self): + print "package ..." + install_py = "" + os.chdir(self.work_path_base) + # 把install 挪到和app 目录同级 + # 判断install.py 存不存在，如果不存在，不必拷贝 + if os.path.exists(os.path.join(self.app_work_path, "install.py")) or os.path.exists( + os.path.join(self.app_work_path, "install.pyc")): + install_py = "install.pyc" + os.system("mv %s/%s ." % (self.app_name, install_py)) + os.system("tar -zcf %s %s/ %s --exclude=.svn --exclude=%s/target --exclude=%s/java " + "> /dev/null " % (self.package_name, self.app_name, install_py, self.app_name, self.app_name)) + else: + print "tar -zcf %s %s/ --exclude=.svn --exclude=%s/target --exclude=%s/java > /dev/null " % ( + self.package_name, self.app_name, self.app_name, self.app_name) + os.system("tar -zcf %s %s/ --exclude=.svn --exclude=%s/target --exclude=%s/java > /dev/null " % ( + self.package_name, self.app_name, self.app_name, self.app_name)) + print '执行成功' + replace_reg = re.compile(r'tar.gz$') + output_name = replace_reg.sub('dat', self.package_name) + do_encrypt = CryptApp(self.package_name, output_name) + do_encrypt.encrypt_app() + # 删除 tar包 保留加密包 + os.system("rm -rf %s " % self.package_name) + if install_py: + os.system("mv %s %s" % (install_py, self.app_name)) + os.system("mv %s %s -f" % (output_name, DIR_PATH)) + self.package_name = output_name + print "package done" + self.__info() + + def __info(self): + print '\033[0;33m' + print "*********************************" + print "package ok. " + print "package name:", self.package_name + outputfile = os.path.join(self.package_out_path, self.package_name) + md5 = os.popen("md5sum %s" % outputfile).readlines()[0].strip() + print "package path:", md5.split(' ')[-1] + print "package md5:", md5.split(' ')[0] + print '>>>>>>', 'is_dat:', 'yes' if self.is_dat else 'no' + print '>>>>>>', 'is_pyc:', 'yes' if self.is_pyc else 'no' + print '>>>>>>', 'is_front_enc:', 'yes' if self.is_front_enc else 'no' + print '>>>>>>', 'is_front_pac:', 'yes' if self.is_front_pac else 'no' + print "*********************************" + print '\033[0m' + + def do_remove_work_dir(self): + cmd = 'rm %s -rf' % self.app_work_path + os.system(cmd) + # print 'remove %s successfully...' % (APP_WORK_PATH) + + +def usage(): + print "usage:" + print "python package.py ---- run script without a parameter, make a encrypted package" + + +if __name__ == '__main__': + ''' + "python package.py ---- run script without a parameter, make a encrypted package" + ''' + main_param = sys.argv + # 清除已经安装的包 + for fileitem in os.listdir(BASE_PATH + "/target"): + if fileitem.endswith(".dat"): + os.remove(BASE_PATH + "/target/" + fileitem) + if len(main_param) == 1: + Package() + else: + usage() diff --git a/urls.py b/urls.py new file mode 100644 index 0000000..5c7fb19 --- /dev/null +++ b/urls.py @@ -0,0 +1,14 @@ +# coding:utf-8 + +from django.conf.urls import url +from UebaMetricsAnalysis.views import dashboard_views +from rest_framework import routers + + +urlpatterns = [ +] + +router = routers.DefaultRouter() +router.register(r'/rule_info', dashboard_views.DashboardViewSets,base_name="dashboard-view") + +urlpatterns += router.urls \ No newline at end of file diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/ext_logging.py b/utils/ext_logging.py new file mode 100644 index 0000000..c212021 --- /dev/null +++ b/utils/ext_logging.py @@ -0,0 +1,38 @@ +# coding=utf-8 +""" + @Author: fu-zhe + @FileName: ext_logging.py + @DateTime: 2024/5/16 14:27 + @Description: +""" +import logging +import os +from mlogging import TimedRotatingFileHandler_MP +from appsUtils import env + +APPFOLDERNAME = 'UebaMetricsAnalysis' + + +def get_logger(logfile): + """ + 获取日志句柄 + """ + logger = logging.getLogger(logfile) + logger.setLevel(logging.DEBUG) + logroot = env.get_isop_root() + "/apps/" + APPFOLDERNAME + "/logs" + if not os.path.exists(logroot): + os.mkdir(logroot) + filehandle = TimedRotatingFileHandler_MP(os.path.normpath(logroot + "/" + \ + logfile + ".log"), 'midnight') + filehandle.suffix = "%Y-%m-%d" + filehandle.setLevel(logging.DEBUG) + consolehandle = logging.StreamHandler() + consolehandle.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + filehandle.setFormatter(formatter) + consolehandle.setFormatter(formatter) + logger.addHandler(filehandle) + return logger + + +logger = get_logger(APPFOLDERNAME) diff --git a/views/__init__.py b/views/__init__.py new file mode 100644 index 0000000..7c34a1b --- /dev/null +++ b/views/__init__.py @@ -0,0 +1,7 @@ +# coding=utf-8 +""" + @Author: tangwy + @FileName: __init__.py.py + @DateTime: 2024/6/6 10:06 + @Description: +""" diff --git a/views/dashboard_views.py b/views/dashboard_views.py new file mode 100644 index 0000000..65f5f64 --- /dev/null +++ b/views/dashboard_views.py @@ -0,0 +1,53 @@ +# coding=utf-8 +""" + @Author: fu-zhe + @FileName: dashboard_views.py + @DateTime: 2024/5/20 17:34 + @Description: +""" + +import json +import traceback + +from rest_framework import viewsets +from rest_framework.decorators import list_route, detail_route +from UebaMetricsAnalysis.utils.ext_logging import logger +from UebaMetricsAnalysis.lib.result import Result + +class DashboardViewSets(viewsets.GenericViewSet): + @list_route(methods=['GET']) + def training_cyber_range(self, request): + type = request.GET.get('type') + try: + return self.get_summary_data(self,type) + except Exception, e: + logger.error("实训大屏数据获取失败, err: {}, traceback: {}".format(str(e), traceback.format_exc())) + return Result.failed("查询失败", str(e)) + + @list_route(methods=['GET']) + def get_summary_data_list(self,request): + data_type = request.GET.get('type') + try: + if data_type == "ip": + return self.get_ip_summary_data() + if data_type == "account": + return self.get_account_summary_data() + if data_type == "interface": + return self.get_interface_summary_data() + if data_type == "menu": + return self.get_menu_summary_data() + return Result.ok({"status":200}) + except Exception, e: + logger.error("实训大屏数据获取失败, err: {}, traceback: {}".format(str(e), traceback.format_exc())) + return Result.failed("查询失败", str(e)) + def get_ip_summary_data(self): + return Result.ok({"status":200}) + + def get_account_summary_data(self): + return Result.ok({"status":200}) + + def get_interface_summary_data(self): + return Result.ok({"status":200}) + + def get_menu_summary_data(self): + return Result.ok({"status":200}) \ No newline at end of file