@ -6,10 +6,11 @@ import json
import os , re
import codecs
import traceback
from datetime import datetime , timedelta
from collections import defaultdict
from dashboard_data_conversion import ip_summary_data_format , account_summary_data_format , \
interface_summary_data_format , menu_summary_data_format , adjust_times , jobnum_region_dict , find_region_by_code
from isoc . utils . esUtil import EsUtil
from isoc . utils . dashboard_data_conversion import ip_summary_data_format , account_summary_data_format , \
interface_summary_data_format , menu_summary_data_format , calculate_time_difference , summary_data_reqs_format
from dataInterface . functions import CFunction
from dataInterface . db . params import CPgSqlParam
@ -33,15 +34,15 @@ def pg_get_ip_group_data(startTime, endTime):
"""
result = [ ]
sql = """ select ip, jobnum, sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
where logdate > = % s and logate < % s and data_type = % s
group by ip , jobnum """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " IP " ] ) ) ) )
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " IP " ] ) ) )
if res :
for item in res :
result . append ( {
" ip " : item [ 0 ] ,
" jobnum " : item [ 1 ] ,
" count " : item [ 2 ] ,
" jobnum " : item [ 2 ] ,
" count " : item [ 3 ] ,
} )
return result
@ -54,17 +55,15 @@ def pg_get_account_group_data(startTime, endTime):
"""
result = [ ]
sql = """ select account, jobnum, sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
where logdate > = % s and logate < % s and data_type = % s
group by account , jobnum """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " ACCOUNT " ] ) ) ) )
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " ACCOUNT " ] ) ) )
if res :
for item in res :
print ( " items: " + json . dumps ( item ) )
print ( " pg_get_account_group_data维度data: " + json . dumps ( item [ 0 ] ) )
result . append ( {
" account " : item [ 0 ] ,
" jobnum " : item [ 1 ] ,
" count " : item [ 2 ] ,
" jobnum " : item [ 2 ] ,
" count " : item [ 3 ] ,
} )
return result
@ -76,18 +75,17 @@ def pg_get_interface_group_data(startTime, endTime):
: param endTime : 结束时间 ,
"""
result = [ ]
sql = """ select interface, ip, jobnum,account , sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
group by interface , ip , jobnum , account """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " INTERFACE " ] ) ) ) )
sql = """ select interface, s ip, jobnum, sum(count) from {TABLE_NAME}
where logdate > = % s and logate < % s and data_type = % s
group by interface , ip , jobnum """ .format(TABLE_NAME=TABLE_NAME)
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " INTERFACE " ] ) ) )
if res :
for item in res :
result . append ( {
" interface " : item [ 0 ] ,
" ip " : item [ 1 ] ,
" jobnum " : item [ 2 ] ,
" account " : item [ 3 ] ,
" count " : item [ 4 ] ,
" count " : item [ 3 ] ,
} )
return result
@ -99,18 +97,17 @@ def pg_get_menu_group_data(startTime, endTime):
: param endTime : 结束时间 ,
"""
result = [ ]
sql = """ select menu, ip,jobnum,account, sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
group by menu , ip , jobnum , account """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " MENU " ] ) ) ) )
sql = """ select menu, sip, jobnum, sum(count) from {TABLE_NAME}
where logdate > = % s and logate < % s and data_type = % s
group by menu , ip , jobnum """ .format(TABLE_NAME=TABLE_NAME)
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " MENU " ] ) ) )
if res :
for item in res :
result . append ( {
" menu " : item [ 0 ] ,
" ip " : item [ 1 ] ,
" jobnum " : item [ 2 ] ,
" account " : item [ 3 ] ,
" count " : item [ 4 ] ,
" count " : item [ 3 ] ,
} )
return result
@ -126,15 +123,16 @@ def pg_get_previous_company_count(startTime, endTime, data_type):
if data_type in DATA_TYPE :
data_type = DATA_TYPE [ data_type ]
sql = """ select jobnum, sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
where logdate > = % s and logate < % s and data_type = % s
group by jobnum """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , data_type ) ) ) )
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , data_type ) ) )
if res :
for item in res :
company = find_region_by_code ( item [ 0 ] , jobnum_region_dict )
result [ company ] + = item [ 1 ]
return result
def pg_get_previous_interface_count ( startTime , endTime ) :
"""
接口维度查询请求总次数
@ -143,9 +141,9 @@ def pg_get_previous_interface_count(startTime, endTime):
"""
result = defaultdict ( int )
sql = """ select interface, sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
where logdate > = % s and logate < % s and data_type = % s
group by interface """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " INTERFACE " ] ) ) ) )
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " INTERFACE " ] ) ) )
if res :
for item in res :
result [ item [ 0 ] ] + = item [ 1 ]
@ -160,31 +158,27 @@ def pg_get_previous_menu_count(startTime, endTime):
"""
result = defaultdict ( int )
sql = """ select menu, sum(count) from {TABLE_NAME}
where logdate > = % s and logd ate < = % s and data_type = % s
where logdate > = % s and logate < % s and data_type = % s
group by menu """ .format(TABLE_NAME=TABLE_NAME)
res = json . loads ( CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " MENU " ] ) ) ) )
res = CFunction . execute ( CPgSqlParam ( sql , params = ( startTime , endTime , DATA_TYPE [ " MENU " ] ) ) )
if res :
for item in res :
result [ item [ 0 ] ] + = item [ 1 ]
return result
def entry ( data_type , start , end ) :
# 前一段开始时间
date_format = " % Y- % m- %d % H: % M: % S "
start = datetime . strptime ( start , date_format )
end = datetime . strptime ( end , date_format )
old_start , old_end = adjust_times ( start , end )
previous_time = calculate_time_difference ( start , end )
try :
data = { }
if data_type == " 1 " :
ip_summary_data = pg_get_ip_group_data ( start , end )
data = ip_summary_data_format ( ip_summary_data )
previous_company_dict = pg_get_previous_company_count ( old_start , start , " IP " )
for d in data [ " summary " ] [ " ip " ] :
if previous_company_dict . get ( d [ " company " ] , 0 ) == 0 :
d [ " trend " ] = 0
else :
previous_company_dict = pg_get_previous_company_count ( previous_time , start , " IP " )
for d in data [ " summary " ] [ " account " ] :
d [ " trend " ] = round (
( d [ " req_frequency " ] - previous_company_dict . get ( d [ " company " ] , 0 ) ) / previous_company_dict . get (
d [ " company " ] , 0 ) , 4 )
@ -193,11 +187,8 @@ def entry(data_type, start, end):
account_summary_data = pg_get_account_group_data ( start , end )
data = account_summary_data_format ( account_summary_data )
previous_company_dict = pg_get_previous_company_count ( old_start , start , " ACCOUNT " )
previous_company_dict = pg_get_previous_company_count ( previous_time , start , " ACCOUNT " )
for d in data [ " summary " ] [ " account " ] :
if previous_company_dict . get ( d [ " company " ] , 0 ) == 0 :
d [ " trend " ] = 0
else :
d [ " trend " ] = round (
( d [ " req_frequency " ] - previous_company_dict . get ( d [ " company " ] , 0 ) ) / previous_company_dict . get (
d [ " company " ] , 0 ) , 4 )
@ -206,26 +197,22 @@ def entry(data_type, start, end):
interface_summary_data = pg_get_interface_group_data ( start , end )
data = interface_summary_data_format ( interface_summary_data )
previous_interface_dict = pg_get_previous_interface_count ( old_start , start )
for d in data [ " summary " ] [ " interface " ] :
if previous_interface_dict . get ( d [ " interface_addr " ] , 0 ) == 0 :
d [ " trend " ] = 0
else :
previous_interface_dict = pg_get_previous_interface_count ( previous_time , start )
for d in data [ " summary " ] [ " account " ] :
d [ " trend " ] = round (
( d [ " req_frequency " ] - previous_interface_dict . get ( d [ " interface_addr " ] , 0 ) ) / previous_interface_dict . get (
d [ " interface_addr " ] , 0 ) , 4 )
( d [ " req_frequency " ] - previous_interface_dict . get ( d [ " company " ] , 0 ) ) / previous_interface_dict . get (
d [ " company " ] , 0 ) , 4 )
if data_type == " 4 " :
menu_summary_data = pg_get_menu_group_data ( start , end )
data = menu_summary_data_format ( menu_summary_data )
previous_menu_dict = pg_get_previous_menu_count ( old_start , start )
for d in data [ " summary " ] [ " menu " ] :
if previous_menu_dict . get ( d [ " menu_name " ] , 0 ) == 0 :
d [ " trend " ] = 0
else :
previous_menu_dict = pg_get_previous_menu_count ( previous_time , start )
for d in data [ " summary " ] [ " account " ] :
d [ " trend " ] = round (
( d [ " req_frequency " ] - previous_menu_dict . get ( d [ " menu_name " ] , 0 ) ) / previous_menu_dict . get (
d [ " menu_name " ] , 0 ) , 4 )
( d [ " req_frequency " ] - previous_menu_dict . get ( d [ " company " ] , 0 ) ) / previous_menu_dict . get (
d [ " company " ] , 0 ) , 4 )
return data
except Exception , e :
logger . error ( " 分析结构获取失败, err: {} , traceback: {} " . format ( str ( e ) , traceback . format_exc ( ) ) )
raise e