From f7d65db16dd3a26f4fc765a3f447d087f7218c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 10:41:58 +0800 Subject: [PATCH 01/30] refactor: add project filter for analysis sql --- tone/core/common/constant.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tone/core/common/constant.py b/tone/core/common/constant.py index b8c678c..631e386 100644 --- a/tone/core/common/constant.py +++ b/tone/core/common/constant.py @@ -378,6 +378,7 @@ ANALYSIS_SUITE_LIST_SQL_MAP = { B.id = A.test_job_id AND B.server_provider = %s AND B.test_type = 'performance' + AND B.project_id = %s AND A.test_suite_id = C.id AND A.test_case_id = D.id AND B.state IN ( 'success', 'fail' ) @@ -404,6 +405,7 @@ ANALYSIS_SUITE_LIST_SQL_MAP = { WHERE B.id = A.test_job_id AND B.test_type = 'functional' + AND B.project_id = %s AND A.test_suite_id = C.id AND A.test_case_id = D.id AND B.state IN ( 'success', 'fail' ) -- Gitee From 2b1beeb7aea5fb95af0f9109b811ec8c618923bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 10:47:16 +0800 Subject: [PATCH 02/30] feature: add job summary for ding talk --- tone/core/common/msg_notice.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tone/core/common/msg_notice.py b/tone/core/common/msg_notice.py index 68f5a3a..332948e 100644 --- a/tone/core/common/msg_notice.py +++ b/tone/core/common/msg_notice.py @@ -18,6 +18,7 @@ from tone.models import RoleMember, WorkspaceMember, Role, ApproveInfo, InSiteWo from tone.serializers.job.test_serializers import JobTestResultSerializer, JobTestSummarySerializer from tone.services.portal.sync_portal_task_servers import save_report +from tone.core.common.job_result_helper import calc_job logger = logging.getLogger('message') @@ -342,7 +343,7 @@ class SimpleMsgHandle(object): simple_msg_list.append(simple_msg_obj) InSiteSimpleMsg.objects.bulk_create(simple_msg_list) # 创建站外消息 - OutSiteMsgHandle().job_handle(job_obj, receiver_list, message_obj.job_state) + OutSiteMsgHandle().job_handle(job_obj, message_obj.job_state) if job_obj.callback_api: JobCallBack(job_id=job_id, callback_type=CallBackType.JOB_COMPLETED).callback() return True @@ -531,8 +532,8 @@ class OutSiteMsgHandle(object): @staticmethod def get_job_content(job_obj): """获取job信息""" - content = '''Tone平台\nID: {task_id}\nTask: {task}\nAuthor: {author}\nDuration: {duration} (hour)''' - content = content.format(task_id=job_obj.id, task=job_obj.name, author=get_user_name(job_obj.creator), + content = '''JobID: {task_id}; Author: {author}\nDuration: {duration} (hour)''' + content = content.format(task_id=job_obj.id, author=get_user_name(job_obj.creator), duration=round( (job_obj.gmt_modified - job_obj.gmt_created).total_seconds() / 60 / 60, 2)) return content @@ -845,7 +846,7 @@ class OutSiteMsgHandle(object): msg_pic=msg_pic, ) - def job_handle(self, job_obj, receiver_list, job_state): + def job_handle(self, job_obj, job_state): """Job站外消息处理""" creator_email = ','.join(self.get_email_list([job_obj.creator])) job_state = get_job_state(job_state, job_obj) @@ -887,6 +888,16 @@ class OutSiteMsgHandle(object): else: msg_pic = 'https://www.iconsdb.com/icons/preview/soylent-red/x-mark-3-xxl.png' if ding_to and settings.MSG_SWITCH_ON: + job_summary = calc_job(job_obj.id) + if job_summary: + if job_obj.test_type == 'functional': + content += '\nStatistics：通过:{};失败:{};跳过:{};警告:{}'. \ + format(job_summary.get('success'), job_summary.get('fail'), job_summary.get('skip'), + job_summary.get('warn')) + else: + content += '\nStatistics：上升:{};下降:{};正常:{};无效:{};NA:{}'. \ + format(job_summary.get('increase'), job_summary.get('decline'), job_summary.get('invalid'), + job_summary.get('invalid'), job_summary.get('na')) OutSiteMsg.objects.create( subject=subject if not ding_subject else ding_subject, content=content, -- Gitee From 0507f5e6c4ad341d1afdf9f024571e0e4f359bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 10:52:53 +0800 Subject: [PATCH 03/30] feature: add server_provider for env info api --- tone/core/handle/report_handle.py | 41 ++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tone/core/handle/report_handle.py b/tone/core/handle/report_handle.py index 00c3a19..ccf79a4 100644 --- a/tone/core/handle/report_handle.py +++ b/tone/core/handle/report_handle.py @@ -746,7 +746,8 @@ def get_server_info(tag, objs): # nq c901 def get_group_server_info(base_group, compare_groups): baseline_id_list = list() - job_id_list = list() + aligroup_job_id_list = list() + cloud_job_id_list = list() base_group['is_base'] = 1 groups = list() groups.append(base_group) @@ -760,16 +761,21 @@ def get_group_server_info(base_group, compare_groups): is_baseline = obj.get('is_baseline', 0) group['is_baseline'] = is_baseline obj_id = obj.get('obj_id') + provider_env = obj.get('server_provider') group['job_id'].append(obj_id) if is_baseline: baseline_id_list.append(obj_id) else: - job_id_list.append(obj_id) + if provider_env == 'aligroup': + aligroup_job_id_list.append(obj_id) + else: + cloud_job_id_list.append(obj_id) env_info = { 'base_group': list(), 'compare_groups': list() } - job_id_str = ','.join(str(e) for e in job_id_list) + aligroup_job_id_str = tuple(aligroup_job_id_list) + cloud_job_id_str = tuple(cloud_job_id_list) baseline_server_info = None all_server_info = None if baseline_id_list: @@ -778,15 +784,26 @@ def get_group_server_info(base_group, compare_groups): 'glibc,memory_info,disk,cpu_info,ether FROM baseline_server_snapshot ' \ 'WHERE baseline_id IN (' + baseline_id_str + ') ' baseline_server_info = query_all_dict(baseline_raw_sql.replace('\'', ''), params=None) - if job_id_str: - raw_sql = 'SELECT job_id,ip,sm_name,distro,rpm_list,kernel_version,gcc,' \ - 'glibc,memory_info,disk,cpu_info,ether FROM test_server_snapshot ' \ - 'WHERE job_id IN (' + job_id_str + ') UNION ' \ - 'SELECT job_id,pub_ip AS ip,instance_type AS sm_name,distro,rpm_list,' \ - 'kernel_version,gcc,glibc,memory_info,disk,cpu_info,ether ' \ - 'FROM cloud_server_snapshot ' \ - 'WHERE job_id IN (' + job_id_str + ')' - all_server_info = query_all_dict(raw_sql.replace('\'', ''), params=None) + aligroup_raw_sql = '' + cloud_raw_sql = '' + raw_sql = '' + params = list() + if aligroup_job_id_str: + aligroup_raw_sql = 'SELECT job_id,ip,sm_name,distro,rpm_list,kernel_version,gcc,' \ + 'glibc,memory_info,disk,cpu_info,ether FROM test_server_snapshot ' \ + 'WHERE job_id IN %s ' + raw_sql = aligroup_raw_sql + params.append(aligroup_job_id_str) + if cloud_job_id_str: + cloud_raw_sql = 'SELECT job_id,pub_ip AS ip,instance_type AS sm_name,distro,rpm_list,' \ + 'kernel_version,gcc,glibc,memory_info,disk,cpu_info,ether ' \ + 'FROM cloud_server_snapshot WHERE job_id IN %s' + raw_sql = cloud_raw_sql + params.append(cloud_job_id_str) + if cloud_job_id_str and aligroup_job_id_str: + raw_sql = aligroup_raw_sql + ' UNION ' + cloud_raw_sql + if raw_sql: + all_server_info = query_all_dict(raw_sql.replace('\'', ''), params=params) for group in groups: group_ip_list = list() server_li = list() -- Gitee From 132bcb8c76faafd3be135132225e8ef911108de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 10:56:15 +0800 Subject: [PATCH 04/30] fix: add baseline_name for performance result --- tone/serializers/job/test_serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tone/serializers/job/test_serializers.py b/tone/serializers/job/test_serializers.py index a62bb76..4cae085 100644 --- a/tone/serializers/job/test_serializers.py +++ b/tone/serializers/job/test_serializers.py @@ -801,6 +801,7 @@ class JobTestCasePerResultSerializer(CommonSerializer): if perf_detail is not None and obj.match_baseline: return {'server_provider': baseline_obj.server_provider, 'test_type': baseline_obj.test_type, + 'baseline_name': baseline_obj.name, 'test_suite_id': perf_detail.test_suite_id, 'test_case_id': perf_detail.test_case_id, 'baseline_id': perf_detail.baseline_id, -- Gitee From 9a6909c5a4d104d6407b2dd629f9b9795276f3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:01:05 +0800 Subject: [PATCH 05/30] refactor: extract package metric and job info --- tone/services/job/result_analysis_services.py | 147 +++++++----------- 1 file changed, 59 insertions(+), 88 deletions(-) diff --git a/tone/services/job/result_analysis_services.py b/tone/services/job/result_analysis_services.py index 28fa4b7..fe9c161 100644 --- a/tone/services/job/result_analysis_services.py +++ b/tone/services/job/result_analysis_services.py @@ -127,54 +127,18 @@ class PerfAnalysisService(CommonService): @staticmethod def get_metric_data(rows, provider_env): metric_data = list() - if provider_env == 'aligroup': - for row in rows: - server = get_job_case_run_server(row['job_case_id']) if row['run_mode'] == 'cluster' else row['ip'] - metric_data.append( - { - 'job_id': row['id'], - 'job_name': row['name'], - 'start_time': datetime.strftime(row['start_time'], "%Y-%m-%d %H:%M:%S") - if row['start_time'] else None, - 'end_time': datetime.strftime(row['end_time'], "%Y-%m-%d %H:%M:%S") - if row['end_time'] else None, - 'commit_id': json.loads(row['build_pkg_info']).get('commit_id', None), - 'creator': row['first_name'] or row['last_name'], - 'server': server, - 'value': row['test_value'], - 'cv_value': row['cv_value'], - 'note': row['note'], - 'result_obj_id': row['result_obj_id'], - # C.id和C.run_mode未取值，所以跳过 - 'creator_id': row['creator_id'] - } - ) - else: - for row in rows: - server = get_job_case_run_server(row['job_case_id']) if row['run_mode'] == 'cluster' \ - else row['private_ip'] - metric_data.append( - { - 'job_id': row['id'], - 'job_name': row['name'], - 'start_time': datetime.strftime(row['start_time'], "%Y-%m-%d %H:%M:%S") - if row['start_time'] else None, - 'end_time': datetime.strftime(row['end_time'], "%Y-%m-%d %H:%M:%S") - if row['end_time'] else None, - 'commit_id': json.loads(row['build_pkg_info']).get('commit_id', None), - 'creator': row['first_name'] or row['last_name'], - 'server': server, - 'value': row['test_value'], - 'cv_value': row['cv_value'], - 'note': row['note'], - 'result_obj_id': row['result_obj_id'], - 'instance_type': row['instance_type'], - 'image': row['image'], - 'bandwidth': row['bandwidth'], - 'run_mode': row['run_mode'], - 'creator_id': row['creator_id'] - } - ) + for row in rows: + if provider_env == 'aligroup': + metric_obj = _package_metric(row, 'ip') + else: + metric_obj = _package_metric(row, 'pub_ip') + metric_obj.update({ + 'instance_type': row['instance_type'], + 'image': row['image'], + 'bandwidth': row['bandwidth'], + 'run_mode': row['run_mode'] + }) + metric_data.append(metric_obj) return metric_data @staticmethod @@ -201,47 +165,12 @@ class PerfAnalysisService(CommonService): def get_job_list(result_data, provider_env): job_id_list = list() job_list = list() - if provider_env == 'aligroup': - for value in result_data.values(): + for value in result_data.values(): + if provider_env == 'aligroup': for _value in value.values(): - if _value and _value.get('job_id') not in job_id_list: - job_list.append({ - 'job_id': _value.get('job_id'), - 'job_name': _value.get('job_name'), - 'start_time': _value.get('start_time'), - 'end_time': _value.get('end_time'), - 'commit_id': _value.get('commit_id'), - 'creator': _value.get('creator'), - 'server': _value.get('server'), - 'server_id': _value.get('server_id'), - 'server_provider': provider_env, - 'value': _value.get('value'), - 'cv_value': _value.get('cv_value'), - 'note': _value.get('note'), - 'result_obj_id': _value.get('result_obj_id'), - 'creator_id': _value.get('creator_id') - }) - job_id_list.append(_value.get('job_id')) - else: - for value in result_data.values(): - if value and value.get('job_id') not in job_id_list: - job_list.append({ - 'job_id': value.get('job_id'), - 'job_name': value.get('job_name'), - 'start_time': value.get('start_time'), - 'end_time': value.get('end_time'), - 'commit_id': value.get('commit_id'), - 'creator': value.get('creator'), - 'server': value.get('server'), - 'server_id': value.get('server_id'), - 'server_provider': provider_env, - 'value': value.get('value'), - 'cv_value': value.get('cv_value'), - 'note': value.get('note'), - 'result_obj_id': value.get('result_obj_id'), - 'creator_id': value.get('creator_id') - }) - job_id_list.append(value.get('job_id')) + package_job_info(_value, job_id_list, job_list) + else: + package_job_info(value, job_id_list, job_list) return list(sorted(job_list, key=lambda x: x.get('job_id'), reverse=True)) def get_suite_case_list(self, data): @@ -316,6 +245,48 @@ class PerfAnalysisService(CommonService): return sql +def package_job_info(job_value, job_id_list, job_list): + if job_value and job_value.get('job_id') not in job_id_list: + job_list.append({ + 'job_id': job_value.get('job_id'), + 'job_name': job_value.get('job_name'), + 'start_time': job_value.get('start_time'), + 'end_time': job_value.get('end_time'), + 'commit_id': job_value.get('commit_id'), + 'creator': job_value.get('creator'), + 'server': job_value.get('server'), + 'value': job_value.get('value'), + 'cv_value': job_value.get('cv_value'), + 'note': job_value.get('note'), + 'result_obj_id': job_value.get('result_obj_id'), + 'creator_id': job_value.get('creator_id') + }) + job_id_list.append(job_value.get('job_id')) + + +def _package_metric(row, column_ip): + server = get_job_case_run_server(row['job_case_id']) if row['run_mode'] == 'cluster' else row[column_ip] + return dict( + { + 'job_id': row['id'], + 'job_name': row['name'], + 'start_time': datetime.strftime(row['start_time'], "%Y-%m-%d %H:%M:%S") + if row['start_time'] else None, + 'end_time': datetime.strftime(row['end_time'], "%Y-%m-%d %H:%M:%S") + if row['end_time'] else None, + 'commit_id': json.loads(row['build_pkg_info']).get('commit_id', None), + 'creator': row['first_name'] or row['last_name'], + 'server': server, + 'value': row['test_value'], + 'cv_value': row['cv_value'], + 'note': row['note'], + 'result_obj_id': row['result_obj_id'], + 'creator_id': row['creator_id'] + } + ) + + + class FuncAnalysisService(CommonService): @staticmethod -- Gitee From 9fdd52067fa2c14436d67efdc37eaa3b8c742e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:07:33 +0800 Subject: [PATCH 06/30] refactor: simplify code for compare --- tone/services/job/result_compare_services.py | 64 +++++--------------- 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/tone/services/job/result_compare_services.py b/tone/services/job/result_compare_services.py index 374fb98..3f791f4 100644 --- a/tone/services/job/result_compare_services.py +++ b/tone/services/job/result_compare_services.py @@ -43,20 +43,16 @@ class CompareSuiteInfoService(CommonService): if not base_obj_li or len(base_obj_li) == 0: return is_baseline = data.get('is_baseline', 0) - base_id_list = ','.join(str(e) for e in base_obj_li) + base_id_list = tuple(base_obj_li) uniq_id = 'baseline_id' if is_baseline else 'job_id' if is_baseline: detail_table = 'func_baseline_detail' if is_func else 'perf_baseline_detail' - raw_sql = 'SELECT a.test_suite_id,a.baseline_id, b.name AS test_suite_name ' \ - 'FROM ' + detail_table + ' a LEFT JOIN test_suite b ON a.test_suite_id = b.id ' \ - 'WHERE a.baseline_id IN (' + \ - base_id_list + ')' else: - raw_sql = 'SELECT a.test_suite_id,a.job_id, b.name AS test_suite_name ' \ - 'FROM test_job_case a LEFT JOIN test_suite b ON a.test_suite_id = b.id ' \ - 'WHERE a.job_id IN (' + \ - base_id_list + ')' - all_test_suites = query_all_dict(raw_sql.replace('\'', ''), params=None) + detail_table = 'test_job_case' + raw_sql = 'SELECT a.test_suite_id,a.{}, b.name AS test_suite_name ' \ + 'FROM {} a LEFT JOIN test_suite b ON a.test_suite_id = b.id ' \ + 'WHERE a.{} IN %s '.format(uniq_id, detail_table, uniq_id) + all_test_suites = query_all_dict(raw_sql.replace('\'', ''), params=[base_id_list]) for test_suite in all_test_suites: if test_suite['test_suite_id'] not in data_dic: data_dic[test_suite['test_suite_id']] = { @@ -69,53 +65,25 @@ class CompareSuiteInfoService(CommonService): if test_suite[uniq_id] not in data_dic[test_suite['test_suite_id']]['test_job_id']: data_dic[test_suite['test_suite_id']]['test_job_id'].append(test_suite[uniq_id]) - def get_red_dot_count(self, func_suite_dic, perf_suite_dic, group_num): - count_li = [0 for _ in range(group_num)] - self.calc_red_dot(func_suite_dic, count_li) - self.calc_red_dot(perf_suite_dic, count_li) - return {'base_group': count_li[0], 'compare_groups': count_li[1:]} - - @staticmethod - def calc_red_dot(data, count_li): - for suite_key, suite_value in data.items(): - for conf_key, conf_value in suite_value.get('conf_dic').items(): - if len(conf_value.get('base_obj_li')) > 1: - count_li[0] += 1 - compare_groups = conf_value.get('compare_groups', list()) - for i in range(len(compare_groups)): - if len(compare_groups[i]) > 1: - count_li[i + 1] += 1 - - @staticmethod - def get_obj(obj_id, is_job): - obj_data = {'obj_id': obj_id, 'is_job': is_job} - if is_job: - test_job = TestJob.objects.get(id=obj_id) - name = test_job.name - obj_data['name'] = name - return obj_data - class CompareConfInfoService(CommonService): + def filter(self, data): suite_id = data.get('suite_id') test_job_list = data.get('test_job_id') + test_type = data.get('test_type') is_baseline = data.get('is_baseline', 0) - job_id_list = ','.join(str(e) for e in test_job_list) + job_id_list = tuple(test_job_list) test_case_data = dict() + uniq_id = 'baseline_id' if is_baseline else 'job_id' if is_baseline: - raw_sql = 'SELECT distinct a.test_case_id,b.name AS conf_name FROM func_baseline_detail a ' \ - 'LEFT JOIN test_case b ON a.test_case_id= b.id WHERE a.test_suite_id=%s AND ' \ - 'a.baseline_id IN (' + job_id_list + ') UNION ' \ - 'SELECT distinct a.test_case_id,b.name AS conf_name FROM perf_baseline_detail a ' \ - 'LEFT JOIN test_case b ON a.test_case_id= b.id WHERE a.test_suite_id=%s AND ' \ - 'a.baseline_id IN (' + job_id_list + ')' - all_test_cases = query_all_dict(raw_sql.replace('\'', ''), [suite_id, suite_id]) + detail_table = 'func_baseline_detail' if test_type == 'functional' else 'perf_baseline_detail' else: - raw_sql = 'SELECT distinct a.test_case_id,b.name AS conf_name FROM test_job_case a ' \ - 'LEFT JOIN test_case b ON a.test_case_id= b.id WHERE a.test_suite_id=%s AND' \ - ' a.job_id IN (' + job_id_list + ')' - all_test_cases = query_all_dict(raw_sql.replace('\'', ''), [suite_id]) + detail_table = 'test_job_case' + raw_sql = 'SELECT distinct a.test_case_id,b.name AS conf_name FROM {} a ' \ + 'LEFT JOIN test_case b ON a.test_case_id= b.id WHERE ' \ + 'a.test_suite_id=%s AND a.{} IN %s'.format(detail_table, uniq_id) + all_test_cases = query_all_dict(raw_sql.replace('\'', ''), [suite_id, job_id_list]) for job_case in all_test_cases: case_id = job_case['test_case_id'] test_case_data[case_id] = { -- Gitee From 25333e8e6b9c1322ea8938fe250b2f485c0ec517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:10:09 +0800 Subject: [PATCH 07/30] fix: set default page to str --- tone/services/job/test_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index de87516..67d97fc 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -95,8 +95,8 @@ class JobTestService(CommonService): def db_filter_job(self, data, operator): # noqa: C901 data = self.check_data_param(data) - page_num = int(data.get('page_num', 1)) - page_size = int(data.get('page_size', 20)) + page_num = data.get('page_num', '1') + page_size = data.get('page_size', '20') res = [] create_name_map = {} project_name_map = {} -- Gitee From 8f71531eed87c66fbd044dc0caed3750f6fb6500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:13:57 +0800 Subject: [PATCH 08/30] fix: set default page to str --- tone/services/job/test_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index 67d97fc..5838e76 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -96,7 +96,7 @@ class JobTestService(CommonService): def db_filter_job(self, data, operator): # noqa: C901 data = self.check_data_param(data) page_num = data.get('page_num', '1') - page_size = data.get('page_size', '20') + page_size = data.get('page_size', 20) res = [] create_name_map = {} project_name_map = {} -- Gitee From 17eadc31af5dab74f752016e27e6ac4f772f2bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:14:56 +0800 Subject: [PATCH 09/30] fix: update report desc and template --- tone/services/job/test_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index 5838e76..67d97fc 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -96,7 +96,7 @@ class JobTestService(CommonService): def db_filter_job(self, data, operator): # noqa: C901 data = self.check_data_param(data) page_num = data.get('page_num', '1') - page_size = data.get('page_size', 20) + page_size = data.get('page_size', '20') res = [] create_name_map = {} project_name_map = {} -- Gitee From 28c1a891eca0c8da3867250e2d5fcabedd2571ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:19:05 +0800 Subject: [PATCH 10/30] fix: query cloud server info by size --- tone/services/sys/server_services.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tone/services/sys/server_services.py b/tone/services/sys/server_services.py index d05988b..315d581 100644 --- a/tone/services/sys/server_services.py +++ b/tone/services/sys/server_services.py @@ -575,10 +575,16 @@ class CloudServerService(CommonService): is_instance = False q &= Q(is_instance=is_instance) if data.get('server_conf'): - if data.get('is_instance') == "true": - q &= Q(instance_name__icontains=data.get('server_conf')) + if data.get('query_size') and data.get('query_size') == '1': + if data.get('is_instance') == "true": + q &= Q(instance_name=data.get('server_conf')) + else: + q &= Q(template_name=data.get('server_conf')) else: - q &= Q(template_name__icontains=data.get('server_conf')) + if data.get('is_instance') == "true": + q &= Q(instance_name__icontains=data.get('server_conf')) + else: + q &= Q(template_name__icontains=data.get('server_conf')) if data.get('description'): q &= Q(description__icontains=data.get('description')) if data.get('ws_id'): -- Gitee From bedb23fc1823bdb37d19cbf7c31f5ee5dc852661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 11:25:13 +0800 Subject: [PATCH 11/30] fix: result analysis for private ip --- tone/services/job/result_analysis_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tone/services/job/result_analysis_services.py b/tone/services/job/result_analysis_services.py index fe9c161..980699e 100644 --- a/tone/services/job/result_analysis_services.py +++ b/tone/services/job/result_analysis_services.py @@ -131,7 +131,7 @@ class PerfAnalysisService(CommonService): if provider_env == 'aligroup': metric_obj = _package_metric(row, 'ip') else: - metric_obj = _package_metric(row, 'pub_ip') + metric_obj = _package_metric(row, 'private_ip') metric_obj.update({ 'instance_type': row['instance_type'], 'image': row['image'], -- Gitee From d1413c6c82a7ce35b529294fa4cbc5c2859be8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 18 Oct 2023 17:15:23 +0800 Subject: [PATCH 12/30] fix: page default to int for job result list --- tone/services/job/test_services.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index 67d97fc..05ef7ba 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -97,6 +97,10 @@ class JobTestService(CommonService): data = self.check_data_param(data) page_num = data.get('page_num', '1') page_size = data.get('page_size', '20') + if not page_num.isdigit() or not page_size.isdigit(): + raise ValueError(ErrorCode.ILLEGALITY_PARAM_ERROR) + page_num = int(page_num) + page_size = int(page_size) res = [] create_name_map = {} project_name_map = {} -- Gitee From abf070e92dc1cdbd02cde8e289aecf3206b6298c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Thu, 19 Oct 2023 17:11:26 +0800 Subject: [PATCH 13/30] fix: job_url return null for job download fail --- tone/services/job/test_services.py | 2 +- tone/views/job/test_view.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index 05ef7ba..0e083b4 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -574,7 +574,7 @@ class JobTestService(CommonService): self.del_dir(job_path) JobDownloadRecord.objects.filter(job_id=test_job_id).update(state='success', job_url=oss_link) else: - JobDownloadRecord.objects.filter(job_id=test_job_id).update(state='success', job_url='ftp upload fail.') + JobDownloadRecord.objects.filter(job_id=test_job_id).update(state='fail', job_url='ftp upload fail.') else: JobDownloadRecord.objects.filter(job_id=test_job_id).update(state='fail', job_url='job not exists') except Exception as e: diff --git a/tone/views/job/test_view.py b/tone/views/job/test_view.py index 7213f1e..160ea93 100644 --- a/tone/views/job/test_view.py +++ b/tone/views/job/test_view.py @@ -587,7 +587,10 @@ class JobDownloadRecordView(CommonAPIView): test_job_id = request.GET.get('job_id') record = JobDownloadRecord.objects.filter(job_id=test_job_id).first() if record: - response_data = self.get_response_only_for_data(record.to_dict()) + record_dict = record.to_dict().copy() + if record.state == 'fail': + record_dict['job_url'] = '' + response_data = self.get_response_only_for_data(record_dict) else: response_data = self.get_response_only_for_data(None) return Response(response_data) -- Gitee From 4fc382c797b8d75663a816d1fd328a741c148ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Thu, 19 Oct 2023 17:23:13 +0800 Subject: [PATCH 14/30] fix: add permission for pub api --- tone/views/api/query_job.py | 78 ++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/tone/views/api/query_job.py b/tone/views/api/query_job.py index 96a7a76..cedd8fd 100644 --- a/tone/views/api/query_job.py +++ b/tone/views/api/query_job.py @@ -21,7 +21,8 @@ from tone.core.common.job_result_helper import calc_job, get_job_case_server, ge splice_job_link from tone.services.sys.testcase_services import TestCaseInfoService from tone.services.job.test_services import package_server_list -from tone.core.utils.permission_manage import check_job_operator_permission, check_ws_operator_permission +from tone.core.utils.permission_manage import check_job_operator_permission, check_ws_operator_permission, \ + check_admin_operator_permission def _replace_statics_key(case_statics): @@ -176,6 +177,8 @@ def get_job_case(request): resp = CommResp() case_data_list = [] test_job = TestJob.objects.filter(id=request.GET.get('job_id')).first() + if not check_job_operator_permission(request.GET.get('username', None), test_job): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) if test_job and test_job.test_type == 'functional': queryset = TestCaseInfoService.filter(FuncResult.objects.all(), request.GET) test_case_list = TestCase.objects.filter(id__in=[case_info.test_case_id for case_info in queryset]).values_list( @@ -210,6 +213,12 @@ def get_job_case(request): @api_catch_error @token_required def get_job_type(request): + ws_id = request.GET.get('ws_id', None) + assert ws_id, ValueError(ErrorCode.WS_NEED) + test_type = request.GET.get('test_type', None) + assert test_type, ValueError(ErrorCode.TEST_TYPE_LACK) + if not check_ws_operator_permission(request.GET.get('username', None), ws_id): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() queryset = JobType.objects.filter(ws_id=request.GET.get('ws_id'), test_type=request.GET.get('test_type'), enable=True) @@ -221,6 +230,10 @@ def get_job_type(request): @api_catch_error @token_required def get_project(request): + ws_id = request.GET.get('ws_id', None) + assert ws_id, ValueError(ErrorCode.WS_NEED) + if not check_ws_operator_permission(request.GET.get('username', None), ws_id): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() queryset = Project.objects.filter(ws_id=request.GET.get('ws_id')) project_list = [{'id': project.id, 'name': project.name, 'is_default': project.is_default} for project in queryset] @@ -231,6 +244,8 @@ def get_project(request): @api_catch_error @token_required def get_workspace(request): + if not check_admin_operator_permission(request.GET.get('username', None)): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() queryset = Workspace.objects.all() ws_list = [{'id': ws.id, 'name': ws.name} for ws in queryset] @@ -279,40 +294,31 @@ def get_job_info_list(request): resp = CommResp() data = json.loads(request.body) product_name = data.get('product', None) - if product_name: - assert isinstance(product_name, str), ValueError(ErrorCode.ILLEGALITY_PARAM_ERROR) project_name = data.get('project', None) - if project_name: - assert isinstance(project_name, str), ValueError(ErrorCode.ILLEGALITY_PARAM_ERROR) tag_name = data.get('tag', None) - if tag_name: - assert isinstance(tag_name, str), ValueError(ErrorCode.ILLEGALITY_PARAM_ERROR) ws_id = data.get('ws_id', None) assert ws_id, ValueError(ErrorCode.WS_NEED) - assert isinstance(ws_id, str), ValueError(ErrorCode.ILLEGALITY_PARAM_ERROR) - raw_sql = "SELECT DISTINCT A.id, A.name AS job_name,A.state,A.start_time,A.end_time,A.test_type,A.test_result," \ - "B.name AS product_name,C.name AS project_name,F.username AS creator FROM " \ - "test_job A, product B, project C, job_tag D, job_tag_relation E, user F " \ - "WHERE A.is_deleted=0 AND B.is_deleted=0 AND C.is_deleted=0 AND D.is_deleted=0 AND " \ - "A.product_id=B.id AND A.project_id=C.id AND A.id=E.job_id AND E.tag_id=D.id AND" \ - " A.creator=F.id AND A.ws_id='" + ws_id + "'" - if not tag_name: - raw_sql = "SELECT DISTINCT A.id, A.name AS job_name,A.state,A.start_time,A.end_time,A.test_type," \ - "A.test_result, B.name AS product_name,C.name AS project_name,F.username AS creator FROM " \ - "test_job A, product B, project C, user F " \ - "WHERE A.is_deleted=0 AND B.is_deleted=0 AND C.is_deleted=0 AND " \ - "A.product_id=B.id AND A.project_id=C.id AND " \ - " A.creator=F.id AND A.ws_id='" + ws_id + "'" + if not check_ws_operator_permission(data.get('username'), ws_id): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) + params = [ws_id] + raw_sql = "SELECT DISTINCT A.id, A.name AS job_name,A.state,A.start_time,A.end_time,A.test_type," \ + "A.test_result, B.name AS product_name,C.name AS project_name,A.project_id," \ + "F.username AS creator FROM " \ + "test_job A, product B, project C, user F " \ + "WHERE A.is_deleted=0 AND B.is_deleted=0 AND C.is_deleted=0 AND " \ + "A.product_id=B.id AND A.project_id=C.id AND " \ + " A.creator=F.id AND A.ws_id=%s" if product_name: - raw_sql += " AND B.name='{}'".format(product_name) + raw_sql += " AND B.name=%s " + params.append(product_name) if project_name: - raw_sql += " AND C.name='{}'".format(project_name) - if tag_name: - tag_list = ','.join('\'' + e + '\'' for e in tag_name.split(',')) - raw_sql += " AND D.name IN ({})".format(tag_list) - job_result_list = query_all_dict(raw_sql, params=None) + raw_sql += " AND C.name=%s " + params.append(project_name) + job_result_list = query_all_dict(raw_sql, params=params) res_list = list() + job_id_list = list() for job_info in job_result_list: + job_id_list.append(job_info['id']) job_dict = dict( job_id=job_info['id'], job_name=job_info['job_name'], @@ -323,8 +329,24 @@ def get_job_info_list(request): test_result=job_info['test_result'], product_name=job_info['product_name'], project_name=job_info['project_name'], + project_id=job_info['project_id'], creator=job_info['creator'] ) res_list.append(job_dict) - resp.data = res_list + tag_sql = "SELECT a.id,a.name,c.id AS tag_id,C.name AS tag_name FROM test_job a LEFT JOIN job_tag_relation b " \ + "ON a.id=b.job_id LEFT JOIN job_tag c ON b.tag_id=c.id where A.is_deleted=0 AND B.is_deleted=0 " \ + "AND C.is_deleted=0 AND a.id IN %s" + tags_result = query_all_dict(tag_sql, params=[tuple(job_id_list)]) + job_filter_tag_list = list() + for res_job_info in res_list: + tags = [tag for tag in tags_result if tag['id'] == res_job_info['job_id']] + tag_info = dict() + tag_name_list = list() + for job_tag in tags: + tag_info[job_tag['tag_id']] = job_tag['tag_name'] + tag_name_list.append(job_tag['tag_name']) + if tag_name and set(tag_name.split(',')).issubset(set(tag_name_list)): + job_filter_tag_list.append(res_job_info) + res_job_info['tags'] = tag_info + resp.data = job_filter_tag_list if tag_name else res_list return resp.json_resp() -- Gitee From 73da8ccb2613d15a9e1a196bdc7caf04e7f27c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Fri, 20 Oct 2023 15:01:16 +0800 Subject: [PATCH 15/30] fix: pub ip for job download url --- tone/services/job/test_services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index 0e083b4..0a88635 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -18,6 +18,7 @@ from django.db.models import Q from django.db import transaction, connection from itertools import chain from threading import Thread +from tone import settings from tone.core.common.callback import CallBackType, JobCallBack from tone.core.utils.common_utils import query_all_dict @@ -570,7 +571,7 @@ class JobTestService(CommonService): ftp_path = oss_file.replace(MEDIA_ROOT.strip('/'), '') res = sftp_client.upload_file(target_file, ftp_path) if res: - oss_link = 'http://' + sftp_client.host + ':' + str(sftp_client.proxy_port) + ftp_path + oss_link = f'http://{settings.TONE_STORAGE_DOMAIN}:' + str(sftp_client.proxy_port) + ftp_path self.del_dir(job_path) JobDownloadRecord.objects.filter(job_id=test_job_id).update(state='success', job_url=oss_link) else: -- Gitee From 8b20f11fe9cf9cc039c06b4aca18aad4ae06fb46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Fri, 20 Oct 2023 15:44:06 +0800 Subject: [PATCH 16/30] fix: add permission for suite pub api --- tone/views/api/get_suite_info.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tone/views/api/get_suite_info.py b/tone/views/api/get_suite_info.py index 586e597..bc31e38 100644 --- a/tone/views/api/get_suite_info.py +++ b/tone/views/api/get_suite_info.py @@ -6,11 +6,15 @@ from tone.core.common.verify_token import token_required from tone.core.common.expection_handler.error_catch import api_catch_error from datetime import datetime from tone.core.handle.report_handle import ReportHandle +from tone.core.utils.permission_manage import check_admin_operator_permission +from tone.core.common.expection_handler.error_code import ErrorCode @api_catch_error @token_required def get_suite_list(request): + if not check_admin_operator_permission(request.GET.get('username', None)): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() serialize_flag, queryset = WorkspaceRetrieveService.filter(WorkspaceCaseRelation.objects.all(), request.GET) resp.data = queryset @@ -20,6 +24,8 @@ def get_suite_list(request): @api_catch_error @token_required def get_case_list(request): + if not check_admin_operator_permission(request.GET.get('username', None)): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() queryset = TestCaseService.filter(TestCase.objects.all(), request.GET) case_data_list = [{'id': case_info.id, 'name': case_info.name, @@ -32,6 +38,8 @@ def get_case_list(request): @api_catch_error @token_required def get_metric_list(request): + if not check_admin_operator_permission(request.GET.get('username', None)): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() queryset = TestMetricService.filter(TestMetric.objects.all(), request.GET) metric_data_list = [{'id': metric.id, 'name': metric.name} for metric in queryset] @@ -39,9 +47,11 @@ def get_metric_list(request): return resp.json_resp() -# @api_catch_error -# @token_required +@api_catch_error +@token_required def get_suite_increase(request): + if not check_admin_operator_permission(request.GET.get('username', None)): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() last_sync_time = request.GET['last_sync_time'] queryset = TestCase.objects.filter( -- Gitee From 43fc508f50c0658f6c5d0aee82097e7a91b1cabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Fri, 20 Oct 2023 15:48:19 +0800 Subject: [PATCH 17/30] fix: add permission for create job pub api --- tone/views/api/create_job.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tone/views/api/create_job.py b/tone/views/api/create_job.py index 1965794..9e1cf1e 100644 --- a/tone/views/api/create_job.py +++ b/tone/views/api/create_job.py @@ -16,7 +16,7 @@ from tone.core.handle.job_handle import JobDataHandle from tone.core.common.verify_token import token_required from tone.core.common.expection_handler.error_code import ErrorCode from tone.core.common.expection_handler.error_catch import api_catch_error -from tone.serializers.job.test_serializers import JobTestSerializer, JobSerializerForAPI +from tone.core.utils.permission_manage import check_ws_operator_permission @api_catch_error @@ -57,6 +57,8 @@ def conversion_data(data): # noqa: C901 ws_name = data.get('workspace') assert ws_name, ValueError(ErrorCode.WS_NAME_NEED) ws_id = get_ws(ws_name) + if not check_ws_operator_permission(data.get('username'), ws_id): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) if data.get('template') or data.get('template_id'): data['data_from'] = 'template' template = TestTemplate.objects.get(id=data.get('template_id')) if data.get('template_id') else get_template( @@ -218,8 +220,13 @@ def get_ws(ws_name): def get_server_list(request): data = request.GET ws_id = data.get('ws_id') + assert ws_id, ValueError(ErrorCode.WS_NEED) provider = data.get('provider') + assert provider, ValueError(ErrorCode.PROVIDER_NEED) run_mode = data.get('run_mode') + assert run_mode, ValueError(ErrorCode.PARAMS_ERROR) + if not check_ws_operator_permission(request.GET.get('username', None), ws_id): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() if provider == 'aligroup' and run_mode == 'standalone': queryset = TestServer.objects.exclude(ip='').\ @@ -232,11 +239,16 @@ def get_server_list(request): values_list('instance_id', flat=True).distinct() resp.data = list(queryset) return resp.json_resp() + + @token_required @api_catch_error def get_server_tag_list(request): data = request.GET ws_id = data.get('ws_id') + assert ws_id, ValueError(ErrorCode.WS_NEED) + if not check_ws_operator_permission(request.GET.get('username', None), ws_id): + assert None, ValueError(ErrorCode.PERMISSION_ERROR) resp = CommResp() queryset = ServerTag.objects.filter(ws_id=ws_id).values_list('name', flat=True) resp.data = list(queryset) -- Gitee From 449563a3b811805c1e6fc92c05c5c783512cee99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Fri, 20 Oct 2023 18:00:42 +0800 Subject: [PATCH 18/30] fix: baseline download and offline file download url to pub domain --- tone/services/job/offline_data_services.py | 2 +- tone/services/sys/baseline_services.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tone/services/job/offline_data_services.py b/tone/services/job/offline_data_services.py index a00b685..f4ee779 100644 --- a/tone/services/job/offline_data_services.py +++ b/tone/services/job/offline_data_services.py @@ -676,7 +676,7 @@ class OfflineDataUploadService(object): # os.remove(local_dir) if ret: file_link = 'http://{}:{}{}/{}'.format( - settings.TONE_STORAGE_HOST, + settings.TONE_STORAGE_DOMAIN, settings.TONE_STORAGE_PROXY_PORT, OFFLINE_DATA_DIR, file_name diff --git a/tone/services/sys/baseline_services.py b/tone/services/sys/baseline_services.py index ebdd599..e64d0ad 100644 --- a/tone/services/sys/baseline_services.py +++ b/tone/services/sys/baseline_services.py @@ -14,6 +14,7 @@ import stat import uuid from django.db import transaction from django_q.tasks import async_task +from tone import settings from tone.core.common.constant import OFFLINE_DATA_DIR from tone.settings import MEDIA_ROOT @@ -245,7 +246,7 @@ class BaselineService(CommonService): ftp_path = oss_file.replace(MEDIA_ROOT.strip('/'), '') res = sftp_client.upload_file(target_file, ftp_path) if res: - oss_link = 'http://' + sftp_client.host + ':' + str(sftp_client.proxy_port) + ftp_path + oss_link = f'http://{settings.TONE_STORAGE_DOMAIN}:' + str(sftp_client.proxy_port) + ftp_path self.del_dir(file_path) BaselineDownloadRecord.objects.filter(baseline_id=baseline_id).\ update(state='success', target_url=oss_link) -- Gitee From 5e5c517e8b181d3fad87e307a4bcd6151e5585bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Mon, 23 Oct 2023 17:31:02 +0800 Subject: [PATCH 19/30] fix: ignore na for conf name --- tone/services/sys/testcase_services.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tone/services/sys/testcase_services.py b/tone/services/sys/testcase_services.py index efada7e..93c2a0c 100644 --- a/tone/services/sys/testcase_services.py +++ b/tone/services/sys/testcase_services.py @@ -578,7 +578,8 @@ class TestSuiteService(CommonService): continue case_name = '' for i, item in enumerate(line.split()): - case_name += f'{title_list[i]}={item},' + if item.lower() != 'na': + case_name += f'{title_list[i]}={item},' case_list.append(case_name.strip(',')) if not case_list: case_list = ['default'] @@ -633,7 +634,8 @@ class TestSuiteService(CommonService): continue case_name = '' for i, item in enumerate(line.split()): - case_name += f'{title_list[i]}={item},' + if item.lower() != 'na': + case_name += f'{title_list[i]}={item},' case_list.append(case_name.strip(',')) if not case_list: case_list = ['default'] -- Gitee From 9346589d3671e4d446d122b43b245dbce4b54c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 24 Oct 2023 09:32:33 +0800 Subject: [PATCH 20/30] fix: analysis suite list add params project_id --- tone/services/job/result_analysis_services.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tone/services/job/result_analysis_services.py b/tone/services/job/result_analysis_services.py index 980699e..20a6ecb 100644 --- a/tone/services/job/result_analysis_services.py +++ b/tone/services/job/result_analysis_services.py @@ -177,12 +177,14 @@ class PerfAnalysisService(CommonService): ws_id = data.get('ws_id') test_type = data.get('test_type') provider_env = data.get('provider_env', 'aligroup') + project_id = data.get('project_id') assert test_type, AnalysisException(ErrorCode.TEST_TYPE_LACK) assert ws_id, AnalysisException(ErrorCode.WS_NEED) + assert project_id, AnalysisException(ErrorCode.PROJECT_ID_NEED) raw_sql = self.get_suite_list_sql(test_type) - params = [provider_env, ws_id] + params = [provider_env, project_id, ws_id] if test_type == 'functional': - params = [ws_id] + params = [project_id, ws_id] suite_res_list = query_all_dict(raw_sql, params=params) suite_case_list = sorted(suite_res_list, key=lambda x: x['test_suite_id'], reverse=True) suite_list = list() -- Gitee From 2e81b219eeba1cb9a7aab8c2ec8106b0f19208dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 25 Oct 2023 10:15:38 +0800 Subject: [PATCH 21/30] fix: remove venv for schedule --- docker/supervisord.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 5cb41ae..def0afe 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -40,13 +40,12 @@ startsecs=60 [program:django_q_schedule] directory=%(ENV_APP_HOME)s -command=python manage.py qcluster +command=python3 manage.py qcluster autostart=true autorestart=true redirect_stderr=true stopasgroup=true stdout_logfile=%(ENV_APP_HOME)s/logs/app/django_q_schedule.log -environment=PATH="/home/admin/tone/virtualenv/bin" stopwaitsecs=1800 stopsignal=TERM startsecs=60 \ No newline at end of file -- Gitee From c02268e7b65c99032748cc8f41d73b23d1d0068c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 25 Oct 2023 10:50:30 +0800 Subject: [PATCH 22/30] fix: sftp host to domain for init data --- initial/base_config/data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/initial/base_config/data.py b/initial/base_config/data.py index 941d0ee..80b7489 100644 --- a/initial/base_config/data.py +++ b/initial/base_config/data.py @@ -94,7 +94,7 @@ BASE_CONFIG_DATA = [ 'config_type': 'script', 'config_key': 'RUN_TEST', 'config_value': RUN_TEST.format( - tone_storage_host=cp.get('tone_storage_host'), + tone_storage_host=cp.get('tone_storage_domain'), tone_storage_sftp_port=cp.get('tone_storage_sftp_port'), tone_storage_proxy_port=cp.get('tone_storage_proxy_port'), tone_storage_user=cp.get('tone_storage_user'), @@ -107,7 +107,7 @@ BASE_CONFIG_DATA = [ 'config_type': 'script', 'config_key': 'UPLOAD', 'config_value': UPLOAD.format( - tone_storage_host=cp.get('tone_storage_host'), + tone_storage_host=cp.get('tone_storage_domain'), tone_storage_sftp_port=cp.get('tone_storage_sftp_port'), tone_storage_proxy_port=cp.get('tone_storage_proxy_port'), tone_storage_user=cp.get('tone_storage_user'), @@ -162,7 +162,7 @@ BASE_CONFIG_DATA = [ 'config_type': 'script', 'config_key': 'RUN_TEST_DEBIAN', 'config_value': RUN_TEST_DEBIAN.format( - tone_storage_host=cp.get('tone_storage_host'), + tone_storage_host=cp.get('tone_storage_domain'), tone_storage_sftp_port=cp.get('tone_storage_sftp_port'), tone_storage_proxy_port=cp.get('tone_storage_proxy_port'), tone_storage_user=cp.get('tone_storage_user'), @@ -175,7 +175,7 @@ BASE_CONFIG_DATA = [ 'config_type': 'script', 'config_key': 'UPLOAD_FILE', 'config_value': UPLOAD_FILE.format( - tone_storage_host=cp.get('tone_storage_host'), + tone_storage_host=cp.get('tone_storage_domain'), tone_storage_sftp_port=cp.get('tone_storage_sftp_port'), tone_storage_proxy_port=cp.get('tone_storage_proxy_port'), tone_storage_user=cp.get('tone_storage_user'), -- Gitee From 74f9ad842fe9d64e8cce0b3d50d8de47a29a3759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 25 Oct 2023 15:16:30 +0800 Subject: [PATCH 23/30] fix: schedule func for auto_job_report --- initial/schedule/initialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initial/schedule/initialize.py b/initial/schedule/initialize.py index 12deb08..88bb285 100644 --- a/initial/schedule/initialize.py +++ b/initial/schedule/initialize.py @@ -22,7 +22,7 @@ def init_schedule(): ) Schedule.objects.create( name='auto_job_report', - func='from tone.core.schedule.schedule_job.auto_job_report', + func='tone.core.schedule.schedule_job.auto_job_report', schedule_type=Schedule.MINUTES, minutes=5 ) -- Gitee From 51dbfe048e589cee7f469f235e2c54d65be7b8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Wed, 25 Oct 2023 15:21:53 +0800 Subject: [PATCH 24/30] fix: job summary for ding talk --- tone/core/common/msg_notice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tone/core/common/msg_notice.py b/tone/core/common/msg_notice.py index 332948e..1d215f2 100644 --- a/tone/core/common/msg_notice.py +++ b/tone/core/common/msg_notice.py @@ -896,7 +896,7 @@ class OutSiteMsgHandle(object): job_summary.get('warn')) else: content += '\nStatistics：上升:{};下降:{};正常:{};无效:{};NA:{}'. \ - format(job_summary.get('increase'), job_summary.get('decline'), job_summary.get('invalid'), + format(job_summary.get('increase'), job_summary.get('decline'), job_summary.get('normal'), job_summary.get('invalid'), job_summary.get('na')) OutSiteMsg.objects.create( subject=subject if not ding_subject else ding_subject, -- Gitee From 5e21bb694616e8431d9457667b61ec622317a5ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Thu, 26 Oct 2023 10:21:09 +0800 Subject: [PATCH 25/30] fix: check email send to none --- tone/services/notice/core/notice_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tone/services/notice/core/notice_service.py b/tone/services/notice/core/notice_service.py index ce9ef08..b3a05a7 100644 --- a/tone/services/notice/core/notice_service.py +++ b/tone/services/notice/core/notice_service.py @@ -41,6 +41,8 @@ def send_ding_message(message_id): # @app.task def send_email_message(message_id): message = OutSiteMsg.objects.get(id=message_id) + if len(message.send_to.strip()) <= 1: + return send_to = message.send_to.strip().split(',') cc_to = message.cc_to.strip().split(',') bcc_to = message.bcc_to.strip().split(',') -- Gitee From c150990f8175c4a9adc4e2918f3984872e82e81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Thu, 26 Oct 2023 10:30:39 +0800 Subject: [PATCH 26/30] fix: not create msg info if check notice info none --- tone/core/common/msg_notice.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tone/core/common/msg_notice.py b/tone/core/common/msg_notice.py index 332948e..7ef388b 100644 --- a/tone/core/common/msg_notice.py +++ b/tone/core/common/msg_notice.py @@ -522,6 +522,8 @@ class OutSiteMsgHandle(object): test_obj = obj_model.objects.filter(id=job_id).first() if test_obj is None: return config_dic + if not test_obj.notice_info or test_obj.notice_info == '[]': + return config_dic for config in test_obj.notice_info: if config['type'] == 'email': config_dic['email'] = config -- Gitee From 52bd968a52919c87cb8a89fd4c0d108d2087941a Mon Sep 17 00:00:00 2001 From: k4xxx Date: Thu, 26 Oct 2023 16:10:48 +0800 Subject: [PATCH 27/30] add auto-confirm param to lftp --- docker/Dockerfile | 3 ++- initial/base_config/script/run_test.py | 2 +- initial/base_config/script/upload.py | 4 ++-- initial/base_config/script/upload_file.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e2e8a04..5c7bf6d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,8 @@ RUN mkdir -p ${APP_HOME}/logs/app && mkdir -p ${APP_HOME}/logs/supervisord ADD . ${APP_HOME} # 更新apt-get、 pip、安装pip包 -RUN ${PIP_INSTALL} supervisor && \ +RUN apt-get install git && \ + ${PIP_INSTALL} supervisor && \ ${PIP_INSTALL} -r ${APP_HOME}/docker/requirements.txt #设置时区 diff --git a/initial/base_config/script/run_test.py b/initial/base_config/script/run_test.py index b8f2631..65a5571 100644 --- a/initial/base_config/script/run_test.py +++ b/initial/base_config/script/run_test.py @@ -25,7 +25,7 @@ TONE_STORAGE_PROXY_PORT={tone_storage_proxy_port} TONE_STORAGE_USER={tone_storage_user} TONE_STORAGE_PASSWORD={tone_storage_password} TONE_STORAGE_BUCKET=${{7:-results}} -SFTP_PARAMS="set ftp:ssl-allow no; set net:timeoit 60; set net:reconnect-interval-base 10; set net:max-retries 2" +SFTP_PARAMS="set sftp:auto-confirm yes; set net:timeout 60; set net:reconnect-interval-base 10; set net:max-retries 2" TONE_RESULT_PATH=$TONE_PATH/result/$TEST_SUITE/$CONF_SHORT_NAME TONE_RESULT_PATH_LEN=$((${{#TONE_RESULT_PATH}}+1)) diff --git a/initial/base_config/script/upload.py b/initial/base_config/script/upload.py index 7af53bb..71867e4 100644 --- a/initial/base_config/script/upload.py +++ b/initial/base_config/script/upload.py @@ -24,7 +24,7 @@ LOG=/tmp/tone_${{TEST_SUITE}}_${{CONF_SHORT_NAME}}.log ALL_LOG=/tmp/tone.log TONE_RESULT_PATH=$TONE_PATH/result/$TEST_SUITE/$CONF_SHORT_NAME TONE_RESULT_PATH_LEN=$((${{#TONE_RESULT_PATH}}+1)) - +SFTP_PARAMS="set sftp:auto-confirm yes; set net:timeout 60; set net:reconnect-interval-base 10; set net:max-retries 2" install_utils() {{ @@ -56,7 +56,7 @@ function upload_file(){{ return fi - lftp -u ${{TONE_STORAGE_USER}},${{TONE_STORAGE_PASSWORD}} sftp://${{TONE_STORAGE_HOST}}:${{TONE_STORAGE_SFTP_PORT}} >> $ALL_LOG 2>&1 <> $ALL_LOG 2>&1 < Date: Fri, 3 Nov 2023 14:30:21 +0800 Subject: [PATCH 28/30] fix: test --- tone/views/api/handle_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tone/views/api/handle_job.py b/tone/views/api/handle_job.py index 95ac12b..a3e7e59 100644 --- a/tone/views/api/handle_job.py +++ b/tone/views/api/handle_job.py @@ -28,4 +28,5 @@ def stop_job(request): state__in=['running', 'pending', 'pending_q'] ).update(state='stop', end_time=datetime.now()) operation_after_stop_job(job_obj, 'The job stop by ' + request.GET.get('username', None)) + # v2需单独调用额外接口 return resp.json_resp() -- Gitee From d38aa7943361715b86d9a992213bf03cd24b0f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=87=AF?= Date: Fri, 3 Nov 2023 14:40:43 +0800 Subject: [PATCH 29/30] fix: test --- tone/views/api/handle_job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tone/views/api/handle_job.py b/tone/views/api/handle_job.py index a3e7e59..95ac12b 100644 --- a/tone/views/api/handle_job.py +++ b/tone/views/api/handle_job.py @@ -28,5 +28,4 @@ def stop_job(request): state__in=['running', 'pending', 'pending_q'] ).update(state='stop', end_time=datetime.now()) operation_after_stop_job(job_obj, 'The job stop by ' + request.GET.get('username', None)) - # v2需单独调用额外接口 return resp.json_resp() -- Gitee From 93045f05b970f5887dafc670fa27a4bc1d0c9fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=87=AF?= Date: Fri, 3 Nov 2023 14:43:34 +0800 Subject: [PATCH 30/30] fix: test --- tone/views/api/handle_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tone/views/api/handle_job.py b/tone/views/api/handle_job.py index 95ac12b..a78cf0a 100644 --- a/tone/views/api/handle_job.py +++ b/tone/views/api/handle_job.py @@ -27,5 +27,6 @@ def stop_job(request): id=job_id, state__in=['running', 'pending', 'pending_q'] ).update(state='stop', end_time=datetime.now()) + operation_after_stop_job(job_obj, 'The job stop by ' + request.GET.get('username', None)) return resp.json_resp() -- Gitee