From fbdd5cce50160e1ef58bb6af5d15cf134fc11799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Thu, 8 Aug 2024 13:25:12 +0800 Subject: [PATCH 01/10] fix: agent connect error msg --- tone/services/sys/server_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tone/services/sys/server_services.py b/tone/services/sys/server_services.py index 398fdf8..062689f 100644 --- a/tone/services/sys/server_services.py +++ b/tone/services/sys/server_services.py @@ -164,7 +164,7 @@ class TestServerService(CommonService): # 向tone-agent平台注册机器 add_agent_result = add_server_to_toneagent(ip) if not add_agent_result.get("SUCCESS"): - return False, add_agent_result.get('RESULT') or add_agent_result.get('msg') + return False, add_agent_result.get('RESULT') or add_agent_result.get('ERROR_MSG') tsn = add_agent_result['RESULT']['TSN'] test_server = TestServer.objects.create( ip=ip, -- Gitee From a51146dd3430ba9298349b059320654c3e902291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 27 Aug 2024 15:40:01 +0800 Subject: [PATCH 02/10] refactor: job filter by ip --- tone/services/job/test_services.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tone/services/job/test_services.py b/tone/services/job/test_services.py index 4ed61e0..6d207f7 100644 --- a/tone/services/job/test_services.py +++ b/tone/services/job/test_services.py @@ -466,9 +466,11 @@ class JobTestService(CommonService): q &= Q(creator=operator.id) if data.get('server'): server = data.get('server') - server_objs = TestServerSnapshot.objects.filter(Q(ip=server) | Q(sn=server)) - cloud_server_objs = CloudServerSnapshot.objects.filter(Q(pub_ip=server) | Q(sn=server)) - id_li = list(set([obj.job_id for obj in server_objs]) | set([obj.job_id for obj in cloud_server_objs])) + server_objs = TestServerSnapshot.objects.filter(Q(ip=server) | Q(sn=server)).\ + values_list('job_id', flat=True) + cloud_server_objs = CloudServerSnapshot.objects.filter(Q(pub_ip=server) | Q(sn=server)).\ + values_list('job_id', flat=True) + id_li = list(set(server_objs) | set(cloud_server_objs)) q &= Q(id__in=id_li) if data.get('collection') or data.get('tab') == 'collection': my_collection = JobCollection.objects.filter(user_id=operator.id) -- Gitee From 8b287c982e304c504fd48f6c85ce6da5e52aa6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 09:43:07 +0800 Subject: [PATCH 03/10] fix: sql for get domain group --- tone/views/api/get_domain_group.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tone/views/api/get_domain_group.py b/tone/views/api/get_domain_group.py index b9c25ee..7c5c83f 100644 --- a/tone/views/api/get_domain_group.py +++ b/tone/views/api/get_domain_group.py @@ -60,13 +60,17 @@ def get_domain_group_v1(suite_list): suite_id_list.append(suite.get('suite_id')) else: conf_id_list.extend(suite.get('conf_list')) + params = [] if len(suite_id_list) > 0 and len(conf_id_list) > 0: - sql_filter += ' a.test_suite_id in (' + ','.join(str(e) for e in suite_id_list) + ')' - sql_filter += ' or a.id in (' + ','.join(str(e) for e in conf_id_list) + ')' + sql_filter += ' a.test_suite_id in %s' + sql_filter += ' or a.id in %s' + params = [tuple(suite_id_list), tuple(conf_id_list)] elif len(suite_id_list) > 0: - sql_filter += ' a.test_suite_id in (' + ','.join(str(e) for e in suite_id_list) + ')' + sql_filter += ' a.test_suite_id in %s' + params = [tuple(suite_id_list)] elif len(conf_id_list) > 0: - sql_filter += ' a.id in (' + ','.join(str(e) for e in conf_id_list) + ')' + sql_filter += ' a.id in %s' + params = [tuple(conf_id_list)] if len(suite_id_list) == 0 and len(conf_id_list) == 0: test_case_list = list() else: @@ -80,7 +84,7 @@ def get_domain_group_v1(suite_list): 'ON a.test_suite_id=b.id WHERE a.id NOT IN (SELECT object_id from domain_relation WHERE ' \ 'object_type="case" AND is_deleted=0) AND ' \ '(' + sql_filter + ')' - test_case_list = query_all_dict(raw_sql) + test_case_list = query_all_dict(raw_sql, params=params + params) for test_case in test_case_list: insert_conf(test_case.get('domain'), res_data, test_case.get('test_type'), test_case.get('test_suite_id'), test_case.get('id')) -- Gitee From 9b4f43ab841e7df298dda6fabfb0cf8905294ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 09:56:21 +0800 Subject: [PATCH 04/10] fix: sql for ws chart --- tone/services/sys/dashboard_services.py | 137 +++++++++++++++++------- 1 file changed, 96 insertions(+), 41 deletions(-) diff --git a/tone/services/sys/dashboard_services.py b/tone/services/sys/dashboard_services.py index 87ee040..7b646bd 100644 --- a/tone/services/sys/dashboard_services.py +++ b/tone/services/sys/dashboard_services.py @@ -5,6 +5,7 @@ from django.db import connection, transaction from tone.core.common.redis_cache import redis_cache from tone.core.common.services import CommonService +from tone.core.utils.common_utils import execute_sql from tone.core.utils.tone_thread import ToneThread from tone.models import TestJob, TestJobCase, FuncResult, PerfResult, \ Workspace, Product, Project, TestSuite, TestCase, Baseline, FuncBaselineDetail, PerfBaselineDetail, TestMetric, \ @@ -556,8 +557,12 @@ class DashboardService(CommonService): def get_ws_chart_data(self, data): project_id = data.get('project_id') + if not (project_id and isinstance(project_id, int) or project_id.isdigit()): + return False, ErrorCode.PARAMS_ERROR.to_api ws_id = data.get('ws_id') - tmp_time = '1970-1-1' + if not Workspace.objects.filter(id=ws_id).exists(): + return False, ErrorCode.WS_NOT_EXISTS.to_api + start_time = data.get('start_time') end_time = data.get('end_time') chart_type = data.get('chart_type', 'job') @@ -579,49 +584,99 @@ class DashboardService(CommonService): date_job = {datetime.strftime(date, '%Y-%m-%d'): [[], 0, 0] for date in date_list[:-1]} start_time = date_list[0] end_time = date_list[-1] - with connection.cursor() as cursor: - query_project_job = """ - SELECT - DATE_FORMAT(gmt_created, "%Y-%m-%d"), id, state - FROM test_job - WHERE is_deleted=0 AND project_id={} AND ws_id="{}" and gmt_created BETWEEN "{}" AND "{}" - """.format(project_id, ws_id, start_time, end_time) - cursor.execute(query_project_job) - rows = cursor.fetchall() - for row_data in rows: - tmp_date = row_data[0] - tmp_job_id = row_data[1] - tmp_state = row_data[2] - if tmp_date in date_job: - date_job[tmp_date][0].append(tmp_job_id) - if tmp_state == 'success': - date_job[tmp_date][1] += 1 - if tmp_state == 'fail': - date_job[tmp_date][2] += 1 + query_project_job = """ + SELECT + gmt_created, id, state + FROM test_job + WHERE is_deleted=0 AND project_id=%s AND ws_id=%s and gmt_created BETWEEN %s AND %s + """ + rows = execute_sql(query_project_job, [project_id, ws_id, start_time, end_time]) + for row_data in rows: + tmp_date = datetime.strftime(row_data[0], '%Y-%m-%d') + tmp_job_id = row_data[1] + tmp_state = row_data[2] + if tmp_date in date_job: + date_job[tmp_date][0].append(tmp_job_id) + if tmp_state == 'success': + date_job[tmp_date][1] += 1 + if tmp_state == 'fail': + date_job[tmp_date][2] += 1 + thread_tasks = [] for temp_date, tmp_data in date_job.items(): - job_id_list = tmp_data[0] - complete_num = tmp_data[1] - fail_num = tmp_data[2] - if chart_type == 'job': - ws_chart_data.append({ - 'date': temp_date, - 'CompleteJob': complete_num, - 'FailJob': fail_num, - }) - else: - suite_fail = TestJobSuite.objects.filter(job_id__in=job_id_list, state='fail').count() - case_fail = TestJobCase.objects.filter(job_id__in=job_id_list, state='fail').count() - func_res_sub_case_fail = FuncResult.objects.filter(test_job_id__in=job_id_list, sub_case_result=2) - func_res_sub_case_fail_count = func_res_sub_case_fail.count() - fail_sub_case_suite = len(set(func_res_sub_case_fail.values_list('test_suite_id', 'test_job_id'))) - perf_decline = PerfResult.objects.filter(test_job_id__in=job_id_list, track_result='decline').count() - ws_chart_data.append({ - 'date': temp_date, - 'FailSuite': suite_fail + fail_sub_case_suite, - 'FailCase': case_fail + func_res_sub_case_fail_count + perf_decline, - }) + thread_tasks.append( + ToneThread(self.get_fail_count, (chart_type, temp_date, tmp_data)) + ) + thread_tasks[-1].start() + for thread_task in thread_tasks: + thread_task.join() + count_data = thread_task.get_result() + if count_data: + ws_chart_data.append(count_data) return True, ws_chart_data + def get_fail_count(self, chart_type, temp_date, tmp_data): + job_id_list = tmp_data[0] + complete_num = tmp_data[1] + fail_num = tmp_data[2] + if chart_type == 'job': + count_data = { + 'date': temp_date, + 'CompleteJob': complete_num, + 'FailJob': fail_num, + } + else: + suite_fail = TestJobSuite.objects.filter(job_id__in=job_id_list, state='fail').count() + case_fail = TestJobCase.objects.filter(job_id__in=job_id_list, state='fail').count() + perf_decline = 0 + func_res_sub_case_fail_count = 0 + fail_sub_case_suite = 0 + if job_id_list: + job_ids = ",".join([str(id) for id in job_id_list]) + query_project_job = """ + SELECT + COUNT(id) + FROM + func_result + WHERE + test_job_id IN (%s) + AND sub_case_result = 2 + UNION ALL + SELECT + COUNT(id) + FROM + ( + SELECT + id + FROM + func_result + WHERE + test_job_id IN (%s) + AND sub_case_result = 2 + group by + test_job_id, + test_suite_id + ) AS T + UNION ALL + SELECT + COUNT(id) + FROM + perf_result + WHERE + test_job_id IN (%s) + AND track_result = 'decline' + """ + rows = execute_sql(query_project_job, [job_ids, job_ids, job_ids]) + if rows and len(rows) == 3 and len(rows[0]) == 1 and len(rows[1]) == 1 and len(rows[2]) == 1: + func_res_sub_case_fail_count = rows[0][0] + fail_sub_case_suite = rows[1][0] + perf_decline = rows[2][0] + count_data = { + 'date': temp_date, + 'FailSuite': suite_fail + fail_sub_case_suite, + 'FailCase': case_fail + func_res_sub_case_fail_count + perf_decline, + } + return count_data + @staticmethod def get_ws_chart_data_v2(data): project_id = data.get('project_id') -- Gitee From d4bc575545d92f2dbe0aac050ff57599adada570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 10:04:09 +0800 Subject: [PATCH 05/10] fix: sql for push job --- tone/services/sys/help_doc_services.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tone/services/sys/help_doc_services.py b/tone/services/sys/help_doc_services.py index 27f7735..e6c8e10 100644 --- a/tone/services/sys/help_doc_services.py +++ b/tone/services/sys/help_doc_services.py @@ -5,6 +5,7 @@ from django_q.tasks import async_task from tone.core.common.constant import PROD_SITE_URL from tone.core.common.services import CommonService +from tone.core.utils.common_utils import execute_sql from tone.models import HelpDoc, datetime, SiteConfig, SitePushConfig, TestJob, Comment, BaseConfig from tone.services.portal.sync_portal_services import SyncPortalService from tone.services.portal.sync_portal_task_servers import sync_job_data @@ -254,14 +255,12 @@ class TestFarmService(CommonService): job_queryset = job_queryset.filter(name__icontains=filter_job) return True, job_queryset.order_by('sync_time', 'id'), 0 else: - cursor = connection.cursor() query_cmd = '''SELECT id, name, sync_time FROM test_job - WHERE ws_id='{}' AND project_id='{}' AND - name REGEXP '{}' AND `start_time` IS NOT NULL and is_deleted=0 - ORDER BY if (`sync_time` IS NULL, 0, 1) ASC, id ASC LIMIT {}, {}'''.format( - ws_id, project_id, job_name_rule, (page_num - 1) * page_size, page_size) - cursor.execute(query_cmd) - job_data = cursor.fetchall() + WHERE ws_id=%s AND project_id=%s AND + name REGEXP %s AND `start_time` IS NOT NULL and is_deleted=0 + ORDER BY if (`sync_time` IS NULL, 0, 1) ASC, id ASC LIMIT %s, %s''' + job_data = execute_sql(query_cmd, [ws_id, project_id, job_name_rule, (page_num - 1) * page_size, + page_size]) return True, [{'id': tmp_data[0], 'name': tmp_data[1], 'sync_time': tmp_data[2]} for tmp_data in job_data], page_num -- Gitee From d91727b991ddae1c753628d42eabe561f8ef0507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 10:10:48 +0800 Subject: [PATCH 06/10] fix: sql for save report --- tone/core/handle/report_handle.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tone/core/handle/report_handle.py b/tone/core/handle/report_handle.py index dfa8b49..21d01e8 100644 --- a/tone/core/handle/report_handle.py +++ b/tone/core/handle/report_handle.py @@ -689,27 +689,27 @@ def get_server_info(tag, objs): # nq c901 baseline_id_list.append(obj_id) else: job_id_list.append(obj_id) - job_id_str = ','.join(str(e) for e in job_id_list) + job_id_str = tuple(job_id_list) if baseline_id_list: perf_base_detail = PerfBaselineDetail.objects.filter(baseline_id__in=baseline_id_list).values_list( 'test_job_id', flat=True).distinct() if perf_base_detail.exists(): - job_id_str = ','.join(str(e) for e in perf_base_detail) + job_id_str = tuple(perf_base_detail) func_base_detail = FuncBaselineDetail.objects.filter(baseline_id__in=baseline_id_list).values_list( 'test_job_id', flat=True).distinct() if func_base_detail.exists(): - job_id_str = ','.join(str(e) for e in func_base_detail) + job_id_str = tuple(func_base_detail) if job_id_str: raw_sql = 'SELECT ip,sm_name,distro,rpm_list,kernel_version,gcc,' \ 'glibc,memory_info,disk,cpu_info,ether FROM test_server_snapshot ' \ 'WHERE is_deleted=0 AND ' \ - 'job_id IN (' + job_id_str + ') AND distro IS NOT NULL UNION ' \ - 'SELECT private_ip AS ip,instance_type AS sm_name,distro,rpm_list,' \ + 'job_id IN %s AND distro IS NOT NULL UNION ' \ + 'SELECT pub_ip AS ip,instance_type AS sm_name,distro,rpm_list,' \ 'kernel_version,gcc,glibc,memory_info,disk,cpu_info,ether ' \ 'FROM cloud_server_snapshot ' \ 'WHERE is_deleted=0 AND ' \ - 'job_id IN (' + job_id_str + ') AND distro IS NOT null' - all_server_info = query_all_dict(raw_sql.replace('\'', ''), params=None) + 'job_id IN %s AND distro IS NOT null' + all_server_info = query_all_dict(raw_sql.replace('\'', ''), params=[job_id_str, job_id_str]) for server_info in all_server_info: ip = server_info.get('ip') if not (server_info.get('distro') or server_info.get('rpm_list') or server_info.get('gcc')) or \ -- Gitee From fc2ce0daadcec04d8e5b1d54076b13a786f9f253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 10:16:13 +0800 Subject: [PATCH 07/10] fix: sql for get_analytics_tags --- tone/views/api/get_analytics_tags.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tone/views/api/get_analytics_tags.py b/tone/views/api/get_analytics_tags.py index a522e8b..4f882f8 100644 --- a/tone/views/api/get_analytics_tags.py +++ b/tone/views/api/get_analytics_tags.py @@ -1,11 +1,5 @@ -# _*_ coding:utf-8 _*_ -""" -Module Description: -Date: -Author: Yfh -""" -from django.db import connection +from tone.core.utils.common_utils import execute_sql from tone.core.utils.helper import CommResp from tone.core.common.expection_handler.error_catch import api_catch_error from tone.core.common.expection_handler.error_code import ErrorCode @@ -19,10 +13,8 @@ def get_analytics_tags(request): data = request.GET project_id = data.get('project_id', None) assert project_id, ValueError(ErrorCode.PROJECT_ID_NEED) - with connection.cursor() as cursor: - cursor.execute(FILTER_JOB_TAG_SQL.format(project_id=project_id)) - rows = cursor.fetchall() - test_jobs = [row[0] for row in rows] + rows = execute_sql(FILTER_JOB_TAG_SQL, [project_id]) + test_jobs = [row[0] for row in rows] tags = list(set([job_tag.tag_id for job_tag in JobTagRelation.objects.filter(job_id__in=test_jobs)])) resp.data = [{'tag_name': tag.name, 'tag_id': tag.id} for tag in JobTag.objects.filter(id__in=tags) if tag.name != 'analytics' and tag.source_tag != 'system_tag'] -- Gitee From 00483587bb30e05e6f96ebb9ceb499b321a5d33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 10:36:57 +0800 Subject: [PATCH 08/10] fix: update django version --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index c6a57c1..416a4e2 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -3,7 +3,7 @@ certifi==2020.4.5.1 chardet==3.0.4 click==7.1.2 cryptography==3.4.7 -Django==3.2.5 +Django==3.2.14 django-cors-headers==3.3.0 django-extensions==2.2.9 djangorestframework==3.11.2 -- Gitee From 8cc8c7c8e9c17f57d45157f289e76a6e9663d915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AD=A6=E5=B3=B0?= Date: Tue, 5 Nov 2024 10:54:27 +0800 Subject: [PATCH 09/10] fix: check token for get oss url --- tone/views/api/get_oss_url.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tone/views/api/get_oss_url.py b/tone/views/api/get_oss_url.py index e4bf354..427fbfb 100644 --- a/tone/views/api/get_oss_url.py +++ b/tone/views/api/get_oss_url.py @@ -10,10 +10,14 @@ import requests from tone import settings from tone.core.utils.helper import CommResp +from tone.core.common.expection_handler.error_catch import api_catch_error +from tone.core.common.verify_token import token_required logger = logging.getLogger() +@api_catch_error +@token_required def get_path(request): resp = CommResp() path = request.GET.get('path') -- Gitee From 6d6d71a9dc802652b4b88e420de07e4f1f444db0 Mon Sep 17 00:00:00 2001 From: zhangxuefeng Date: Mon, 23 Dec 2024 11:10:28 +0800 Subject: [PATCH 10/10] fix: check path for static file --- tone/core/utils/permission_manage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tone/core/utils/permission_manage.py b/tone/core/utils/permission_manage.py index 38dd629..baa423a 100644 --- a/tone/core/utils/permission_manage.py +++ b/tone/core/utils/permission_manage.py @@ -226,6 +226,8 @@ class ValidPermission(MiddlewareMixin): status=403, data={'code': 401, 'msg': '未登录，请登录后再试'} ) + if './' in current_path or '../' in current_path: + return response_401 if 'admin/' in current_path: if settings.ALLOW_ACCESS_ADMIN_URLS and settings.ADMIN_URLS_TOKEN \ and token == settings.ADMIN_URLS_TOKEN: -- Gitee