From 0638263522afb4e3eec1e5ee1f56ca488eb737ec Mon Sep 17 00:00:00 2001 From: ydzhang Date: Fri, 20 Oct 2023 15:01:59 +0800 Subject: [PATCH 001/176] add ReleasedHotfix Model design and the backend method This commit is to upload the Released hotfix list Model design and the backend method. --- .../sysom_hotfix/apps/hotfix/models.py | 29 +++++++ .../sysom_hotfix/apps/hotfix/serializer.py | 8 +- sysom_server/sysom_hotfix/apps/hotfix/urls.py | 5 +- .../sysom_hotfix/apps/hotfix/views.py | 79 ++++++++++++++++++- sysom_server/sysom_hotfix/lib/function.py | 23 +++++- .../sysom_vul/apps/vul/async_fetch.py | 9 +++ sysom_server/sysom_vul/apps/vul/vul.py | 12 +-- 7 files changed, 155 insertions(+), 10 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/models.py b/sysom_server/sysom_hotfix/apps/hotfix/models.py index 6b2ef848..88f010b4 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/models.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/models.py @@ -60,3 +60,32 @@ class KernelVersionModel(BaseModel): class Meta: db_table = "sys_hotfix_kernelversion" ordering = ['-created_at'] + + +class ReleasedHotfixListModule(BaseModel): + serious_choice = [ + (0, u'可选安装'), + (1, u'建议安装'), + (2, u'需要安装') + ] + system_choice = [ + (0, u'调度'), + (1, u'内存'), + (2, u'网络'), + (3, u'存储'), + (4, u'其他') + ] + deprecated_choice = [ + (0, u'正常'), + (1, u'废弃') + ] + hotfix_id = models.CharField(max_length=50, verbose_name="hotfix id") + released_kernel_version = models.CharField(max_length=60, verbose_name="hotfix发布的内核版本") + serious = models.IntegerField(default=0, choices=serious_choice, verbose_name="hotfix推荐安装级别") + description = models.TextField(default="", verbose_name="hotfix问题描述") + fix_system = models.IntegerField(default=0, choices=system_choice, verbose_name="涉及子系统") + released_time = models.DateTimeField(auto_now=True, verbose_name="发布时间") + download_link = models.TextField(default="", verbose_name="hotfix下载链接") + deprecated = models.IntegerField(default=0, choices=deprecated_choice, verbose_name="该hotfix是否被废弃") + deprecated_info = models.TextField(default="", verbose_name="废弃原因或信息") + modified_time = models.DateTimeField(auto_now=True, verbose_name='记录修改时间') \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py index 25177524..91e716ba 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py @@ -1,6 +1,6 @@ from clogger import logger from rest_framework import serializers -from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel +from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel, ReleasedHotfixListModule class HotfixSerializer(serializers.ModelSerializer): @@ -20,3 +20,9 @@ class KernelSerializer(serializers.ModelSerializer): class Meta: model = KernelVersionModel fields = '__all__' + +class ReleasedHotfixSerializer(serializers.ModelSerializer): + + class Meta: + model = ReleasedHotfixListModule + fields = '__all__' diff --git a/sysom_server/sysom_hotfix/apps/hotfix/urls.py b/sysom_server/sysom_hotfix/apps/hotfix/urls.py index 0762e342..013296ec 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/urls.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/urls.py @@ -7,7 +7,7 @@ from apps.hotfix import views router = DefaultRouter() router.register('hotfix', views.HotfixAPIView) - +router.register('hotfix_info', views.ReleaseHotfixListAPIView) urlpatterns = [ path('api/v1/hotfix/create_hotfix/', views.HotfixAPIView.as_view({'post': 'create_hotfix'})), path('api/v1/hotfix/get_hotfix_list/', views.HotfixAPIView.as_view({'get': 'get_hotfixlist'})), @@ -34,5 +34,8 @@ urlpatterns = [ path('api/v1/hotfix/rebuild_hotfix/',views.HotfixAPIView.as_view({'post': 'rebuild_hotfix'})), path('api/v1/hotfix/oneclick_deploy/', views.HotfixAPIView.as_view({'post': 'oneclick_deploy'})), path('api/v1/hotfix/health_check/', views.HealthViewset.as_view({'get': 'health_check'})), + path('ap1/v1/hotfix_info/get_all_released_hotfix/', views.ReleaseHotfixListAPIView.as_view({'get': 'get_all_released_hotfix'})), + path('api/v1/hotfix_info/insert_released_hotfix_info/', views.ReleaseHotfixListAPIView.as_view({'post': 'add_one_released_hotfix'})), + path('api/v1/hotfix_info/get_filterd_released_hotfix/', views.ReleaseHotfixListAPIView.as_view({'post': 'filter_released_hotfix'})), path('api/v1/', include(router.urls)), ] diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 2f7b325f..359f8376 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -19,7 +19,7 @@ from rest_framework.viewsets import GenericViewSet import re from apps.hotfix import serializer -from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel +from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel, ReleasedHotfixListModule from lib.response import * from lib.utils import human_datetime, datetime, datetime_str from lib.exception import APIException @@ -608,4 +608,79 @@ class HotfixAPIView(GenericViewSet, class HealthViewset(CommonModelViewSet): def health_check(self, request, *args, **kwargs): - return success(result={}) \ No newline at end of file + return success(result={}) + +class ReleaseHotfixListAPIView(GenericViewSet, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.UpdateModelMixin, + mixins.DestroyModelMixin): + http_method_names = ['get', 'post', 'patch', 'delete'] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + + """this function returns all the records in database + """ + def get_all_released_hotfix(self, request): + try: + queryset = ReleasedHotfixListModule.objects.all().filter(deleted_at=None) + all_released_result = serializer.ReleasedHotfixSerializer(queryset, many=True) + return success(result=all_released_result.data, msg={"msg":"get all released hotfix success"}) + except Exception as e: + logger.error(str(e)) + return other_response(message="%s"%str(e), code=400) + + """insert one released hotfix record into database + """ + def add_one_released_hotfix(self, request): + hotfix_id = request.get("hotfix_id") + kernel_version = request.get("kernel_version") + serious = request.get("serious") + description = request.get("description") + fix_system = request.get("fix_system") + released_time = request.get("release_time") + download_link = request.get("download_link") + + try: + released_hotfix = ReleasedHotfixListModule.objects.filter(hotfix_id=hotfix_id) + if released_hotfix is None: + released_hotfix = ReleasedHotfixListModule.objects.create( + hotfix_id = hotfix_id, + released_kernel_version = kernel_version, + serious = serious, + description = description, + fix_system = fix_system, + released_time = released_time, + download_link = download_link + ) + except Exception as e: + logger.error("ERROR: insert released hotfix information.") + logger.error(str(e)) + return other_response(message="ERROR:%s"%str(e), code=400) + + return success(result={"msg":"Insert hotfix id : %s success" % hotfix_id}) + + """given parameters, filter the record in database from the given parameter + """ + def filter_released_hotfix(self, request): + search_hotfix_id = request.get("search_hotfix_id") if request.get("search_hotfix_id") is not None else None + search_kernel_version = request.get("search_kernel_version") if request.get("search_kernel_version") is not None else None + search_serious = request.get("search_serios") if request.get("search_serios") is not None else None + search_released_time = request.get("search_released_time") if request.get("search_released_time") is not None else None + search_fix_system = request.get("fix_system") if request.get("fix system") is not None else None + try: + queryset = self.function.query_released_hotfix_by_para(search_hotfix_id,search_kernel_version, + search_serious, search_released_time, + search_fix_system) + response = serializer.ReleasedHotfixSerializer(queryset, many=True) + except Exception as e: + logger.error("ERROR: query released hotfix") + logger.error(str(e)) + return other_response(message="Error when query released hotfix information") + return success(result=response.data, message="invoke get_formal_hotfixlist") + + + def sync_hotfix_information_with_build_list(self, request): + pass \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/lib/function.py b/sysom_server/sysom_hotfix/lib/function.py index 63c3a368..0da68cd1 100644 --- a/sysom_server/sysom_hotfix/lib/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -11,7 +11,7 @@ import requests import json import time -from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel +from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel, ReleasedHotfixListModule from cec_base.producer import dispatch_producer, Producer from cec_base.consumer import Consumer, dispatch_consumer from cec_base.admin import dispatch_admin @@ -426,6 +426,27 @@ class FunctionClass(): os_type_object.save() except Exception as e: logger.error(e) + + def query_released_hotfix_by_para(search_hotfix_id,search_kernel_version, + search_serious, search_released_time, search_fix_system): + try: + objects = ReleasedHotfixListModule.objects.all() + if search_hotfix_id is not None: + objects = objects.filter(hotfix_id=search_hotfix_id) + if search_kernel_version is not None: + objects = objects.filter(released_kernel_version=search_kernel_version) + if search_serious is not None: + objects = objects.filter(serious=search_serious) + if search_released_time is not None: + objects = objects.filter(released_time = search_released_time) + if search_fix_system is not None: + objects = objects.filter(fix_system=search_fix_system) + return objects + except Exception as e: + logger.error("Error when filtering released hotfix database") + logger.error("query_released_hotfix_by_para: %s " % str(e)) + return None + """ CECListener listen topic of hotfix_msg, which is send by builder diff --git a/sysom_server/sysom_vul/apps/vul/async_fetch.py b/sysom_server/sysom_vul/apps/vul/async_fetch.py index 56df8b4b..1eb8a46f 100644 --- a/sysom_server/sysom_vul/apps/vul/async_fetch.py +++ b/sysom_server/sysom_vul/apps/vul/async_fetch.py @@ -110,6 +110,7 @@ class FetchVulData: 'params': json.loads(instance.params), 'auth': auth } + 返回参数对象 """ kwargs = dict() @@ -132,6 +133,14 @@ class FetchVulData: @classmethod def _get_page_total_num(cls, kwargs) -> Union[bool, int]: + """向漏洞库请求数据 + + Args: + kwargs (_type_): 创建的结构化参数 + + Returns: + Union[bool, int]: 如果请求成功,返回请求数据的从页;失败返回False + """ response = requests.request(**kwargs) if response.status_code == 200: result = response.json() diff --git a/sysom_server/sysom_vul/apps/vul/vul.py b/sysom_server/sysom_vul/apps/vul/vul.py index aaf7289a..c470c8ef 100644 --- a/sysom_server/sysom_vul/apps/vul/vul.py +++ b/sysom_server/sysom_vul/apps/vul/vul.py @@ -39,21 +39,23 @@ def update_vul_db(): 更新漏洞数据库数据 """ logger.info("Begin to get vul db address") - vul_addrs = VulAddrModel.objects.all() + vul_addrs = VulAddrModel.objects.all() # 获取漏洞数据库中所有的漏洞库信息 for vul_addr in vul_addrs: logger.info("Try to get vul db info") - vul_addr_obj = VulDataParse(vul_addr) + vul_addr_obj = VulDataParse(vul_addr) # 生成漏洞库操作实例 try: - for res in vul_addr_obj._get_vul_data(): + for res in vul_addr_obj._get_vul_data(): # 获取每一项cve数据,解析返回数据,根据cve是否存在进行更新或者插入 vul_addr_obj.parse_and_store_vul_data(res) except Exception as e: logger.warning(e) logger.warning(f"failed in {vul_addr.url}") - + """ + VulDataParse:CVE漏洞库操作实例 + """ class VulDataParse(object): def __init__(self, vul_addr_obj: VulAddrModel): - self.vul_addr_obj = vul_addr_obj + self.vul_addr_obj = vul_addr_obj self.cve_data_path = list(filter(None, self._parser_cve_item_path)) @property -- Gitee From d00df7dd4121fbea1092df2a3ee9278a541bb848 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Thu, 16 Nov 2023 15:24:30 +0800 Subject: [PATCH 002/176] Upload backend function of released hotfix information This commit upload backend function of released hotfix list information: 1. insert one released hotfix information 2. update released hotfix information 3. get all released hotfix information This commit contains adjust the database model and urls. --- .../0005_releasedhotfixlistmodule.py | 37 ++++ .../sysom_hotfix/apps/hotfix/models.py | 9 +- sysom_server/sysom_hotfix/apps/hotfix/urls.py | 15 +- .../sysom_hotfix/apps/hotfix/views.py | 205 ++++++++++++++++-- sysom_server/sysom_hotfix/lib/function.py | 13 +- 5 files changed, 251 insertions(+), 28 deletions(-) create mode 100644 sysom_server/sysom_hotfix/apps/hotfix/migrations/0005_releasedhotfixlistmodule.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/migrations/0005_releasedhotfixlistmodule.py b/sysom_server/sysom_hotfix/apps/hotfix/migrations/0005_releasedhotfixlistmodule.py new file mode 100644 index 00000000..98e413d5 --- /dev/null +++ b/sysom_server/sysom_hotfix/apps/hotfix/migrations/0005_releasedhotfixlistmodule.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.16 on 2023-11-16 06:14 + +from django.db import migrations, models +import lib.utils + + +class Migration(migrations.Migration): + + dependencies = [ + ('hotfix', '0004_auto_20231020_1552'), + ] + + operations = [ + migrations.CreateModel( + name='ReleasedHotfixListModule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.CharField(default=lib.utils.human_datetime, max_length=20, verbose_name='创建时间')), + ('deleted_at', models.CharField(max_length=20, null=True)), + ('hotfix_id', models.CharField(max_length=50, verbose_name='hotfix id')), + ('released_kernel_version', models.CharField(max_length=60, verbose_name='hotfix发布的内核版本')), + ('serious', models.IntegerField(choices=[(0, '可选安装'), (1, '建议安装'), (2, '需要安装')], default=0, verbose_name='hotfix推荐安装级别')), + ('description', models.TextField(default='', verbose_name='hotfix问题描述')), + ('fix_system', models.IntegerField(choices=[(0, '调度'), (1, '内存'), (2, '网络'), (3, '存储'), (4, '其他')], default=0, verbose_name='涉及子系统')), + ('released_time', models.DateTimeField(verbose_name='发布时间')), + ('download_link', models.TextField(default='', verbose_name='hotfix下载链接')), + ('deprecated', models.IntegerField(choices=[(0, '正常'), (1, '废弃')], default=0, verbose_name='该hotfix是否被废弃')), + ('deprecated_info', models.TextField(default='', verbose_name='废弃原因或信息')), + ('modified_time', models.DateTimeField(auto_now=True, verbose_name='记录修改时间')), + ('modified_user', models.CharField(default='', max_length=20, null=True, verbose_name='用于记录最后一次修改的人')), + ], + options={ + 'db_table': 'sys_released_hotfix', + 'ordering': ['-created_at'], + }, + ), + ] diff --git a/sysom_server/sysom_hotfix/apps/hotfix/models.py b/sysom_server/sysom_hotfix/apps/hotfix/models.py index 88f010b4..0119fa9b 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/models.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/models.py @@ -84,8 +84,13 @@ class ReleasedHotfixListModule(BaseModel): serious = models.IntegerField(default=0, choices=serious_choice, verbose_name="hotfix推荐安装级别") description = models.TextField(default="", verbose_name="hotfix问题描述") fix_system = models.IntegerField(default=0, choices=system_choice, verbose_name="涉及子系统") - released_time = models.DateTimeField(auto_now=True, verbose_name="发布时间") + released_time = models.DateTimeField(auto_now=False, verbose_name="发布时间") download_link = models.TextField(default="", verbose_name="hotfix下载链接") deprecated = models.IntegerField(default=0, choices=deprecated_choice, verbose_name="该hotfix是否被废弃") deprecated_info = models.TextField(default="", verbose_name="废弃原因或信息") - modified_time = models.DateTimeField(auto_now=True, verbose_name='记录修改时间') \ No newline at end of file + modified_time = models.DateTimeField(auto_now=True, verbose_name='记录修改时间') + modified_user = models.CharField(default="", max_length=20, null=True, verbose_name="用于记录最后一次修改的人") + + class Meta: + db_table = "sys_released_hotfix" + ordering = ['-created_at'] \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/urls.py b/sysom_server/sysom_hotfix/apps/hotfix/urls.py index 013296ec..09e512fb 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/urls.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/urls.py @@ -1,13 +1,14 @@ -from django.urls import path +from django.urls import path, include, re_path from django.urls.conf import include from rest_framework.routers import DefaultRouter from apps.hotfix import views -router = DefaultRouter() +router = DefaultRouter() router.register('hotfix', views.HotfixAPIView) -router.register('hotfix_info', views.ReleaseHotfixListAPIView) +# router.register('releasehotfix', views.ReleaseHotfixListAPIView, basename='releasehotfix') + urlpatterns = [ path('api/v1/hotfix/create_hotfix/', views.HotfixAPIView.as_view({'post': 'create_hotfix'})), path('api/v1/hotfix/get_hotfix_list/', views.HotfixAPIView.as_view({'get': 'get_hotfixlist'})), @@ -34,8 +35,10 @@ urlpatterns = [ path('api/v1/hotfix/rebuild_hotfix/',views.HotfixAPIView.as_view({'post': 'rebuild_hotfix'})), path('api/v1/hotfix/oneclick_deploy/', views.HotfixAPIView.as_view({'post': 'oneclick_deploy'})), path('api/v1/hotfix/health_check/', views.HealthViewset.as_view({'get': 'health_check'})), - path('ap1/v1/hotfix_info/get_all_released_hotfix/', views.ReleaseHotfixListAPIView.as_view({'get': 'get_all_released_hotfix'})), - path('api/v1/hotfix_info/insert_released_hotfix_info/', views.ReleaseHotfixListAPIView.as_view({'post': 'add_one_released_hotfix'})), - path('api/v1/hotfix_info/get_filterd_released_hotfix/', views.ReleaseHotfixListAPIView.as_view({'post': 'filter_released_hotfix'})), + path('api/v1/hotfix/get_all_released_hotfix/', views.ReleaseHotfixListAPIView.as_view({'get': 'get_all_released_hotfix'})), + path('api/v1/hotfix/insert_released_hotfix_info/', views.ReleaseHotfixListAPIView.as_view({'post': 'add_one_released_hotfix'})), + path('api/v1/hotfix/get_filtered_released_hotfix/', views.ReleaseHotfixListAPIView.as_view({'get': 'filter_released_hotfix'})), + path('api/v1/hotfix/import_from_tablefile/', views.ReleaseHotfixListAPIView.as_view({'post': 'import_from_table'})), + path('api/v1/hotfix/update_one_released_hotfix_info/', views.ReleaseHotfixListAPIView.as_view({'post': 'update_released_hotfix_record'})), path('api/v1/', include(router.urls)), ] diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 359f8376..89d73a16 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -16,7 +16,8 @@ from django_filters.rest_framework import DjangoFilterBackend from rest_framework.exceptions import ValidationError from django.conf import settings from rest_framework.viewsets import GenericViewSet -import re +import sys +import pandas as pd from apps.hotfix import serializer from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel, ReleasedHotfixListModule @@ -91,7 +92,6 @@ class SaveUploadFile(APIView): raise APIException(message=f"Upload Failed: {e}") return success(result={"patch_name":patch_file_name}, message="Upload success") - class HotfixAPIView(GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, @@ -617,6 +617,9 @@ class ReleaseHotfixListAPIView(GenericViewSet, mixins.UpdateModelMixin, mixins.DestroyModelMixin): http_method_names = ['get', 'post', 'patch', 'delete'] + serializer_class = serializer.ReleasedHotfixSerializer + filter_backends = [DjangoFilterBackend] + http_method_names = ['get', 'post', 'patch', 'delete'] def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -627,37 +630,48 @@ class ReleaseHotfixListAPIView(GenericViewSet, try: queryset = ReleasedHotfixListModule.objects.all().filter(deleted_at=None) all_released_result = serializer.ReleasedHotfixSerializer(queryset, many=True) - return success(result=all_released_result.data, msg={"msg":"get all released hotfix success"}) except Exception as e: logger.error(str(e)) return other_response(message="%s"%str(e), code=400) + return success(result=all_released_result.data, msg={"msg":"get all released hotfix success"}) """insert one released hotfix record into database """ def add_one_released_hotfix(self, request): - hotfix_id = request.get("hotfix_id") - kernel_version = request.get("kernel_version") - serious = request.get("serious") - description = request.get("description") - fix_system = request.get("fix_system") - released_time = request.get("release_time") - download_link = request.get("download_link") + hotfix_id = request.data["hotfix_id"] + released_kernel_version = request.data["released_kernel_version"] + serious = request.data["serious"] + description = request.data["description"] + fix_system = request.data["fix_system"] + released_time = request.data["release_time"] + download_link = request.data["download_link"] + # the above information must included + deprecated = request.data["deprecated"] + deprecated_info = request.data["deprecated_info"] + + if (deprecated > 0 and len(deprecated_info) == 0) or (deprecated == 0 and len(deprecated_info) > 0): + logger.error("{} : {}".format(sys._getframe().f_code.co_name, "Invalid deprecated and info status!")) + return other_response(message=("Invalid deprecated and info status!"), code=400) try: - released_hotfix = ReleasedHotfixListModule.objects.filter(hotfix_id=hotfix_id) + # one hotfix id with same kernel version should view as the same hotfix + released_hotfix = ReleasedHotfixListModule.objects.filter(hotfix_id=hotfix_id).filter(released_kernel_version=released_kernel_version).first() if released_hotfix is None: released_hotfix = ReleasedHotfixListModule.objects.create( hotfix_id = hotfix_id, - released_kernel_version = kernel_version, + released_kernel_version = released_kernel_version, serious = serious, description = description, fix_system = fix_system, released_time = released_time, - download_link = download_link + download_link = download_link, + deprecated = deprecated, + deprecated_info = deprecated_info ) + else: + self.update_released_hotfix_record(request=request) except Exception as e: - logger.error("ERROR: insert released hotfix information.") - logger.error(str(e)) + logger.error("ERROR: insert released hotfix information. Exception: %s" % str(e) ) return other_response(message="ERROR:%s"%str(e), code=400) return success(result={"msg":"Insert hotfix id : %s success" % hotfix_id}) @@ -676,11 +690,164 @@ class ReleaseHotfixListAPIView(GenericViewSet, search_fix_system) response = serializer.ReleasedHotfixSerializer(queryset, many=True) except Exception as e: - logger.error("ERROR: query released hotfix") - logger.error(str(e)) + logger.error("{} : ERROR: query released hotfix. Exception: {}".format(sys._getframe().f_code.co_name, str(e))) return other_response(message="Error when query released hotfix information") return success(result=response.data, message="invoke get_formal_hotfixlist") + """this function is used to update one hotfix record + However, the information inside just can update, but cannot delete! + You can update it into blank message " ", but I hope this message should never be deleted! + """ + def update_released_hotfix_record(self, request): + database_id = request.data["id"] if request.data["id"] is not None else None + hotfix_id = request.data['hotfix_id'] + serious = request.data["serious"] + description = request.data["description"] + fix_system = request.data["fix_system"] + released_time = request.data["release_time"] + download_link = request.data["download_link"] + # the above information must included + deprecated = request.data["deprecated"] + deprecated_info = request.data["deprecated_info"] + + if (deprecated == 0 and len(deprecated_info) > 0) or (deprecated > 0 and len(deprecated_info) == 0): + logger.error("{} : Error : deprecated status and deprecated_info is not the same!".format(sys._getframe().f_code.co_name)) + return other_response(message="Different derepcated status of info and status, id : {}".format(str(hotfix_id))) + + if database_id is None: + logger.error("{} : Error : update relased hotfix with no database id!".format(sys._getframe().f_code.co_name)) + return other_response(message="Error: {} : Cannot update hotfix info with no hotfix id".format(sys._getframe().f_code.co_name)) - def sync_hotfix_information_with_build_list(self, request): - pass \ No newline at end of file + try: + released_hotfix = ReleasedHotfixListModule.objects.filter(id=database_id).first() + if released_hotfix is None: + logger.error("{}: Error: {}".format(sys._getframe().f_code.co_name, "using release hotfix is not exist!")) + return other_response(message="This released hotfix is not exist!",code=400) + released_hotfix.serious = serious + released_hotfix.description = description + released_hotfix.fix_system = fix_system + released_hotfix.released_time = released_time + released_hotfix.download_link = download_link + if deprecated is not None: + released_hotfix.deprecated = deprecated + released_hotfix.deprecated_info = deprecated_info + released_hotfix.save() + except Exception as e: + logger.error("{} : Exception : {}".format(sys._getframe().f_code.co_name, str(e))) + return other_response(message="Error when updating hotfix info, id : %s" % str(hotfix_id)) + + return success(result={"msg":"success updating released hotfix information, id : {}".format(hotfix_id)}) + + + """ + Import Released Hotfix Information from table file + eg. Excel, csv, etc. + """ + def import_from_table(self, request): + # upload the table file + table_file = request.data.get('file', None) + catalogue = request.data.get('catalogue', None) + if not table_file: + return APIException(message="Upload Failed: file required!") + requirement_file_type = ["csv", "xlsx", "xls"] + file_upload_name = table_file.name + file_upload_type = file_upload_name.split(".")[-1] + if file_upload_type not in requirement_file_type: + return other_response(msg=f"file type error, we can not support .{file_upload_type}, please upload correct file type", code=400) + tmp_file_path = "/tmp/" + if not os.path.exists(tmp_file_path): + os.makedirs(tmp_file_path) + patch_file_name = table_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S') + "." + file_upload_type + file_path = os.path.join(tmp_file_path, patch_file_name) + try: + with open(file_path, 'wb') as f: + for chunk in file_path.chunks(): + f.write(chunk) + except Exception as e: + logger.error(e) + raise APIException(message=f"Upload Failed: {e}") + + try: + # from now, read the table file data + if file_upload_type in ["csv"]: + # csv file type + df = pd.read_csv(file_path) + all_results = ReleasedHotfixListModule.objects.all() + for index, row in df.iterrows(): + hotfix_id = row[1] + released_kernel_version = row[2] + serious = row[3] + description = row[4] + fix_system = row[5] + released_time = row[6] + download_link = row[7] + deprecated = row[8] + deprecated_info = row[9] + + record = all_results.filter(hotfix_id=hotfix_id).filter(released_kernel_version=released_kernel_version).first() + # if this hotfix with the same kernel is exist, update it + if record: + record.serious = serious + record.description = description + record.fix_system = fix_system + record.released_time = released_time + record.download_link = download_link + record.deprecated = deprecated + record.deprecated_info = deprecated_info + record.save() + else: + # this released record is not exist, create it + record = ReleasedHotfixListModule.objects.create( + hotfix_id = hotfix_id, + released_kernel_version = released_kernel_version, + serious = serious, + description = description, + fix_system = fix_system, + released_time = released_time, + download_link = download_link, + deprecated = deprecated, + deprecated_info = deprecated_info + ) + else: + # excel file type + df = pd.read_excel(file_path) + for index, row in df.iterrows(): + hotfix_id = row[1] + released_kernel_version = row[2] + serious = row[3] + description = row[4] + fix_system = row[5] + released_time = row[6] + download_link = row[7] + deprecated = row[8] + deprecated_info = row[9] + + record = all_results.filter(hotfix_id=hotfix_id).filter(released_kernel_version=released_kernel_version).first() + # if this hotfix with the same kernel is exist, update it + if record: + record.serious = serious + record.description = description + record.fix_system = fix_system + record.released_time = released_time + record.download_link = download_link + record.deprecated = deprecated + record.deprecated_info = deprecated_info + record.save() + else: + # this released record is not exist, create it + record = ReleasedHotfixListModule.objects.create( + hotfix_id = hotfix_id, + released_kernel_version = released_kernel_version, + serious = serious, + description = description, + fix_system = fix_system, + released_time = released_time, + download_link = download_link, + deprecated = deprecated, + deprecated_info = deprecated_info + ) + except Exception as e: + logger.error("Error when sync released hotfix record from file...") + return other_response(message="{} : Error when sync released hotfix record from file...".format(sys._getframe().f_code.co_name), code=400) + + return success(result={"msg":"success sync released hotfix record from file..."}) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/lib/function.py b/sysom_server/sysom_hotfix/lib/function.py index 0da68cd1..0d56b7ca 100644 --- a/sysom_server/sysom_hotfix/lib/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -554,4 +554,15 @@ class CECListener(): except Exception as e: logger.error(str(e)) - \ No newline at end of file + +""" +Hotfix Server Exception +""" +class HotfixServerException(Exception): + + def __init__(self, *args: object) -> None: + super().__init__(*args) + + @staticmethod + def msg(self, msg: str) -> str: + return msg \ No newline at end of file -- Gitee From 8a0bf982dc9ef0293ecee918080035dcee80b58b Mon Sep 17 00:00:00 2001 From: ydzhang Date: Fri, 17 Nov 2023 11:47:25 +0800 Subject: [PATCH 003/176] (Bugfix) remove code of handling git_rule This commit remove the code of handling git_rule. Because git_rule function is abandoned. If this code continue exits, it will cause None type error. --- sysom_server/sysom_hotfix/apps/hotfix/views.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 89d73a16..055c82a8 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -558,14 +558,6 @@ class HotfixAPIView(GenericViewSet, os_type_object.source_debuginfo = request.data['source_debuginfo'] # src_pkg_mark = request.data.get("src_pkg_mark", None) git_rule = request.data.get("git_rule", None) - logger.info(os_type_object.git_rule) - logger.info(git_rule) - if git_rule != os_type_object.git_rule: - patch_file_repo = os.path.join(settings.HOTFIX_FILE_BRANCH_RULE) - file_path = os.path.join(patch_file_repo, git_rule) - logger.info(file_path) - os.remove(file_path) - os_type_object.git_rule = request.data['git_rule'] os_type_object.save() thread_runner = threading.Thread(target=self.function.sync_kernel, name="sync_kernel",args=(os_type_object.id,)) thread_runner.start() -- Gitee From 2f28a2b53407218b4ac0a3b71f8c2f3922b079fb Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 20 Nov 2023 10:45:29 +0800 Subject: [PATCH 004/176] fix(web.diagnosis): channel filter not effect --- sysom_web/src/pages/diagnose/diagnose.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysom_web/src/pages/diagnose/diagnose.jsx b/sysom_web/src/pages/diagnose/diagnose.jsx index 97ea0181..f8da5225 100644 --- a/sysom_web/src/pages/diagnose/diagnose.jsx +++ b/sysom_web/src/pages/diagnose/diagnose.jsx @@ -113,7 +113,7 @@ const Diagnose = (props) => { /> - { const recorded = record; const msg = await getTask(record.task_id); -- Gitee From e13dee0a9f6771d397f8a5557eacead911fae66c Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 20 Nov 2023 15:56:14 +0800 Subject: [PATCH 005/176] fix(web.diagnosis): Offline import does not carry diagnostic parameters --- .../components/OfflineImportModal.jsx | 70 +++++++---- sysom_web/src/pages/diagnose/diagnose.jsx | 111 ++++++++++-------- 2 files changed, 109 insertions(+), 72 deletions(-) diff --git a/sysom_web/src/pages/diagnose/components/OfflineImportModal.jsx b/sysom_web/src/pages/diagnose/components/OfflineImportModal.jsx index 51495a6a..d85190b4 100644 --- a/sysom_web/src/pages/diagnose/components/OfflineImportModal.jsx +++ b/sysom_web/src/pages/diagnose/components/OfflineImportModal.jsx @@ -1,4 +1,4 @@ -import { ModalForm, ProFormText, ProFormTextArea, ProFormSelect } from '@ant-design/pro-form'; +import { ModalForm, ProFormTextArea, ProFormDigit, ProFormText, ProFormSelect } from '@ant-design/pro-form'; import { forwardRef } from 'react'; import * as PropTypes from 'prop-types'; import { useIntl, FormattedMessage } from 'umi'; @@ -18,7 +18,10 @@ let OfflineImportModal = (props, ref) => { visible, modalWidth, onVisibleChange, - onFinish + onFinish, + taskForm, + serviceName, + queryParams, } = props; const intl = useIntl(); @@ -46,29 +49,50 @@ let OfflineImportModal = (props, ref) => { }) }} > +